Promote lucy to a TLP -- INFRA-4590
git-svn-id: https://svn.apache.org/repos/asf/lucy/branches/0.1@1305401 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/CHANGES b/CHANGES
new file mode 100644
index 0000000..c7c7ca3
--- /dev/null
+++ b/CHANGES
@@ -0,0 +1,52 @@
+Revision history for Lucy
+
+0.1.1 XXXX-XX-XX
+
+ Bugfixes:
+
+ * Bad prototype for MAKE_MESS when no variadic macros available.
+ https://issues.apache.org/jira/browse/LUCY-152
+ * XSLoader version string mismatch.
+ https://issues.apache.org/jira/browse/LUCY-153
+ * Double-quote all #error directives.
+ https://issues.apache.org/jira/browse/LUCY-154
+ * TestFSFileHandle's Close() test on Windows fails to clean up.
+ https://issues.apache.org/jira/browse/LUCY-155
+ * Address GCC warnings for Charmonizer code
+ https://issues.apache.org/jira/browse/LUCY-156
+ * Add -pthread linker flag on OpenBSD
+ https://issues.apache.org/jira/browse/LUCY-157
+ * Disable LockFreeRegistry test by default.
+ https://issues.apache.org/jira/browse/LUCY-158
+ * Always disable symlink tests on Windows
+ https://issues.apache.org/jira/browse/LUCY-160
+ * Always use CreateHardLink() on Windows
+ https://issues.apache.org/jira/browse/LUCY-161
+ * Use shell to redirect Charmonizer stderr on Windows
+ https://issues.apache.org/jira/browse/LUCY-162
+ * Target Windows XP
+ https://issues.apache.org/jira/browse/LUCY-163
+ * Skip forking tests under Cygwin
+ https://issues.apache.org/jira/browse/LUCY-164
+ * Line-ending-agnostic parsing of sample docs
+ https://issues.apache.org/jira/browse/LUCY-165
+ * Prefer POSIX over windows.h for process ID
+ https://issues.apache.org/jira/browse/LUCY-166
+ * INCREF/DECREF symbol collisions under Windows in FSDirHandle.c
+ https://issues.apache.org/jira/browse/LUCY-167
+ * Directory handling under Cygwin
+ https://issues.apache.org/jira/browse/LUCY-168
+ * Improve cleanup after MSVC
+ https://issues.apache.org/jira/browse/LUCY-169
+ * Charmonizer test compiles should use obj rather than exe when possible
+ https://issues.apache.org/jira/browse/LUCY-170
+ * Turn off stupid MSVC warnings
+ https://issues.apache.org/jira/browse/LUCY-171
+ * Clownfish should slurp files in text mode
+ https://issues.apache.org/jira/browse/LUCY-172
+
+
+0.1.0 2011-05-20
+
+ Initial release, adapted from a software grant for the KinoSearch codebase.
+
diff --git a/INSTALL b/INSTALL
new file mode 100644
index 0000000..70fc164
--- /dev/null
+++ b/INSTALL
@@ -0,0 +1,10 @@
+Build Instructions
+==================
+
+Apache Lucy is a C library targeted at dynamic languages. Currently it is
+available via Perl bindings.
+
+To build Lucy with Perl bindings, chdir to the "perl" subdirectory underneath
+the "lucy" top level and follow the instructions in the INSTALL document
+there.
+
diff --git a/KEYS b/KEYS
new file mode 100644
index 0000000..778bfe8
--- /dev/null
+++ b/KEYS
@@ -0,0 +1,56 @@
+This file contains the PGP keys of various developers that work on
+the Apache Lucy project.
+
+Please don't use these keys for email unless you have asked the owner
+because some keys are only used for code signing.
+
+Please realize that this file itself or the public key servers may be
+compromised.
+
+Apache users: pgp < KEYS
+Apache developers:
+ (pgpk -ll <your name> && pgpk -xa <your name>) >> this file.
+ or
+ (gpg --fingerprint --list-sigs <your name>
+ && gpg --armor --export <your name>) >> this file.
+
+Apache developers: please ensure that your key is also available via the
+PGP keyservers (such as pgp.mit.edu).
+
+
+pub 1024D/B876884A 2007-12-24
+uid Chris Mattmann (CODE SIGNING KEY) <mattmann@apache.org>
+sig 3 B876884A 2007-12-24 Chris Mattmann (CODE SIGNING KEY) <mattmann@apache.org>
+sub 2048g/D3B4F350 2007-12-24
+sig B876884A 2007-12-24 Chris Mattmann (CODE SIGNING KEY) <mattmann@apache.org>
+
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1.4.8 (Darwin)
+
+mQGiBEdvL9QRBACuaV06by+pxZHXIxBsfAFYJk7XJgsqR23m5ClCDPusMeaI4XGB
+eU8Nw4iVwgG3p5VLWLXeMIm/KPz3pmxiNyEP/dHoDxOPR+hAqlP5v03D1iK19H7q
+46BIecIwo8q0ei70fBLvMQN+apIFlvYDqVCTm1lxoCQafagqd9p2JtTf+wCg70yM
+nGtrejB+ZTTcb08f7SAHsLED/11vIdcxViN3u+3klhbb99bd/g9KvCU/I/7+MDx1
+3zrSvJV2b2wrxabUJ1Oxsb4/4BXq8A1FyhC1h/d2PsawqiY0GZ02cucbzEmdXH51
+UnrRLM9/txtZ2b7V6YkDmPf0k6rD0SjqAAy1ERekEVUOxnY4sPGmJoyac4j9+pO9
+1vH/A/9LRoJlPTfv/mFYty6/Egckhv48YoRUBo1dNh6IPQY0oVpAFbcXc3GiTyCu
+5iQp7utxP7hoJTUM2Hn5tF9D7IniRC9wsrcW8Gi/f82O4HlmyV4+Tt75nWx018oI
+ObGmwitT27EkOnFcQc9F+Q53nKr+a22SBbpfffF9Xdbkw7V73bQ3Q2hyaXMgTWF0
+dG1hbm4gKENPREUgU0lHTklORyBLRVkpIDxtYXR0bWFubkBhcGFjaGUub3JnPohg
+BBMRAgAgBQJHby/UAhsDBgsJCAcDAgQVAggDBBYCAwECHgECF4AACgkQcPCcxrh2
+iEr8KwCffMIKMu3TBrGZVu1BPLbMBhjsrl8AoI15rg+tzYZZmZJD6tDS40klTsVA
+uQINBEdvL9QQCAClHjwXMu38iDR3nvbYkWmcz5rfBFvDm/KVQGLnnY96C1r890Ir
+cHxAlSpbGb6qPi5n27v87LoS2bYEitqCUUwB7AQLOgqmLvqMJ4qp5HUfTQ/wH9Br
+wK2LX1oGFJXH14lbZ7xW36n9A/JtXHY8vGz3GuDvKYqbdOCFo8fBLwotdFOHhNYy
+bBYS1G4gtmemXwzH8kcuoIW6LuoRNxluHi1tJGFC1F1uBoxKir7F7BC38DDNvhak
+dSJpm3WxFkEEkIUyIERVGVRoFzLlk72W0R3kZVvnXbtgPklTg/2Sy13Gb+MzTBYt
+5TF841neM/kHdgt45EgBhchHN3Ys3ljabihbAAMFB/4ke4Xe573V78UR/WTMUzfw
+pIysMUzEjNKqOfnAoNnR4WDDca4MwIUl62QqGTRrWZxTD8fAGYxc+m0qmygGKtYq
+LUYB5N/pLGu1sg2j23G8aBKthiCCE+jOr3uebU/j0BTzN/BwXCqIGogELFlPC5Tj
+Hr6c8LpkRFIOjVfuYB2TV4o2RfSFzrSFHCbrU82ojxhYSwyqDGAdD6EGtbbqaEMX
+tGZzHaMVm2gDeV9W2veurxOulgndNg2+FXvgUlOa+KZ2J2DxNBcJv1uBtDAWDyR9
+dTgTbK62ZnSjsnRYbgf0HdA+kW9n9XBMEHwgYk0q+doOWUOQFqC84TgrrhyDd1XZ
+iEkEGBECAAkFAkdvL9QCGwwACgkQcPCcxrh2iEplXwCgraY3ELlDStqpJDSUzVsN
+rGuNiwsAoKz92ycEjcMnoLnX8AaPADdo1m/P
+=zEfO
+-----END PGP PUBLIC KEY BLOCK-----
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..9cb2bfe
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,332 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+Some code in core/Lucy/Util/StringHelper2.c was derived from ICU4C,
+available from <http://icu-project.org/>. Here is the license for those
+materials:
+
+ ICU License - ICU 1.8.1 and later
+
+ COPYRIGHT AND PERMISSION NOTICE
+
+ Copyright (c) 1995-2010 International Business Machines Corporation and others
+
+ All rights reserved.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"),
+ to deal in the Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, and/or sell
+ copies of the Software, and to permit persons
+ to whom the Software is furnished to do so, provided that the above
+ copyright notice(s) and this permission notice appear in all copies
+ of the Software and that both the above copyright notice(s) and this
+ permission notice appear in supporting documentation.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+ PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL
+ THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM,
+ OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER
+ RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+ NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE
+ USE OR PERFORMANCE OF THIS SOFTWARE.
+
+ Except as contained in this notice, the name of a copyright holder shall not be
+ used in advertising or otherwise to promote the sale, use or other dealings in
+ this Software without prior written authorization of the copyright holder.
+
+ All trademarks and registered trademarks mentioned herein are the property
+ of their respective owners.
+
+
+Some code in core/Lucy/Util/StringHelper3.c was derived using data files
+from the Unicode Consortium under <http://www.unicode.org/Public/>. Here is
+the license for those materials:
+
+ EXHIBIT 1 UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+
+ Unicode Data Files include all data files under the directories
+ http://www.unicode.org/Public/, http://www.unicode.org/reports/, and
+ http://www.unicode.org/cldr/data/ . Unicode Software includes any source code
+ published in the Unicode Standard or under the directories
+ http://www.unicode.org/Public/, http://www.unicode.org/reports/, and
+ http://www.unicode.org/cldr/data/.
+
+ NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING,
+ INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA FILES ("DATA
+ FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO
+ BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT
+ AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR
+ SOFTWARE.
+
+ COPYRIGHT AND PERMISSION NOTICE
+
+ Copyright 1991-2010 Unicode, Inc. All rights reserved. Distributed under the
+ Terms of Use in http://www.unicode.org/copyright.html.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy of
+ the Unicode data files and any associated documentation (the "Data Files") or
+ Unicode software and any associated documentation (the "Software") to deal in
+ the Data Files or Software without restriction, including without limitation
+ the rights to use, copy, modify, merge, publish, distribute, and/or sell copies
+ of the Data Files or Software, and to permit persons to whom the Data Files or
+ Software are furnished to do so, provided that (a) the above copyright
+ notice(s) and this permission notice appear with all copies of the Data Files
+ or Software, (b) both the above copyright notice(s) and this permission notice
+ appear in associated documentation, and (c) there is clear notice in each
+ modified Data File or in the Software as well as in the documentation
+ associated with the Data File(s) or Software that the data or software has been
+ modified.
+
+ THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD
+ PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN
+ THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+ DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+ WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR
+ SOFTWARE.
+
+ Except as contained in this notice, the name of a copyright holder shall not be
+ used in advertising or otherwise to promote the sale, use or other dealings in
+ these Data Files or Software without prior written authorization of the
+ copyright holder.
+
+ Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
+ registered in some jurisdictions. All other trademarks and registered
+ trademarks mentioned herein are the property of their respective owners.
+
+Portions of the Snowball stemming library are bundled with this distribution
+under modules/analysis/snowstem and modules/analysis/snowstop. Here is the
+license for those materials:
+
+ Copyright (c) 2001, Dr Martin Porter
+ Copyright (c) 2002, Richard Boulton
+ All rights reserved.
+
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice,
+ this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+ * Neither the name of the copyright holders nor the names of its contributors
+ may be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+ FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..5253ee3
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,19 @@
+Apache Lucy
+Copyright 2010-2011 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This software contains code derived from ICU4C, Copyright (c) 1995-2010
+International Business Machines Corporation and others.
+
+This software contains code derived from Unicode data available from
+<http://www.unicode.org/Public/> and Copyright 1991-2010 Unicode, Inc.
+
+This software bundles code developed by the Snowball project at
+<http://snowball.tartarus.org>, Copyright (c) 2001, Dr Martin Porter and
+Copyright (c) 2002, Richard Boulton.
+
+This product bundles the text of the Constitution of the United States of
+America, which is in the public domain.
+
diff --git a/README b/README
new file mode 100644
index 0000000..9b189ad
--- /dev/null
+++ b/README
@@ -0,0 +1,24 @@
+Apache Lucy
+===========
+
+OVERVIEW
+
+Apache Lucy is a fulltext search engine library, written in C and targeted at
+dynamic language users.
+
+DISCLAIMER
+
+Apache Lucy is an effort undergoing incubation at The Apache Software
+Foundation (ASF), sponsored by the Incubator. Incubation is required of all
+newly accepted projects until a further review indicates that the
+infrastructure, communications, and decision making process have stabilized in
+a manner consistent with other successful ASF projects. While incubation
+status is not necessarily a reflection of the completeness or stability of the
+code, it does indicate that the project has yet to be fully endorsed by the
+ASF.
+
+INSTALLATION
+
+To build and install Lucy, follow the instructions in the accompanying
+"INSTALL" document.
+
diff --git a/charmonizer/README b/charmonizer/README
new file mode 100644
index 0000000..389e72c
--- /dev/null
+++ b/charmonizer/README
@@ -0,0 +1,110 @@
+NAME
+
+ Charmonizer - Use C to configure C.
+
+OVERVIEW
+
+ Charmonizer is a tool for probing, normalizing, and testing the build
+ environment provided by a C compiler and an operating system. It works by
+ attempting to compile lots of little programs and analyzing the output
+ from those that build successfully. `
+
+ Charmonizer modules are ordinary ANSI C files, and the configuration
+ application that you write is an ordinary C executable.
+
+REQUIREMENTS
+
+ Charmonizer's only prerequisite is an ISO C90-compliant compiler which can
+ be invoked from C via the system() command.
+
+PROBING
+
+ #include "Charmonizer/Probe.h"
+ #include "Charmonizer/Probe/Integers.h"
+ #include "Charmonizer/Probe/LargeFiles.h"
+
+ int main()
+ {
+ /* Tell Charmonizer about your OS and compiler. */
+ chaz_Probe_init("cc", "-I/usr/local/include", NULL);
+
+ /* Run desired Charmonizer modules. */
+ chaz_Integers_run();
+ chaz_LargeFiles_run();
+
+ /* Tear down. */
+ chaz_Probe_clean_up();
+
+ return 0;
+ }
+
+ The purpose of Charmonizer's probing toolset is to generate a single C
+ header file called "charmony.h", gathering together information that is
+ ordinarily only available at runtime and assigning predictable names to
+ functionality which may go by many different aliases on different systems.
+
+ One header file, "Charmonizer/Probe.h", provides the primary interface and
+ a suite of topically oriented modules -- e.g.
+ "Charmonizer/Probe/LargeFiles.h", "Charmonizer/Probe/Integers.h" -- do the
+ heavy lifting. Each topical module exports 1 main function,
+ ModuleName_run(), which runs all the relevant compiler probes and appends
+ output to charmony.h. As you run each module in turn, "charmony.h" gets
+ built up incrementally; it can be further customized by writing your own
+ content to it at any point.
+
+TESTING
+
+ #include "Charmonizer/Test.h"
+ #include "Charmonizer/Test/Integers.h"
+ #include "Charmonizer/Test/Largefile.h"
+ #include "MyTest.h"
+
+ int main() {
+ int all_tests_pass = 0;
+
+ /* Set up. */
+ chaz_Test_init();
+ chaz_Integers_init_test();
+ chaz_LargeFiles_init_test();
+ MyTest_init_test();
+
+ /* Run all the tests */
+ all_tests_pass = chaz_Test_run_all_tests();
+
+ /* Tear down. */
+ chaz_Test_clean_up();
+
+ return all_tests_pass;
+ }
+
+ Charmonizer provides both a general test harness for writing your own
+ tests, and a corresponding test module for each probing module. The stock
+ tests can be found within "Charmonizer/Test" -- e.g. at
+ "Charmonizer/Test/Integers.h".
+
+ The stock tests require access to "charmony.h". Not all tests will pass
+ in every environment, and the expectation is that you will append
+ charmony.h with ifdef tests as necessary to draw in supplementary code:
+
+ #ifndef HAS_DIRENT_H
+ #include "my/dirent.h"
+ #endif
+
+ Charmonizer restricts itself to working with what it finds, and
+ does not supply a library of compatibility functions.
+
+C NAMESPACE
+
+ The "charmony.h" header prepends a prefix onto most of the symbols it
+ exports: either "chy_" or "CHY_". For public code, such as header files,
+ this helps avoid namespace collisions. For private code, the prefixes can
+ be stripped via the CHY_USE_SHORT_NAMES symbol.
+
+ #define CHY_USE_SHORT_NAMES
+ #ifdef HAS_LONG_LONG /* alias for CHY_HAS_LONG_LONG */
+
+FILESYSTEM NAMESPACE
+
+ Charmonizer creates a number of temporary files within the current working
+ directory while it runs. These files all begin with "_charm".
+
diff --git a/charmonizer/charmonize.c b/charmonizer/charmonize.c
new file mode 100644
index 0000000..a5a9ab5
--- /dev/null
+++ b/charmonizer/charmonize.c
@@ -0,0 +1,99 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonize.c -- Create Charmony.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "Charmonizer/Probe.h"
+#include "Charmonizer/Probe/AtomicOps.h"
+#include "Charmonizer/Probe/DirManip.h"
+#include "Charmonizer/Probe/Floats.h"
+#include "Charmonizer/Probe/FuncMacro.h"
+#include "Charmonizer/Probe/Headers.h"
+#include "Charmonizer/Probe/Integers.h"
+#include "Charmonizer/Probe/LargeFiles.h"
+#include "Charmonizer/Probe/Memory.h"
+#include "Charmonizer/Probe/UnusedVars.h"
+#include "Charmonizer/Probe/VariadicMacros.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+
+int main(int argc, char **argv) {
+ /* Parse and process arguments. */
+ if (argc < 3) {
+ fprintf(stderr,
+ "Usage: ./charmonize CC_COMMAND CC_FLAGS [VERBOSITY]\n");
+ exit(1);
+ }
+ else {
+ char *cc_command = argv[1];
+ char *cc_flags = argv[2];
+ if (argc > 3) {
+ const long verbosity = strtol(argv[3], NULL, 10);
+ chaz_Probe_set_verbosity(verbosity);
+ }
+ chaz_Probe_init(cc_command, cc_flags, NULL);
+ }
+
+ /* Run probe modules. */
+ chaz_DirManip_run();
+ chaz_Headers_run();
+ chaz_AtomicOps_run();
+ chaz_FuncMacro_run();
+ chaz_Integers_run();
+ chaz_Floats_run();
+ chaz_LargeFiles_run();
+ chaz_Memory_run();
+ chaz_UnusedVars_run();
+ chaz_VariadicMacros_run();
+
+ /* Write custom postamble. */
+ chaz_ConfWriter_append_conf(
+ "#ifdef CHY_HAS_SYS_TYPES_H\n"
+ " #include <sys/types.h>\n"
+ "#endif\n\n"
+ );
+ chaz_ConfWriter_append_conf(
+ "#ifdef CHY_HAS_ALLOCA_H\n"
+ " #include <alloca.h>\n"
+ "#elif defined(CHY_HAS_MALLOC_H)\n"
+ " #include <malloc.h>\n"
+ "#elif defined(CHY_ALLOCA_IN_STDLIB_H)\n"
+ " #include <stdlib.h>\n"
+ "#endif\n\n"
+ );
+ chaz_ConfWriter_append_conf(
+ "#ifdef CHY_HAS_WINDOWS_H\n"
+ " /* Target Windows XP. */\n"
+ " #ifndef WINVER\n"
+ " #define WINVER 0x0500\n"
+ " #endif\n"
+ " #ifndef _WIN32_WINNT\n"
+ " #define _WIN32_WINNT 0x0500\n"
+ " #endif\n"
+ "#endif\n\n"
+ );
+
+ /* Clean up. */
+ chaz_Probe_clean_up();
+
+ return 0;
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Compiler.c b/charmonizer/src/Charmonizer/Core/Compiler.c
new file mode 100644
index 0000000..1ec5e5d
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Compiler.c
@@ -0,0 +1,296 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <string.h>
+#include <stdlib.h>
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/OperatingSystem.h"
+
+/* Temporary files. */
+#define TRY_SOURCE_PATH "_charmonizer_try.c"
+#define TRY_BASENAME "_charmonizer_try"
+#define TARGET_PATH "_charmonizer_target"
+
+/* Static vars. */
+static char *cc_command = NULL;
+static char *cc_flags = NULL;
+static char **inc_dirs = NULL;
+static char *try_exe_name = NULL;
+static char *try_obj_name = NULL;
+
+/* Detect a supported compiler, or assume a generic GCC-compatible compiler
+ * and hope for the best. */
+#ifdef __GNUC__
+static const char *include_flag = "-I ";
+static const char *object_flag = "-o ";
+static const char *exe_flag = "-o ";
+#elif defined(_MSC_VER)
+static const char *include_flag = "/I";
+static const char *object_flag = "/Fo";
+static const char *exe_flag = "/Fe";
+#else
+static const char *include_flag = "-I ";
+static const char *object_flag = "-o ";
+static const char *exe_flag = "-o ";
+#endif
+
+static void
+S_do_test_compile(void);
+
+void
+CC_init(const char *compiler_command, const char *compiler_flags) {
+ const char *code = "int main() { return 0; }\n";
+
+ if (Util_verbosity) { printf("Creating compiler object...\n"); }
+
+ /* Assign. */
+ cc_command = Util_strdup(compiler_command);
+ cc_flags = Util_strdup(compiler_flags);
+
+ /* Init. */
+ inc_dirs = (char**)calloc(sizeof(char*), 1);
+
+ /* Add the current directory as an include dir. */
+ CC_add_inc_dir(".");
+
+ /* Set names for the targets which we "try" to compile. */
+ {
+ const char *exe_ext = OS_exe_ext();
+ const char *obj_ext = OS_obj_ext();
+ size_t exe_len = strlen(TRY_BASENAME) + strlen(exe_ext) + 1;
+ size_t obj_len = strlen(TRY_BASENAME) + strlen(obj_ext) + 1;
+ try_exe_name = (char*)malloc(exe_len);
+ try_obj_name = (char*)malloc(obj_len);
+ sprintf(try_exe_name, "%s%s", TRY_BASENAME, exe_ext);
+ sprintf(try_obj_name, "%s%s", TRY_BASENAME, obj_ext);
+ }
+
+ /* If we can't compile anything, game over. */
+ if (Util_verbosity) {
+ printf("Trying to compile a small test file...\n");
+ }
+ if (!CC_test_compile(code, strlen(code))) {
+ Util_die("Failed to compile a small test file");
+ }
+}
+
+void
+CC_clean_up(void) {
+ char **dirs;
+
+ for (dirs = inc_dirs; *dirs != NULL; dirs++) {
+ free(*dirs);
+ }
+ free(inc_dirs);
+
+ free(cc_command);
+ free(cc_flags);
+
+ free(try_obj_name);
+ free(try_exe_name);
+}
+
+static char*
+S_inc_dir_string(void) {
+ size_t needed = 0;
+ char *inc_dir_string;
+ char **dirs;
+ for (dirs = inc_dirs; *dirs != NULL; dirs++) {
+ needed += strlen(include_flag) + 2;
+ needed += strlen(*dirs);
+ }
+ inc_dir_string = (char*)malloc(needed + 1);
+ inc_dir_string[0] = '\0';
+ for (dirs = inc_dirs; *dirs != NULL; dirs++) {
+ strcat(inc_dir_string, include_flag);
+ strcat(inc_dir_string, *dirs);
+ strcat(inc_dir_string, " ");
+ }
+ return inc_dir_string;
+}
+
+chaz_bool_t
+CC_compile_exe(const char *source_path, const char *exe_name,
+ const char *code, size_t code_len) {
+ const char *exe_ext = OS_exe_ext();
+ size_t exe_file_buf_size = strlen(exe_name) + strlen(exe_ext) + 1;
+ char *exe_file = (char*)malloc(exe_file_buf_size);
+ size_t junk_buf_size = exe_file_buf_size + 3;
+ char *junk = (char*)malloc(junk_buf_size);
+ size_t exe_file_buf_len = sprintf(exe_file, "%s%s", exe_name, exe_ext);
+ char *inc_dir_string = S_inc_dir_string();
+ size_t command_max_size = strlen(cc_command)
+ + strlen(source_path)
+ + strlen(exe_flag)
+ + exe_file_buf_len
+ + strlen(inc_dir_string)
+ + strlen(cc_flags)
+ + 200; /* command start, _charm_run, etc. */
+ char *command = (char*)malloc(command_max_size);
+ chaz_bool_t result;
+ (void)code_len; /* Unused. */
+
+ /* Write the source file. */
+ Util_write_file(source_path, code);
+
+ /* Prepare and run the compiler command. */
+ sprintf(command, "%s %s %s%s %s %s",
+ cc_command, source_path,
+ exe_flag, exe_file,
+ inc_dir_string, cc_flags);
+ if (Util_verbosity < 2) {
+ OS_run_quietly(command);
+ }
+ else {
+ system(command);
+ }
+
+#ifdef _MSC_VER
+ /* Zap MSVC junk. */
+ /* TODO: Key this off the compiler supplied as argument, not the compiler
+ * used to compile Charmonizer. */
+ sprintf(junk, "%s.obj", exe_name);
+ remove(junk);
+ sprintf(junk, "%s.ilk", exe_name);
+ remove(junk);
+ sprintf(junk, "%s.pdb", exe_name);
+ remove(junk);
+#endif
+
+ /* See if compilation was successful. Remove the source file. */
+ result = Util_can_open_file(exe_file);
+ if (!Util_remove_and_verify(source_path)) {
+ Util_die("Failed to remove '%s'", source_path);
+ }
+
+ free(command);
+ free(inc_dir_string);
+ free(junk);
+ free(exe_file);
+ return result;
+}
+
+chaz_bool_t
+CC_compile_obj(const char *source_path, const char *obj_name,
+ const char *code, size_t code_len) {
+ const char *obj_ext = OS_obj_ext();
+ size_t obj_file_buf_size = strlen(obj_name) + strlen(obj_ext) + 1;
+ char *obj_file = (char*)malloc(obj_file_buf_size);
+ size_t obj_file_buf_len = sprintf(obj_file, "%s%s", obj_name, obj_ext);
+ char *inc_dir_string = S_inc_dir_string();
+ size_t command_max_size = strlen(cc_command)
+ + strlen(source_path)
+ + strlen(object_flag)
+ + obj_file_buf_len
+ + strlen(inc_dir_string)
+ + strlen(cc_flags)
+ + 200; /* command start, _charm_run, etc. */
+ char *command = (char*)malloc(command_max_size);
+ chaz_bool_t result;
+ (void)code_len; /* Unused. */
+
+ /* Write the source file. */
+ Util_write_file(source_path, code);
+
+ /* Prepare and run the compiler command. */
+ sprintf(command, "%s %s %s%s %s %s",
+ cc_command, source_path,
+ object_flag, obj_file,
+ inc_dir_string,
+ cc_flags);
+ if (Util_verbosity < 2) {
+ OS_run_quietly(command);
+ }
+ else {
+ system(command);
+ }
+
+ /* See if compilation was successful. Remove the source file. */
+ result = Util_can_open_file(obj_file);
+ if (!Util_remove_and_verify(source_path)) {
+ Util_die("Failed to remove '%s'", source_path);
+ }
+
+ free(command);
+ free(inc_dir_string);
+ free(obj_file);
+ return result;
+}
+
+chaz_bool_t
+CC_test_compile(const char *source, size_t source_len) {
+ chaz_bool_t compile_succeeded;
+ if (!Util_remove_and_verify(try_obj_name)) {
+ Util_die("Failed to delete file '%s'", try_obj_name);
+ }
+ compile_succeeded = CC_compile_obj(TRY_SOURCE_PATH, TRY_BASENAME,
+ source, source_len);
+ remove(try_obj_name);
+ return compile_succeeded;
+}
+
+char*
+CC_capture_output(const char *source, size_t source_len, size_t *output_len) {
+ char *captured_output = NULL;
+ chaz_bool_t compile_succeeded;
+
+ /* Clear out previous versions and test to make sure removal worked. */
+ if (!Util_remove_and_verify(try_exe_name)) {
+ Util_die("Failed to delete file '%s'", try_exe_name);
+ }
+ if (!Util_remove_and_verify(TARGET_PATH)) {
+ Util_die("Failed to delete file '%s'", TARGET_PATH);
+ }
+
+ /* Attempt compilation; if successful, run app and slurp output. */
+ compile_succeeded = CC_compile_exe(TRY_SOURCE_PATH, TRY_BASENAME,
+ source, source_len);
+ if (compile_succeeded) {
+ OS_run_local(try_exe_name, NULL);
+ captured_output = Util_slurp_file(TARGET_PATH, output_len);
+ }
+ else {
+ *output_len = 0;
+ }
+
+ /* Remove all the files we just created. */
+ remove(TRY_SOURCE_PATH);
+ OS_remove_exe(TRY_BASENAME);
+ remove(TARGET_PATH);
+
+ return captured_output;
+}
+
+void
+CC_add_inc_dir(const char *dir) {
+ size_t num_dirs = 0;
+ char **dirs = inc_dirs;
+
+ /* Count up the present number of dirs, reallocate. */
+ while (*dirs++ != NULL) { num_dirs++; }
+ num_dirs += 1; /* Passed-in dir. */
+ inc_dirs = (char**)realloc(inc_dirs, (num_dirs + 1) * sizeof(char*));
+
+ /* Put the passed-in dir at the end of the list. */
+ inc_dirs[num_dirs - 1] = Util_strdup(dir);
+ inc_dirs[num_dirs] = NULL;
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Compiler.h b/charmonizer/src/Charmonizer/Core/Compiler.h
new file mode 100644
index 0000000..d28ff06
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Compiler.h
@@ -0,0 +1,91 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Core/Compiler.h
+ */
+
+#ifndef H_CHAZ_COMPILER
+#define H_CHAZ_COMPILER
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include "Charmonizer/Core/Defines.h"
+
+/* Attempt to compile and link an executable. Return true if the executable
+ * file exists after the attempt.
+ */
+chaz_bool_t
+chaz_CC_compile_exe(const char *source_path, const char *exe_path,
+ const char *code, size_t code_len);
+
+/* Attempt to compile an object file. Return true if the object file
+ * exists after the attempt.
+ */
+chaz_bool_t
+chaz_CC_compile_obj(const char *source_path, const char *obj_path,
+ const char *code, size_t code_len);
+
+/* Attempt to compile the supplied source code and return true if the
+ * effort succeeds.
+ */
+chaz_bool_t
+chaz_CC_test_compile(const char *source, size_t source_len);
+
+/* Attempt to compile the supplied source code. If successful, capture the
+ * output of the program and return a pointer to a newly allocated buffer.
+ * If the compilation fails, return NULL. The length of the captured
+ * output will be placed into the integer pointed to by [output_len].
+ */
+char*
+chaz_CC_capture_output(const char *source, size_t source_len,
+ size_t *output_len);
+
+/* Add an include directory which will be used for all future compilation
+ * attempts.
+ */
+void
+chaz_CC_add_inc_dir(const char *dir);
+
+/** Initialize the compiler environment.
+ */
+void
+chaz_CC_init(const char *cc_command, const char *cc_flags);
+
+/* Clean up the environment.
+ */
+void
+chaz_CC_clean_up(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define CC_compile_exe chaz_CC_compile_exe
+ #define CC_compile_obj chaz_CC_compile_obj
+ #define CC_add_inc_dir chaz_CC_add_inc_dir
+ #define CC_clean_up chaz_CC_clean_up
+ #define CC_test_compile chaz_CC_test_compile
+ #define CC_capture_output chaz_CC_capture_output
+ #define CC_init chaz_CC_init
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_COMPILER */
+
+
diff --git a/charmonizer/src/Charmonizer/Core/ConfWriter.c b/charmonizer/src/Charmonizer/Core/ConfWriter.c
new file mode 100644
index 0000000..75ee2c1
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/ConfWriter.c
@@ -0,0 +1,123 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/OperatingSystem.h"
+#include "Charmonizer/Core/Compiler.h"
+#include <errno.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* Static vars. */
+static FILE *charmony_fh = NULL;
+
+void
+ConfWriter_init(void) {
+ return;
+}
+
+void
+ConfWriter_open_charmony_h(const char *charmony_start) {
+ /* Open the filehandle. */
+ charmony_fh = fopen("charmony.h", "w+");
+ if (charmony_fh == NULL) {
+ Util_die("Can't open 'charmony.h': %s", strerror(errno));
+ }
+
+ /* Print supplied text (if any) along with warning, open include guard. */
+ if (charmony_start != NULL) {
+ fwrite(charmony_start, sizeof(char), strlen(charmony_start),
+ charmony_fh);
+ }
+ fprintf(charmony_fh,
+ "/* Header file auto-generated by Charmonizer. \n"
+ " * DO NOT EDIT THIS FILE!!\n"
+ " */\n\n"
+ "#ifndef H_CHARMONY\n"
+ "#define H_CHARMONY 1\n\n"
+ );
+}
+
+FILE*
+ConfWriter_get_charmony_fh(void) {
+ return charmony_fh;
+}
+
+void
+ConfWriter_clean_up(void) {
+ /* Write the last bit of charmony.h and close. */
+ fprintf(charmony_fh, "#endif /* H_CHARMONY */\n\n");
+ if (fclose(charmony_fh)) {
+ Util_die("Couldn't close 'charmony.h': %s", strerror(errno));
+ }
+}
+
+void
+ConfWriter_append_conf(const char *fmt, ...) {
+ va_list args;
+
+ va_start(args, fmt);
+ vfprintf(charmony_fh, fmt, args);
+ va_end(args);
+}
+
+void
+ConfWriter_start_short_names(void) {
+ ConfWriter_append_conf(
+ "\n#if defined(CHY_USE_SHORT_NAMES) "
+ "|| defined(CHAZ_USE_SHORT_NAMES)\n"
+ );
+}
+
+void
+ConfWriter_end_short_names(void) {
+ ConfWriter_append_conf("#endif /* USE_SHORT_NAMES */\n");
+}
+
+void
+ConfWriter_start_module(const char *module_name) {
+ if (chaz_Util_verbosity > 0) {
+ printf("Running %s module...\n", module_name);
+ }
+ ConfWriter_append_conf("\n/* %s */\n", module_name);
+}
+
+void
+ConfWriter_end_module(void) {
+ ConfWriter_append_conf("\n");
+}
+
+void
+ConfWriter_shorten_macro(const char *sym) {
+ ConfWriter_append_conf(" #define %s CHY_%s\n", sym, sym);
+}
+
+void
+ConfWriter_shorten_typedef(const char *sym) {
+ ConfWriter_append_conf(" #define %s chy_%s\n", sym, sym);
+}
+
+void
+ConfWriter_shorten_function(const char *sym) {
+ ConfWriter_append_conf(" #define %s chy_%s\n", sym, sym);
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/ConfWriter.h b/charmonizer/src/Charmonizer/Core/ConfWriter.h
new file mode 100644
index 0000000..3608eb0
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/ConfWriter.h
@@ -0,0 +1,116 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Core/ConfWriter.h -- Write to a config file.
+ */
+
+#ifndef H_CHAZ_CONFWRITER
+#define H_CHAZ_CONFWRITER 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stddef.h>
+#include "Charmonizer/Core/Defines.h"
+
+/* Initialize elements needed by ConfWriter. Must be called before anything
+ * else, but after os and compiler are initialized.
+ */
+void
+chaz_ConfWriter_init(void);
+
+/* Open the charmony.h file handle. Print supplied text to it, if non-null.
+ * Print an explanatory comment and open the include guard.
+ */
+void
+chaz_ConfWriter_open_charmony_h(const char *charmony_start);
+
+/* Return the config file's file handle.
+ */
+FILE*
+chaz_ConfWriter_get_charmony_fh(void);
+
+/* Close the include guard on charmony.h, then close the file. Delete temp
+ * files and perform any other needed cleanup.
+ */
+void
+chaz_ConfWriter_clean_up(void);
+
+/* Print output to charmony.h.
+ */
+void
+chaz_ConfWriter_append_conf(const char *fmt, ...);
+
+/* Start a short names block.
+ */
+void
+chaz_ConfWriter_start_short_names(void);
+
+/* Close a short names block.
+ */
+void
+chaz_ConfWriter_end_short_names(void);
+
+/* Define a shortened version of a macro symbol (minus the "CHY_" prefix);
+ */
+void
+chaz_ConfWriter_shorten_macro(const char *symbol);
+
+/* Define a shortened version of a typedef symbol (minus the "chy_" prefix);
+ */
+void
+chaz_ConfWriter_shorten_typedef(const char *symbol);
+
+/* Define a shortened version of a function symbol (minus the "chy_" prefix);
+ */
+void
+chaz_ConfWriter_shorten_function(const char *symbol);
+
+/* Print a "chapter heading" comment in the conf file when starting a module.
+ */
+void
+chaz_ConfWriter_start_module(const char *module_name);
+
+/* Leave a little whitespace at the end of each module.
+ */
+void
+chaz_ConfWriter_end_module(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define ConfWriter_init chaz_ConfWriter_init
+ #define ConfWriter_open_charmony_h chaz_ConfWriter_open_charmony_h
+ #define ConfWriter_get_charmony_fh chaz_ConfWriter_get_charmony_fh
+ #define ConfWriter_clean_up chaz_ConfWriter_clean_up
+ #define ConfWriter_build_charm_run chaz_ConfWriter_build_charm_run
+ #define ConfWriter_start_module chaz_ConfWriter_start_module
+ #define ConfWriter_end_module chaz_ConfWriter_end_module
+ #define ConfWriter_start_short_names chaz_ConfWriter_start_short_names
+ #define ConfWriter_end_short_names chaz_ConfWriter_end_short_names
+ #define ConfWriter_append_conf chaz_ConfWriter_append_conf
+ #define ConfWriter_shorten_macro chaz_ConfWriter_shorten_macro
+ #define ConfWriter_shorten_typedef chaz_ConfWriter_shorten_typedef
+ #define ConfWriter_shorten_function chaz_ConfWriter_shorten_function
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_CONFWRITER */
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Defines.h b/charmonizer/src/Charmonizer/Core/Defines.h
new file mode 100644
index 0000000..58d7827
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Defines.h
@@ -0,0 +1,44 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Core/Defines.h -- Universal definitions.
+ */
+#ifndef H_CHAZ_DEFINES
+#define H_CHAZ_DEFINES 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef int chaz_bool_t;
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CHAZ_QUOTE(x) #x "\n"
+
+#if (defined(CHAZ_USE_SHORT_NAMES) || defined(CHY_USE_SHORT_NAMES))
+ #define QUOTE CHAZ_QUOTE
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_DEFINES */
+
diff --git a/charmonizer/src/Charmonizer/Core/Dir.c b/charmonizer/src/Charmonizer/Core/Dir.c
new file mode 100644
index 0000000..2163dbe
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Dir.c
@@ -0,0 +1,161 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "Charmonizer/Core/Dir.h"
+
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/OperatingSystem.h"
+#include "Charmonizer/Core/Util.h"
+
+static chaz_bool_t mkdir_available = false;
+static chaz_bool_t rmdir_available = false;
+static chaz_bool_t initialized = false;
+int Dir_mkdir_num_args = 0;
+static char mkdir_command[7];
+char *Dir_mkdir_command = mkdir_command;
+
+/* Source code for standard POSIX mkdir */
+static const char posix_mkdir_code[] =
+ QUOTE( #include <%s> )
+ QUOTE( int main(int argc, char **argv) { )
+ QUOTE( if (argc != 2) { return 1; } )
+ QUOTE( if (mkdir(argv[1], 0777) != 0) { return 2; } )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+/* Source code for Windows _mkdir. */
+static const char win_mkdir_code[] =
+ QUOTE( #include <direct.h> )
+ QUOTE( int main(int argc, char **argv) { )
+ QUOTE( if (argc != 2) { return 1; } )
+ QUOTE( if (_mkdir(argv[1]) != 0) { return 2; } )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+/* Source code for rmdir. */
+static const char rmdir_code[] =
+ QUOTE( #include <%s> )
+ QUOTE( int main(int argc, char **argv) { )
+ QUOTE( if (argc != 2) { return 1; } )
+ QUOTE( if (rmdir(argv[1]) != 0) { return 2; } )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static chaz_bool_t
+S_try_init_posix_mkdir(const char *header) {
+ size_t needed = sizeof(posix_mkdir_code) + 30;
+ char *code_buf = (char*)malloc(needed);
+
+ /* Attempt compilation. */
+ sprintf(code_buf, posix_mkdir_code, header);
+ mkdir_available = CC_compile_exe("_charm_mkdir.c", "_charm_mkdir",
+ code_buf, strlen(code_buf));
+
+ /* Set vars on success. */
+ if (mkdir_available) {
+ strcpy(mkdir_command, "mkdir");
+ if (strcmp(header, "direct.h") == 0) {
+ Dir_mkdir_num_args = 1;
+ }
+ else {
+ Dir_mkdir_num_args = 2;
+ }
+ }
+
+ free(code_buf);
+ return mkdir_available;
+}
+
+static chaz_bool_t
+S_try_init_win_mkdir(void) {
+ mkdir_available = CC_compile_exe("_charm_mkdir.c", "_charm_mkdir",
+ win_mkdir_code, strlen(win_mkdir_code));
+ if (mkdir_available) {
+ strcpy(mkdir_command, "_mkdir");
+ Dir_mkdir_num_args = 1;
+ }
+ return mkdir_available;
+}
+
+static void
+S_init_mkdir(void) {
+ if (Util_verbosity) {
+ printf("Attempting to compile _charm_mkdir utility...\n");
+ }
+ if (HeadCheck_check_header("windows.h")) {
+ if (S_try_init_win_mkdir()) { return; }
+ if (S_try_init_posix_mkdir("direct.h")) { return; }
+ }
+ if (S_try_init_posix_mkdir("sys/stat.h")) { return; }
+}
+
+static chaz_bool_t
+S_try_init_rmdir(const char *header) {
+ size_t needed = sizeof(posix_mkdir_code) + 30;
+ char *code_buf = (char*)malloc(needed);
+ sprintf(code_buf, rmdir_code, header);
+ rmdir_available = CC_compile_exe("_charm_rmdir.c", "_charm_rmdir",
+ code_buf, strlen(code_buf));
+ free(code_buf);
+ return rmdir_available;
+}
+
+static void
+S_init_rmdir(void) {
+ if (Util_verbosity) {
+ printf("Attempting to compile _charm_rmdir utility...\n");
+ }
+ if (S_try_init_rmdir("unistd.h")) { return; }
+ if (S_try_init_rmdir("dirent.h")) { return; }
+ if (S_try_init_rmdir("direct.h")) { return; }
+}
+
+/* Compile _charm_mkdir and _charm_rmdir. */
+void
+Dir_init(void) {
+ if (!initialized) {
+ initialized = true;
+ S_init_mkdir();
+ S_init_rmdir();
+ }
+}
+
+void
+Dir_clean_up(void) {
+ OS_remove_exe("_charm_mkdir");
+ OS_remove_exe("_charm_rmdir");
+}
+
+chaz_bool_t
+Dir_mkdir(const char *filepath) {
+ if (!initialized) { Dir_init(); }
+ return OS_run_local("_charm_mkdir ", filepath, NULL);
+}
+
+chaz_bool_t
+Dir_rmdir(const char *filepath) {
+ if (!initialized) { Dir_init(); }
+ return OS_run_local("_charm_rmdir ", filepath, NULL);
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Dir.h b/charmonizer/src/Charmonizer/Core/Dir.h
new file mode 100644
index 0000000..2e39764
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Dir.h
@@ -0,0 +1,72 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Core/Dir.h - Directory manipulation routines.
+ *
+ * Compile utilities to create and remove directories.
+ */
+
+#ifndef H_CHAZ_DIR
+#define H_CHAZ_DIR
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Charmonizer/Core/Defines.h"
+
+/* Compile the utilities which execute the Dir_mkdir and Dir_rmdir functions. */
+void
+chaz_Dir_init(void);
+
+/* Tear down.
+ */
+void
+chaz_Dir_clean_up(void);
+
+/* Attempt to create a directory. Returns true on success, false on failure.
+ */
+chaz_bool_t
+chaz_Dir_mkdir(const char *filepath);
+
+/* Attempt to remove a directory, which must be empty. Returns true on
+ * success, false on failure.
+ */
+chaz_bool_t
+chaz_Dir_rmdir(const char *filepath);
+
+/* The string command for mkdir. */
+extern char* chaz_Dir_mkdir_command;
+
+/* Indicate whether the mkdir takes 1 or 2 args. */
+extern int chaz_Dir_mkdir_num_args;
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Dir_init chaz_Dir_init
+ #define Dir_clean_up chaz_Dir_clean_up
+ #define Dir_mkdir chaz_Dir_mkdir
+ #define Dir_rmdir chaz_Dir_rmdir
+ #define Dir_mkdir_command chaz_Dir_mkdir_command
+ #define Dir_mkdir_num_args chaz_Dir_mkdir_num_args
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_DIR */
+
+
diff --git a/charmonizer/src/Charmonizer/Core/HeaderChecker.c b/charmonizer/src/Charmonizer/Core/HeaderChecker.c
new file mode 100644
index 0000000..c3d862e
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/HeaderChecker.c
@@ -0,0 +1,214 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include <string.h>
+#include <stdlib.h>
+
+typedef struct Header {
+ const char *name;
+ chaz_bool_t exists;
+} Header;
+
+/* "hello_world.c" without the hello or the world. */
+static const char test_code[] = "int main() { return 0; }\n";
+
+/* Keep a sorted, dynamically-sized array of names of all headers we've
+ * checked for so far.
+ */
+static int cache_size = 0;
+static Header **header_cache = NULL;
+
+/* Comparison function to feed to qsort, bsearch, etc.
+ */
+static int
+S_compare_headers(const void *vptr_a, const void *vptr_b);
+
+/* Run a test compilation and return a new Header object encapsulating the
+ * results.
+ */
+static Header*
+S_discover_header(const char *header_name);
+
+/* Extend the cache, add this Header object to it, and sort.
+ */
+static void
+S_add_to_cache(Header *header);
+
+/* Like add_to_cache, but takes a individual elements instead of a Header* and
+ * checks if header exists in array first.
+ */
+static void
+S_maybe_add_to_cache(const char *header_name, chaz_bool_t exists);
+
+void
+HeadCheck_init(void) {
+ Header *null_header = (Header*)malloc(sizeof(Header));
+
+ /* Create terminating record for the dynamic array of Header objects. */
+ null_header->name = NULL;
+ null_header->exists = false;
+ header_cache = (Header**)malloc(sizeof(void*));
+ *header_cache = null_header;
+ cache_size = 1;
+}
+
+chaz_bool_t
+HeadCheck_check_header(const char *header_name) {
+ Header *header;
+ Header key;
+ Header *fake = &key;
+ Header **header_ptr;
+
+ /* Fake up a key to feed to bsearch; see if the header's already there. */
+ key.name = header_name;
+ key.exists = false;
+ header_ptr = (Header**)bsearch(&fake, header_cache, cache_size,
+ sizeof(void*), S_compare_headers);
+
+ /* If it's not there, go try a test compile. */
+ if (header_ptr == NULL) {
+ header = S_discover_header(header_name);
+ S_add_to_cache(header);
+ }
+ else {
+ header = *header_ptr;
+ }
+
+ return header->exists;
+}
+
+chaz_bool_t
+HeadCheck_check_many_headers(const char **header_names) {
+ chaz_bool_t success;
+ int i;
+ char *code_buf = Util_strdup("");
+ size_t needed = sizeof(test_code) + 20;
+
+ /* Build the source code string. */
+ for (i = 0; header_names[i] != NULL; i++) {
+ needed += strlen(header_names[i]);
+ needed += sizeof("#include <>\n");
+ }
+ code_buf = (char*)malloc(needed);
+ code_buf[0] = '\0';
+ for (i = 0; header_names[i] != NULL; i++) {
+ strcat(code_buf, "#include <");
+ strcat(code_buf, header_names[i]);
+ strcat(code_buf, ">\n");
+ }
+ strcat(code_buf, test_code);
+
+ /* If the code compiles, bulk add all header names to the cache. */
+ success = CC_test_compile(code_buf, strlen(code_buf));
+ if (success) {
+ for (i = 0; header_names[i] != NULL; i++) {
+ S_maybe_add_to_cache(header_names[i], true);
+ }
+ }
+
+ free(code_buf);
+ return success;
+}
+
+static const char contains_code[] =
+ QUOTE( #include <stddef.h> )
+ QUOTE( %s )
+ QUOTE( int main() { return offsetof(%s, %s); } );
+
+chaz_bool_t
+HeadCheck_contains_member(const char *struct_name, const char *member,
+ const char *includes) {
+ long needed = sizeof(contains_code)
+ + strlen(struct_name)
+ + strlen(member)
+ + strlen(includes)
+ + 10;
+ char *buf = (char*)malloc(needed);
+ chaz_bool_t retval;
+ sprintf(buf, contains_code, includes, struct_name, member);
+ retval = CC_test_compile(buf, strlen(buf));
+ free(buf);
+ return retval;
+}
+
+static int
+S_compare_headers(const void *vptr_a, const void *vptr_b) {
+ Header *const *const a = (Header*const*)vptr_a;
+ Header *const *const b = (Header*const*)vptr_b;
+
+ /* (NULL is "greater than" any string.) */
+ if ((*a)->name == NULL) { return 1; }
+ else if ((*b)->name == NULL) { return -1; }
+ else { return strcmp((*a)->name, (*b)->name); }
+}
+
+static Header*
+S_discover_header(const char *header_name) {
+ Header* header = (Header*)malloc(sizeof(Header));
+ size_t needed = strlen(header_name) + sizeof(test_code) + 50;
+ char *include_test = (char*)malloc(needed);
+
+ /* Assign. */
+ header->name = Util_strdup(header_name);
+
+ /* See whether code that tries to pull in this header compiles. */
+ sprintf(include_test, "#include <%s>\n%s", header_name, test_code);
+ header->exists = CC_test_compile(include_test, strlen(include_test));
+
+ free(include_test);
+ return header;
+}
+
+static void
+S_add_to_cache(Header *header) {
+ /* Realloc array -- inefficient, but this isn't a bottleneck. */
+ cache_size++;
+ header_cache = (Header**)realloc(header_cache,
+ (cache_size * sizeof(void*)));
+ header_cache[cache_size - 1] = header;
+
+ /* Keep the list of headers sorted. */
+ qsort(header_cache, cache_size, sizeof(*header_cache), S_compare_headers);
+}
+
+static void
+S_maybe_add_to_cache(const char *header_name, chaz_bool_t exists) {
+ Header *header;
+ Header key;
+ Header *fake = &key;
+
+ /* Fake up a key and bsearch for it. */
+ key.name = header_name;
+ key.exists = exists;
+ header = (Header*)bsearch(&fake, header_cache, cache_size,
+ sizeof(void*), S_compare_headers);
+
+ /* We've already done the test compile, so skip that step and add it. */
+ if (header == NULL) {
+ header = (Header*)malloc(sizeof(Header));
+ header->name = Util_strdup(header_name);
+ header->exists = exists;
+ S_add_to_cache(header);
+ }
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/HeaderChecker.h b/charmonizer/src/Charmonizer/Core/HeaderChecker.h
new file mode 100644
index 0000000..0985ec9
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/HeaderChecker.h
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/HeaderChecker.h
+ */
+
+#ifndef H_CHAZ_HEAD_CHECK
+#define H_CHAZ_HEAD_CHECK
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Charmonizer/Core/Defines.h"
+
+/* Bootstrap the HeadCheck. Call this before anything else.
+ */
+void
+chaz_HeadCheck_init(void);
+
+/* Check for a particular header and return true if it's available. The
+ * test-compile is only run the first time a given request is made.
+ */
+chaz_bool_t
+chaz_HeadCheck_check_header(const char *header_name);
+
+/* Attempt to compile a file which pulls in all the headers specified by name
+ * in a null-terminated array. If the compile succeeds, add them all to the
+ * internal register and return true.
+ */
+chaz_bool_t
+chaz_HeadCheck_check_many_headers(const char **header_names);
+
+/* Return true if the member is present in the struct. */
+chaz_bool_t
+chaz_HeadCheck_contains_member(const char *struct_name, const char *member,
+ const char *includes);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define HeadCheck_init chaz_HeadCheck_init
+ #define HeadCheck_contains_member chaz_HeadCheck_contains_member
+ #define HeadCheck_check_header chaz_HeadCheck_check_header
+ #define HeadCheck_check_many_headers chaz_HeadCheck_check_many_headers
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_HEAD_CHECK */
+
+
diff --git a/charmonizer/src/Charmonizer/Core/OperatingSystem.c b/charmonizer/src/Charmonizer/Core/OperatingSystem.c
new file mode 100644
index 0000000..df399f7
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/OperatingSystem.c
@@ -0,0 +1,231 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdarg.h>
+
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/OperatingSystem.h"
+
+static char dev_null[20] = "";
+
+#ifdef _WIN32
+static const char *exe_ext = ".exe";
+static const char *obj_ext = ".obj";
+static const char *local_command_start = ".\\";
+#else
+static const char *exe_ext = "";
+static const char *obj_ext = "";
+static const char *local_command_start = "./";
+#endif
+
+static void
+S_probe_dev_null(void);
+
+/* Compile a small wrapper application which is used to redirect error output
+ * to dev_null.
+ */
+static void
+S_build_charm_run(void);
+
+static chaz_bool_t charm_run_initialized = false;
+static chaz_bool_t charm_run_ok = false;
+
+void
+OS_init(void) {
+ if (Util_verbosity) {
+ printf("Initializing Charmonizer/Core/OperatingSystem...\n");
+ }
+
+ S_probe_dev_null();
+}
+
+static void
+S_probe_dev_null(void) {
+ if (Util_verbosity) {
+ printf("Trying to find a bit-bucket a la /dev/null...\n");
+ }
+
+#ifdef _WIN32
+ strcpy(dev_null, "nul");
+#else
+ {
+ const char *const options[] = {
+ "/dev/null",
+ "/dev/nul",
+ NULL
+ };
+ int i;
+
+ /* Iterate through names of possible devnulls trying to open them. */
+ for (i = 0; options[i] != NULL; i++) {
+ if (Util_can_open_file(options[i])) {
+ strcpy(dev_null, options[i]);
+ return;
+ }
+ }
+
+ /* Bail out because we couldn't find anything like /dev/null. */
+ Util_die("Couldn't find anything like /dev/null");
+ }
+#endif
+}
+
+void
+OS_clean_up(void) {
+ OS_remove_exe("_charm_run");
+}
+
+const char*
+OS_exe_ext(void) {
+ return exe_ext;
+}
+
+const char*
+OS_obj_ext(void) {
+ return obj_ext;
+}
+
+const char*
+OS_dev_null(void) {
+ return dev_null;
+}
+
+static const char charm_run_code[] =
+ QUOTE( #include <stdio.h> )
+ QUOTE( #include <stdlib.h> )
+ QUOTE( #include <string.h> )
+ QUOTE( #include <stddef.h> )
+ QUOTE( int main(int argc, char **argv) )
+ QUOTE( { )
+ QUOTE( char *command; )
+ QUOTE( size_t command_len = 1; /* Terminating null. */ )
+ QUOTE( int i; )
+ QUOTE( int retval; )
+ /* Rebuild command line args. */
+ QUOTE( for (i = 1; i < argc; i++) { )
+ QUOTE( command_len += strlen(argv[i]) + 1; )
+ QUOTE( } )
+ QUOTE( command = (char*)calloc(command_len, sizeof(char)); )
+ QUOTE( if (command == NULL) { )
+ QUOTE( fprintf(stderr, "calloc failed\n"); )
+ QUOTE( exit(1); )
+ QUOTE( } )
+ QUOTE( for (i = 1; i < argc; i++) { )
+ QUOTE( strcat( strcat(command, " "), argv[i] ); )
+ QUOTE( } )
+ /* Redirect all output to /dev/null or equivalent. */
+ QUOTE( freopen("%s", "w", stdout); )
+ QUOTE( freopen("%s", "w", stderr); )
+ /* Run commmand and return its value to parent. */
+ QUOTE( retval = system(command); )
+ QUOTE( free(command); )
+ QUOTE( return retval; )
+ QUOTE( } );
+
+static void
+S_build_charm_run(void) {
+ chaz_bool_t compile_succeeded = false;
+ size_t needed = sizeof(charm_run_code)
+ + strlen(dev_null)
+ + strlen(dev_null)
+ + 20;
+ char *code = (char*)malloc(needed);
+
+ sprintf(code, charm_run_code, dev_null, dev_null);
+ compile_succeeded = CC_compile_exe("_charm_run.c", "_charm_run",
+ code, strlen(code));
+ if (!compile_succeeded) {
+ Util_die("failed to compile _charm_run helper utility");
+ }
+
+ remove("_charm_run.c");
+ free(code);
+ charm_run_ok = true;
+}
+
+void
+OS_remove_exe(const char *name) {
+ char *exe_name = (char*)malloc(strlen(name) + strlen(exe_ext) + 1);
+ sprintf(exe_name, "%s%s", name, exe_ext);
+ remove(exe_name);
+ free(exe_name);
+}
+
+void
+OS_remove_obj(const char *name) {
+ char *obj_name = (char*)malloc(strlen(name) + strlen(obj_ext) + 1);
+ sprintf(obj_name, "%s%s", name, obj_ext);
+ remove(obj_name);
+ free(obj_name);
+}
+
+int
+OS_run_local(const char *arg1, ...) {
+ va_list args;
+ size_t len = strlen(local_command_start) + strlen(arg1);
+ char *command = (char*)malloc(len + 1);
+ int retval;
+ char *arg;
+
+ /* Append all supplied texts. */
+ sprintf(command, "%s%s", local_command_start, arg1);
+ va_start(args, arg1);
+ while (NULL != (arg = va_arg(args, char*))) {
+ len += strlen(arg);
+ command = (char*)realloc(command, len + 1);
+ strcat(command, arg);
+ }
+ va_end(args);
+
+ /* Run the command. */
+ retval = system(command);
+ free(command);
+ return retval;
+}
+
+int
+OS_run_quietly(const char *command) {
+ int retval = 1;
+#ifdef _WIN32
+ char pattern[] = "%s > NUL 2> NUL";
+ size_t size = sizeof(pattern) + strlen(command) + 10;
+ char *quiet_command = (char*)malloc(size);
+ sprintf(quiet_command, pattern, command);
+ retval = system(quiet_command);
+ free(quiet_command);
+#else
+ if (!charm_run_initialized) {
+ charm_run_initialized = true;
+ S_build_charm_run();
+ }
+ if (!charm_run_ok) {
+ retval = system(command);
+ }
+ else {
+ retval = OS_run_local("_charm_run ", command, NULL);
+ }
+#endif
+
+ return retval;
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/OperatingSystem.h b/charmonizer/src/Charmonizer/Core/OperatingSystem.h
new file mode 100644
index 0000000..dfcb57e
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/OperatingSystem.h
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Core/OperatingSystem.h - abstract an operating system down to a few
+ * variables.
+ */
+
+#ifndef H_CHAZ_OPER_SYS
+#define H_CHAZ_OPER_SYS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Remove an executable file named [name], appending the exe_ext if needed.
+ */
+void
+chaz_OS_remove_exe(const char *name);
+
+/* Remove an object file named [name], appending the obj_ext if needed.
+ */
+void
+chaz_OS_remove_obj(const char *name);
+
+/* Concatenate all arguments in a NULL-terminated list into a single command
+ * string, prepend the appropriate prefix, and invoke via system().
+ */
+int
+chaz_OS_run_local(const char *arg1, ...);
+
+/* Invoke a command and attempt to suppress output from both stdout and stderr
+ * (as if they had been sent to /dev/null). If it's not possible to run the
+ * command quietly, run it anyway.
+ */
+int
+chaz_OS_run_quietly(const char *command);
+
+/* Return the extension for an executable on this system.
+ */
+const char*
+chaz_OS_exe_ext(void);
+
+/* Return the extension for a compiled object on this system.
+ */
+const char*
+chaz_OS_obj_ext(void);
+
+/* Return the equivalent of /dev/null on this system.
+ */
+const char*
+chaz_OS_dev_null(void);
+
+/* Initialize the Charmonizer/Core/OperatingSystem module.
+ */
+void
+chaz_OS_init(void);
+
+/* Tear down the Charmonizer/Core/OperatingSystem module.
+ */
+void
+chaz_OS_clean_up(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define OS_remove_exe chaz_OS_remove_exe
+ #define OS_remove_obj chaz_OS_remove_obj
+ #define OS_run_local chaz_OS_run_local
+ #define OS_run_quietly chaz_OS_run_quietly
+ #define OS_exe_ext chaz_OS_exe_ext
+ #define OS_obj_ext chaz_OS_obj_ext
+ #define OS_dev_null chaz_OS_dev_null
+ #define OS_init chaz_OS_init
+ #define OS_clean_up chaz_OS_clean_up
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_COMPILER */
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Stat.c b/charmonizer/src/Charmonizer/Core/Stat.c
new file mode 100644
index 0000000..5634682
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Stat.c
@@ -0,0 +1,104 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "Charmonizer/Core/Stat.h"
+
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/OperatingSystem.h"
+#include "Charmonizer/Core/Util.h"
+
+static chaz_bool_t initialized = false;
+static chaz_bool_t stat_available = false;
+
+/* Lazily compile _charm_stat. */
+static void
+S_init(void);
+
+void
+Stat_stat(const char *filepath, Stat *target) {
+ char *stat_output;
+ size_t output_len;
+
+ /* Failsafe. */
+ target->valid = false;
+
+ /* Lazy init. */
+ if (!initialized) { S_init(); }
+
+ /* Bail out if we didn't succeed in compiling/using _charm_stat. */
+ if (!stat_available) { return; }
+
+ /* Run _charm_stat. */
+ Util_remove_and_verify("_charm_statout");
+ OS_run_local("_charm_stat ", filepath, NULL);
+ stat_output = Util_slurp_file("_charm_statout", &output_len);
+ Util_remove_and_verify("_charm_statout");
+
+ /* Parse the output of _charm_stat and store vars in Stat struct. */
+ if (stat_output != NULL) {
+ char *end_ptr = stat_output;
+ target->size = strtol(stat_output, &end_ptr, 10);
+ stat_output = end_ptr;
+ target->blocks = strtol(stat_output, &end_ptr, 10);
+ target->valid = true;
+ }
+
+ return;
+}
+
+/* Source code for the _charm_stat utility. */
+static const char charm_stat_code[] =
+ QUOTE( #include <stdio.h> )
+ QUOTE( #include <sys/stat.h> )
+ QUOTE( int main(int argc, char **argv) { )
+ QUOTE( FILE *out_fh = fopen("_charm_statout", "w+"); )
+ QUOTE( struct stat st; )
+ QUOTE( if (argc != 2) { return 1; } )
+ QUOTE( if (stat(argv[1], &st) == -1) { return 2; } )
+ QUOTE( fprintf(out_fh, "%ld ", (long)st.st_size); )
+ QUOTE( fprintf(out_fh, "%ld\n", (long)st.st_blocks); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static void
+S_init(void) {
+ /* Only try this once. */
+ initialized = true;
+ if (Util_verbosity) {
+ printf("Attempting to compile _charm_stat utility...\n");
+ }
+
+ /* Bail if sys/stat.h isn't available. */
+ if (!HeadCheck_check_header("sys/stat.h")) { return; }
+
+ /* If the compile succeeds, open up for business. */
+ stat_available = CC_compile_exe("_charm_stat.c", "_charm_stat",
+ charm_stat_code, strlen(charm_stat_code));
+ remove("_charm_stat.c");
+}
+
+void
+Stat_clean_up(void) {
+ OS_remove_exe("_charm_stat");
+}
+
diff --git a/charmonizer/src/Charmonizer/Core/Stat.h b/charmonizer/src/Charmonizer/Core/Stat.h
new file mode 100644
index 0000000..9a1a6f3
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Stat.h
@@ -0,0 +1,69 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Core/Stat.h - stat a file, if possible.
+ *
+ * This component works by attempting to compile a utility program called
+ * "_charm_stat". When Charmonizer needs to stat a file, it shells out to
+ * this utility, which communicates via a file a la capture_output().
+ *
+ * Since we don't know whether we have 64-bit integers when Charmonizer itself
+ * gets compiled, the items in the stat structure are whatever size longs are.
+ *
+ * TODO: probe for which fields are available.
+ */
+
+#ifndef H_CHAZ_STAT
+#define H_CHAZ_STAT
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Charmonizer/Core/Defines.h"
+
+typedef struct chaz_Stat chaz_Stat;
+
+struct chaz_Stat {
+ chaz_bool_t valid;
+ long size;
+ long blocks;
+};
+
+/* Attempt to stat a file. If successful, store the set target->valid to true
+ * and store the results in the stat structure. If unsuccessful, set
+ * target->valid to false.
+ */
+void
+chaz_Stat_stat(const char *filepath, chaz_Stat *target);
+
+void
+chaz_Stat_clean_up(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Stat chaz_Stat
+ #define Stat_stat chaz_Stat_stat
+ #define Stat_clean_up chaz_Stat_clean_up
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_COMPILER */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Util.c b/charmonizer/src/Charmonizer/Core/Util.c
new file mode 100644
index 0000000..5c19184
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Util.c
@@ -0,0 +1,157 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <errno.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include "Charmonizer/Core/Util.h"
+
+/* Global verbosity setting. */
+int Util_verbosity = 1;
+
+void
+Util_write_file(const char *filename, const char *content) {
+ FILE *fh = fopen(filename, "w+");
+ size_t content_len = strlen(content);
+ if (fh == NULL) {
+ Util_die("Couldn't open '%s': %s", filename, strerror(errno));
+ }
+ fwrite(content, sizeof(char), content_len, fh);
+ if (fclose(fh)) {
+ Util_die("Error when closing '%s': %s", filename, strerror(errno));
+ }
+}
+
+char*
+Util_slurp_file(const char *file_path, size_t *len_ptr) {
+ FILE *const file = fopen(file_path, "r");
+ char *contents;
+ size_t len;
+ long check_val;
+
+ /* Sanity check. */
+ if (file == NULL) {
+ Util_die("Error opening file '%s': %s", file_path, strerror(errno));
+ }
+
+ /* Find length; return NULL if the file has a zero-length. */
+ len = Util_flength(file);
+ if (len == 0) {
+ *len_ptr = 0;
+ return NULL;
+ }
+
+ /* Allocate memory and read the file. */
+ contents = (char*)malloc(len * sizeof(char) + 1);
+ if (contents == NULL) {
+ Util_die("Out of memory at %d, %s", __FILE__, __LINE__);
+ }
+ contents[len] = '\0';
+ check_val = fread(contents, sizeof(char), len, file);
+
+ /* Weak error check, because CRLF might result in fewer chars read. */
+ if (check_val <= 0) {
+ Util_die("Tried to read %d characters of '%s', got %d", (int)len,
+ file_path, check_val);
+ }
+
+ /* Set length pointer for benefit of caller. */
+ *len_ptr = check_val;
+
+ /* Clean up. */
+ if (fclose(file)) {
+ Util_die("Error closing file '%s': %s", file_path, strerror(errno));
+ }
+
+ return contents;
+}
+
+long
+Util_flength(void *file) {
+ FILE *f = (FILE*)file;
+ const long bookmark = ftell(f);
+ long check_val;
+ long len;
+
+ /* Seek to end of file and check length. */
+ check_val = fseek(f, 0, SEEK_END);
+ if (check_val == -1) { Util_die("fseek error : %s\n", strerror(errno)); }
+ len = ftell(f);
+ if (len == -1) { Util_die("ftell error : %s\n", strerror(errno)); }
+
+ /* Return to where we were. */
+ check_val = fseek(f, bookmark, SEEK_SET);
+ if (check_val == -1) { Util_die("fseek error : %s\n", strerror(errno)); }
+
+ return len;
+}
+
+char*
+Util_strdup(const char *string) {
+ size_t len = strlen(string);
+ char *copy = (char*)malloc(len + 1);
+ strncpy(copy, string, len);
+ copy[len] = '\0';
+ return copy;
+}
+
+void
+Util_die(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+ fprintf(stderr, "\n");
+ exit(1);
+}
+
+void
+Util_warn(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+ fprintf(stderr, "\n");
+}
+
+int
+Util_remove_and_verify(const char *file_path) {
+ /* Try to remove the file. */
+ remove(file_path);
+
+ /* Return what *might* be success or failure. */
+ return Util_can_open_file(file_path) ? 0 : 1;
+}
+
+int
+Util_can_open_file(const char *file_path) {
+ FILE *garbage_fh;
+
+ /* Use fopen as a portable test for the existence of a file. */
+ garbage_fh = fopen(file_path, "r");
+ if (garbage_fh == NULL) {
+ return 0;
+ }
+ else {
+ fclose(garbage_fh);
+ return 1;
+ }
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Core/Util.h b/charmonizer/src/Charmonizer/Core/Util.h
new file mode 100644
index 0000000..7d95bae
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Core/Util.h
@@ -0,0 +1,96 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Chaz/Core/Util.h -- miscellaneous utilities.
+ */
+
+#ifndef H_CHAZ_UTIL
+#define H_CHAZ_UTIL 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stddef.h>
+#include <stdarg.h>
+
+extern int chaz_Util_verbosity;
+
+/* Open a file (truncating if necessary) and write [content] to it. Util_die() if
+ * an error occurs.
+ */
+void
+chaz_Util_write_file(const char *filename, const char *content);
+
+/* Read an entire file into memory.
+ */
+char*
+chaz_Util_slurp_file(const char *file_path, size_t *len_ptr);
+
+/* Return a newly allocated copy of a NULL-terminated string.
+ */
+char*
+chaz_Util_strdup(const char *string);
+
+/* Get the length of a file (may overshoot on text files under DOS).
+ */
+long
+chaz_Util_flength(void *file);
+
+/* Print an error message to stderr and exit.
+ */
+void
+chaz_Util_die(const char *format, ...);
+
+/* Print an error message to stderr.
+ */
+void
+chaz_Util_warn(const char *format, ...);
+
+/* Attept to delete a file. Don't error if the file wasn't there to begin
+ * with. Return 1 if it seems like the file is gone because an attempt to
+ * open it for reading fails (this doesn't guarantee that the file is gone,
+ * but it works well enough for our purposes). Return 0 if we can still
+ * read the file.
+ */
+int
+chaz_Util_remove_and_verify(const char *file_path);
+
+/* Attempt to open a file for reading, then close it immediately.
+ */
+int
+chaz_Util_can_open_file(const char *file_path);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Util_verbosity chaz_Util_verbosity
+ #define Util_write_file chaz_Util_write_file
+ #define Util_slurp_file chaz_Util_slurp_file
+ #define Util_flength chaz_Util_flength
+ #define Util_die chaz_Util_die
+ #define Util_warn chaz_Util_warn
+ #define Util_strdup chaz_Util_strdup
+ #define Util_remove_and_verify chaz_Util_remove_and_verify
+ #define Util_can_open_file chaz_Util_can_open_file
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_UTIL */
+
+
diff --git a/charmonizer/src/Charmonizer/Probe.c b/charmonizer/src/Charmonizer/Probe.c
new file mode 100644
index 0000000..3ee5ff8
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe.c
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "Charmonizer/Probe.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Dir.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/OperatingSystem.h"
+#include "Charmonizer/Core/Stat.h"
+
+/* Write the "_charm.h" file used by every probe.
+ */
+static void
+S_write_charm_h(void);
+
+static void
+S_remove_charm_h(void);
+
+void
+Probe_init(const char *cc_command, const char *cc_flags,
+ const char *charmony_start) {
+ /* Dispatch other initializers. */
+ OS_init();
+ CC_init(cc_command, cc_flags);
+ ConfWriter_init();
+ HeadCheck_init();
+ ConfWriter_open_charmony_h(charmony_start);
+ S_write_charm_h();
+
+ if (Util_verbosity) { printf("Initialization complete.\n"); }
+}
+
+void
+Probe_clean_up(void) {
+ if (Util_verbosity) { printf("Cleaning up...\n"); }
+
+ /* Dispatch various clean up routines. */
+ S_remove_charm_h();
+ ConfWriter_clean_up();
+ Stat_clean_up();
+ Dir_clean_up();
+ CC_clean_up();
+ OS_clean_up();
+
+ if (Util_verbosity) { printf("Cleanup complete.\n"); }
+}
+
+void
+Probe_set_verbosity(int level) {
+ Util_verbosity = level;
+}
+
+FILE*
+Probe_get_charmony_fh(void) {
+ return ConfWriter_get_charmony_fh();
+}
+
+static const char charm_h_code[] =
+ QUOTE( #ifndef CHARM_H )
+ QUOTE( #define CHARM_H 1 )
+ QUOTE( #include <stdio.h> )
+ QUOTE( #define Charm_Setup freopen("_charmonizer_target", "w", stdout) )
+ QUOTE( #endif );
+
+static void
+S_write_charm_h(void) {
+ Util_write_file("_charm.h", charm_h_code);
+}
+
+static void
+S_remove_charm_h(void) {
+ remove("_charm.h");
+}
+
diff --git a/charmonizer/src/Charmonizer/Probe.h b/charmonizer/src/Charmonizer/Probe.h
new file mode 100644
index 0000000..a9b261a
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe.h
@@ -0,0 +1,72 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CHAZ
+#define H_CHAZ 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include <stdio.h>
+
+/* Set up the Charmonizer environment. This should be called before anything
+ * else.
+ *
+ * @param cc_command the string used to invoke the C compiler via system()
+ * @param cc_flags flags which will be passed on to the C compiler
+ * @param charmony_start Code to prepend onto the front of charmony.h
+ */
+void
+chaz_Probe_init(const char *cc_command, const char *cc_flags,
+ const char *charmony_start);
+
+/* Clean up the Charmonizer environment -- deleting tempfiles, etc. This
+ * should be called only after everything else finishes.
+ */
+void
+chaz_Probe_clean_up(void);
+
+/* Determine how much feedback Charmonizer provides.
+ * 0 - silent
+ * 1 - normal
+ * 2 - debugging
+ */
+void
+chaz_Probe_set_verbosity(int level);
+
+/* Access the FILE* used to write charmony.h, so that you can write your own
+ * content to it. Should not be called before chaz_Probe_init() or after
+ * chaz_Probe_clean_up().
+ */
+FILE*
+chaz_Probe_get_charmony_fh(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Probe_init chaz_Probe_init
+ #define Probe_clean_up chaz_Probe_clean_up
+ #define Probe_set_verbosity chaz_Probe_set_verbosity
+ #define Probe_get_charmony_fh chaz_Probe_get_charmony_fh
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* Include guard. */
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/AtomicOps.c b/charmonizer/src/Charmonizer/Probe/AtomicOps.c
new file mode 100644
index 0000000..bd6bc35
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/AtomicOps.c
@@ -0,0 +1,91 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/AtomicOps.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static const char osatomic_casptr_code[] =
+ QUOTE( #include <libkern/OSAtomic.h> )
+ QUOTE( #include <libkern/OSAtomic.h> )
+ QUOTE( int main() { )
+ QUOTE( int foo = 1; )
+ QUOTE( int *foo_ptr = &foo; )
+ QUOTE( int *target = NULL; )
+ QUOTE( OSAtomicCompareAndSwapPtr(NULL, foo_ptr, (void**)&target); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+void
+AtomicOps_run(void) {
+ chaz_bool_t has_libkern_osatomic_h = false;
+ chaz_bool_t has_osatomic_cas_ptr = false;
+ chaz_bool_t has_sys_atomic_h = false;
+ chaz_bool_t has_intrin_h = false;
+
+ ConfWriter_start_module("AtomicOps");
+
+ if (HeadCheck_check_header("libkern/OSAtomic.h")) {
+ has_libkern_osatomic_h = true;
+ ConfWriter_append_conf("#define CHY_HAS_LIBKERN_OSATOMIC_H\n");
+
+ /* Check for OSAtomicCompareAndSwapPtr, introduced in later versions
+ * of OSAtomic.h. */
+ has_osatomic_cas_ptr = CC_test_compile(osatomic_casptr_code,
+ strlen(osatomic_casptr_code));
+ if (has_osatomic_cas_ptr) {
+ ConfWriter_append_conf("#define CHY_HAS_OSATOMIC_CAS_PTR\n");
+ }
+ }
+ if (HeadCheck_check_header("sys/atomic.h")) {
+ has_sys_atomic_h = true;
+ ConfWriter_append_conf("#define CHY_HAS_SYS_ATOMIC_H\n");
+ }
+ if (HeadCheck_check_header("windows.h")
+ && HeadCheck_check_header("intrin.h")
+ ) {
+ has_intrin_h = true;
+ ConfWriter_append_conf("#define CHY_HAS_INTRIN_H\n");
+ }
+
+ /* Shorten */
+ ConfWriter_start_short_names();
+ if (has_libkern_osatomic_h) {
+ ConfWriter_shorten_macro("HAS_LIBKERN_OSATOMIC_H");
+ if (has_osatomic_cas_ptr) {
+ ConfWriter_shorten_macro("HAS_OSATOMIC_CAS_PTR");
+ }
+ }
+ if (has_sys_atomic_h) {
+ ConfWriter_shorten_macro("HAS_SYS_ATOMIC_H");
+ }
+ if (has_intrin_h) {
+ ConfWriter_shorten_macro("HAS_INTRIN_H");
+ }
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/AtomicOps.h b/charmonizer/src/Charmonizer/Probe/AtomicOps.h
new file mode 100644
index 0000000..277fbf9
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/AtomicOps.h
@@ -0,0 +1,55 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/AtomicOps.h
+ */
+
+#ifndef H_CHAZ_ATOMICOPS
+#define H_CHAZ_ATOMICOPS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+
+/* Run the AtomicOps module.
+ *
+ * These following symbols will be defined if the associated headers are
+ * available:
+ *
+ * HAS_LIBKERN_OSATOMIC_H <libkern/OSAtomic.h> (Mac OS X)
+ * HAS_SYS_ATOMIC_H <sys/atomic.h> (Solaris)
+ * HAS_INTRIN_H <intrin.h> (Windows)
+ *
+ * This symbol is defined if OSAtomicCompareAndSwapPtr is available:
+ *
+ * HAS_OSATOMIC_CAS_PTR
+ */
+void chaz_AtomicOps_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define AtomicOps_run chaz_AtomicOps_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_ATOMICOPS */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/DirManip.c b/charmonizer/src/Charmonizer/Probe/DirManip.c
new file mode 100644
index 0000000..0e8aa4e
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/DirManip.c
@@ -0,0 +1,124 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/Dir.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Probe/DirManip.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static const char cygwin_code[] =
+ QUOTE(#ifndef __CYGWIN__ )
+ QUOTE( #error "Not Cygwin" )
+ QUOTE(#endif )
+ QUOTE(int main() { return 0; } );
+
+void
+DirManip_run(void) {
+ FILE *f;
+ char dir_sep[3];
+ chaz_bool_t remove_zaps_dirs = false;
+ chaz_bool_t has_dirent_h = HeadCheck_check_header("dirent.h");
+ chaz_bool_t has_direct_h = HeadCheck_check_header("direct.h");
+ chaz_bool_t has_dirent_d_namlen = false;
+ chaz_bool_t has_dirent_d_type = false;
+
+ ConfWriter_start_module("DirManip");
+ Dir_init();
+
+ /* Header checks. */
+ if (has_dirent_h) {
+ ConfWriter_append_conf("#define CHY_HAS_DIRENT_H\n");
+ }
+ if (has_direct_h) {
+ ConfWriter_append_conf("#define CHY_HAS_DIRECT_H\n");
+ }
+
+ /* Check for members in struct dirent. */
+ if (has_dirent_h) {
+ has_dirent_d_namlen = HeadCheck_contains_member(
+ "struct dirent", "d_namlen",
+ "#include <sys/types.h>\n#include <dirent.h>"
+ );
+ if (has_dirent_d_namlen) {
+ ConfWriter_append_conf("#define CHY_HAS_DIRENT_D_NAMLEN\n", dir_sep);
+ }
+ has_dirent_d_type = HeadCheck_contains_member(
+ "struct dirent", "d_type",
+ "#include <sys/types.h>\n#include <dirent.h>"
+ );
+ if (has_dirent_d_type) {
+ ConfWriter_append_conf("#define CHY_HAS_DIRENT_D_TYPE\n", dir_sep);
+ }
+ }
+
+ if (Dir_mkdir_num_args == 2) {
+ /* It's two args, but the command isn't "mkdir". */
+ ConfWriter_append_conf("#define chy_makedir(_dir, _mode) %s(_dir, _mode)\n",
+ Dir_mkdir_command);
+ ConfWriter_append_conf("#define CHY_MAKEDIR_MODE_IGNORED 0\n");
+ }
+ else if (Dir_mkdir_num_args == 1) {
+ /* It's one arg... mode arg will be ignored. */
+ ConfWriter_append_conf("#define chy_makedir(_dir, _mode) %s(_dir)\n",
+ Dir_mkdir_command);
+ ConfWriter_append_conf("#define CHY_MAKEDIR_MODE_IGNORED 1\n");
+ }
+
+ if (CC_test_compile(cygwin_code, strlen(cygwin_code))) {
+ strcpy(dir_sep, "/");
+ }
+ else if (HeadCheck_check_header("windows.h")) {
+ strcpy(dir_sep, "\\\\");
+ }
+ else {
+ strcpy(dir_sep, "/");
+ }
+
+ ConfWriter_append_conf("#define CHY_DIR_SEP \"%s\"\n", dir_sep);
+
+ /* See whether remove works on directories. */
+ Dir_mkdir("_charm_test_remove_me");
+ if (0 == remove("_charm_test_remove_me")) {
+ remove_zaps_dirs = true;
+ ConfWriter_append_conf("#define CHY_REMOVE_ZAPS_DIRS\n");
+ }
+ Dir_rmdir("_charm_test_remove_me");
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ ConfWriter_shorten_macro("DIR_SEP");
+ if (has_dirent_h) { ConfWriter_shorten_macro("HAS_DIRENT_H"); }
+ if (has_direct_h) { ConfWriter_shorten_macro("HAS_DIRECT_H"); }
+ if (has_dirent_d_namlen) { ConfWriter_shorten_macro("HAS_DIRENT_D_NAMLEN"); }
+ if (has_dirent_d_type) { ConfWriter_shorten_macro("HAS_DIRENT_D_TYPE"); }
+ ConfWriter_shorten_function("makedir");
+ ConfWriter_shorten_macro("MAKEDIR_MODE_IGNORED");
+ if (remove_zaps_dirs) { ConfWriter_shorten_macro("REMOVE_ZAPS_DIRS"); }
+
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/DirManip.h b/charmonizer/src/Charmonizer/Probe/DirManip.h
new file mode 100644
index 0000000..1a7f65d
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/DirManip.h
@@ -0,0 +1,72 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/DirManip.h
+ */
+
+#ifndef H_CHAZ_DIRMANIP
+#define H_CHAZ_DIRMANIP
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The DirManip module exports or aliases symbols related to directory and file
+ * manipulation.
+ *
+ * Defined if the header files dirent.h and direct.h are available, respectively:
+ *
+ * HAS_DIRENT_H
+ * HAS_DIRECT_H
+ *
+ * Defined if struct dirent has these members.
+ *
+ * HAS_DIRENT_D_NAMLEN
+ * HAS_DIRENT_D_TYPE
+ *
+ * The "makedir" macro will be aliased to the POSIX-specified two-argument
+ * "mkdir" interface:
+ *
+ * makedir
+ *
+ * On some systems, the second argument to makedir will be ignored, in which
+ * case this symbol will be true; otherwise, it will be false: (TODO: This
+ * isn't verified and may sometimes be incorrect.)
+ *
+ * MAKEDIR_MODE_IGNORED
+ *
+ * String representing the system's directory separator:
+ *
+ * DIR_SEP
+ *
+ * True if the remove() function removes directories, false otherwise:
+ *
+ * REMOVE_ZAPS_DIRS
+ */
+void chaz_DirManip_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define DirManip_run chaz_DirManip_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_DIR_SEP */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Floats.c b/charmonizer/src/Charmonizer/Probe/Floats.c
new file mode 100644
index 0000000..eff8c4e
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Floats.c
@@ -0,0 +1,62 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/Floats.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+void
+Floats_run(void) {
+ ConfWriter_start_module("Floats");
+
+ ConfWriter_append_conf(
+ "typedef float chy_f32_t;\n"
+ "typedef double chy_f64_t;\n"
+ "#define CHY_HAS_F32_T\n"
+ "#define CHY_HAS_F64_T\n"
+ );
+
+ ConfWriter_append_conf(
+ "typedef union { chy_u32_t i; float f; } chy_floatu32;\n"
+ "static const chy_floatu32 chy_f32inf = {CHY_U32_C(0x7f800000)};\n"
+ "static const chy_floatu32 chy_f32neginf = {CHY_U32_C(0xff800000)};\n"
+ "static const chy_floatu32 chy_f32nan = {CHY_U32_C(0x7fc00000)};\n"
+ "#define CHY_F32_INF (chy_f32inf.f)\n"
+ "#define CHY_F32_NEGINF (chy_f32neginf.f)\n"
+ "#define CHY_F32_NAN (chy_f32nan.f)\n"
+ );
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ ConfWriter_shorten_typedef("f32_t");
+ ConfWriter_shorten_typedef("f64_t");
+ ConfWriter_shorten_macro("HAS_F32_T");
+ ConfWriter_shorten_macro("HAS_F64_T");
+ ConfWriter_shorten_macro("F32_INF");
+ ConfWriter_shorten_macro("F32_NEGINF");
+ ConfWriter_shorten_macro("F32_NAN");
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Floats.h b/charmonizer/src/Charmonizer/Probe/Floats.h
new file mode 100644
index 0000000..c0c92ea
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Floats.h
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/Floats.h -- floating point types.
+ *
+ * The following symbols will be created if the platform supports IEEE 754
+ * floating point types:
+ *
+ * F32_NAN
+ * F32_INF
+ * F32_NEGINF
+ *
+ * The following typedefs will be created if the platform supports IEEE 754
+ * floating point types:
+ *
+ * f32_t
+ * f64_t
+ *
+ * Availability of the preceding typedefs is indicated by which of these are
+ * defined:
+ *
+ * HAS_F32_T
+ * HAS_F64_T
+ *
+ * TODO: Actually test to see whether IEEE 754 is supported, rather than just
+ * lying about it.
+ */
+
+#ifndef H_CHAZ_FLOATS
+#define H_CHAZ_FLOATS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Run the Floats module.
+ */
+void
+chaz_Floats_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Floats_run chaz_Floats_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_FLOATS */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/FuncMacro.c b/charmonizer/src/Charmonizer/Probe/FuncMacro.c
new file mode 100644
index 0000000..322361d
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/FuncMacro.c
@@ -0,0 +1,154 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/FuncMacro.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/* Code for verifying ISO func macro. */
+static const char iso_func_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("%s", __func__); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+/* Code for verifying GNU func macro. */
+static const char gnuc_func_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("%s", __FUNCTION__); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+/* Code for verifying inline keyword. */
+static const char inline_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( static %s int foo() { return 1; } )
+ QUOTE( int main() { )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("%%d", foo()); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static char*
+S_try_inline(const char *keyword, size_t *output_len) {
+ char code[sizeof(inline_code) + 30];
+ sprintf(code, inline_code, keyword);
+ return CC_capture_output(code, strlen(code), output_len);
+}
+
+static const char* inline_options[] = {
+ "__inline",
+ "__inline__",
+ "inline"
+};
+static int num_inline_options = sizeof(inline_options) / sizeof(void*);
+
+void
+FuncMacro_run(void) {
+ int i;
+ char *output;
+ size_t output_len;
+ chaz_bool_t has_funcmac = false;
+ chaz_bool_t has_iso_funcmac = false;
+ chaz_bool_t has_gnuc_funcmac = false;
+ chaz_bool_t has_inline = false;
+
+ ConfWriter_start_module("FuncMacro");
+
+ /* Check for ISO func macro. */
+ output = CC_capture_output(iso_func_code, strlen(iso_func_code),
+ &output_len);
+ if (output != NULL && strncmp(output, "main", 4) == 0) {
+ has_funcmac = true;
+ has_iso_funcmac = true;
+ }
+ free(output);
+
+ /* Check for GNUC func macro. */
+ output = CC_capture_output(gnuc_func_code, strlen(gnuc_func_code),
+ &output_len);
+ if (output != NULL && strncmp(output, "main", 4) == 0) {
+ has_funcmac = true;
+ has_gnuc_funcmac = true;
+ }
+ free(output);
+
+ /* Write out common defines. */
+ if (has_funcmac) {
+ const char *macro_text = has_iso_funcmac
+ ? "__func__"
+ : "__FUNCTION__";
+ ConfWriter_append_conf(
+ "#define CHY_HAS_FUNC_MACRO\n"
+ "#define CHY_FUNC_MACRO %s\n",
+ macro_text
+ );
+ }
+
+ /* Write out specific defines. */
+ if (has_iso_funcmac) {
+ ConfWriter_append_conf("#define CHY_HAS_ISO_FUNC_MACRO\n");
+ }
+ if (has_gnuc_funcmac) {
+ ConfWriter_append_conf("#define CHY_HAS_GNUC_FUNC_MACRO\n");
+ }
+
+ /* Check for inline keyword. */
+
+ for (i = 0; i < num_inline_options; i++) {
+ const char *inline_option = inline_options[i];
+ output = S_try_inline(inline_option, &output_len);
+ if (output != NULL) {
+ has_inline = true;
+ ConfWriter_append_conf("#define CHY_INLINE %s\n", inline_option);
+ free(output);
+ break;
+ }
+ }
+ if (!has_inline) {
+ ConfWriter_append_conf("#define CHY_INLINE\n");
+ }
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ if (has_iso_funcmac) {
+ ConfWriter_shorten_macro("HAS_ISO_FUNC_MACRO");
+ }
+ if (has_gnuc_funcmac) {
+ ConfWriter_shorten_macro("HAS_GNUC_FUNC_MACRO");
+ }
+ if (has_iso_funcmac || has_gnuc_funcmac) {
+ ConfWriter_shorten_macro("HAS_FUNC_MACRO");
+ ConfWriter_shorten_macro("FUNC_MACRO");
+ }
+ ConfWriter_shorten_macro("INLINE");
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/FuncMacro.h b/charmonizer/src/Charmonizer/Probe/FuncMacro.h
new file mode 100644
index 0000000..20a2d94
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/FuncMacro.h
@@ -0,0 +1,63 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/FuncMacro.h
+ */
+
+#ifndef H_CHAZ_FUNC_MACRO
+#define H_CHAZ_FUNC_MACRO
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+
+/* Run the FuncMacro module.
+ *
+ * If __func__ successfully resolves, this will be defined:
+ *
+ * HAS_ISO_FUNC_MACRO
+ *
+ * If __FUNCTION__ successfully resolves, this will be defined:
+ *
+ * HAS_GNUC_FUNC_MACRO
+ *
+ * If one or the other succeeds, these will be defined:
+ *
+ * HAS_FUNC_MACRO
+ * FUNC_MACRO
+ *
+ * The "inline" keyword will also be probed for. If it is available, the
+ * following macro will be defined to "inline", otherwise it will be defined
+ * to nothing.
+ *
+ * INLINE
+ */
+void chaz_FuncMacro_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define FuncMacro_run chaz_FuncMacro_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_FUNC_MACRO */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Headers.c b/charmonizer/src/Charmonizer/Probe/Headers.c
new file mode 100644
index 0000000..ddbd954
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Headers.c
@@ -0,0 +1,230 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/Headers.h"
+#include <ctype.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+
+/* Keep track of which headers have succeeded. */
+static int keeper_count = 0;
+#define MAX_KEEPER_COUNT 200
+static const char *keepers[MAX_KEEPER_COUNT + 1] = { NULL };
+
+/* Add a header to the keepers array.
+ */
+static void
+S_keep(const char *header_name);
+
+static size_t aff_buf_size = 0;
+static char *aff_buf = NULL;
+
+/* Transform "header.h" into "CHY_HAS_HEADER_H, storing the result in
+ * [aff_buf].
+ */
+static void
+S_encode_affirmation(const char *header_name);
+
+#define NUM_C89_HEADERS 15
+const char *c89_headers[] = {
+ "assert.h",
+ "ctype.h",
+ "errno.h",
+ "float.h",
+ "limits.h",
+ "locale.h",
+ "math.h",
+ "setjmp.h",
+ "signal.h",
+ "stdarg.h",
+ "stddef.h",
+ "stdio.h",
+ "stdlib.h",
+ "string.h",
+ "time.h",
+ NULL
+};
+
+#define NUM_POSIX_HEADERS 14
+const char *posix_headers[] = {
+ "cpio.h",
+ "dirent.h",
+ "fcntl.h",
+ "grp.h",
+ "pwd.h",
+ "sys/stat.h",
+ "sys/times.h",
+ "sys/types.h",
+ "sys/utsname.h",
+ "sys/wait.h",
+ "tar.h",
+ "termios.h",
+ "unistd.h",
+ "utime.h",
+ NULL
+};
+
+#define NUM_WIN_HEADERS 3
+const char *win_headers[] = {
+ "io.h",
+ "windows.h",
+ "process.h",
+ NULL
+};
+
+chaz_bool_t
+Headers_check(const char *header_name) {
+ return HeadCheck_check_header(header_name);
+}
+
+void
+Headers_run(void) {
+ int i;
+ chaz_bool_t has_posix = false;
+ chaz_bool_t has_c89 = false;
+
+ keeper_count = 0;
+
+ ConfWriter_start_module("Headers");
+
+ /* Try for all POSIX headers in one blast. */
+ if (HeadCheck_check_many_headers((const char**)posix_headers)) {
+ has_posix = true;
+ ConfWriter_append_conf("#define CHY_HAS_POSIX\n");
+ for (i = 0; posix_headers[i] != NULL; i++) {
+ S_keep(posix_headers[i]);
+ }
+ }
+ /* Test one-at-a-time. */
+ else {
+ for (i = 0; posix_headers[i] != NULL; i++) {
+ if (HeadCheck_check_header(posix_headers[i])) {
+ S_keep(posix_headers[i]);
+ }
+ }
+ }
+
+ /* Test for all c89 headers in one blast. */
+ if (HeadCheck_check_many_headers((const char**)c89_headers)) {
+ has_c89 = true;
+ ConfWriter_append_conf("#define CHY_HAS_C89\n");
+ ConfWriter_append_conf("#define CHY_HAS_C90\n");
+ for (i = 0; c89_headers[i] != NULL; i++) {
+ S_keep(c89_headers[i]);
+ }
+ }
+ /* Test one-at-a-time. */
+ else {
+ for (i = 0; c89_headers[i] != NULL; i++) {
+ if (HeadCheck_check_header(c89_headers[i])) {
+ S_keep(c89_headers[i]);
+ }
+ }
+ }
+
+ /* Test for all Windows headers in one blast */
+ if (HeadCheck_check_many_headers((const char**)win_headers)) {
+ for (i = 0; win_headers[i] != NULL; i++) {
+ S_keep(win_headers[i]);
+ }
+ }
+ /* Test one-at-a-time. */
+ else {
+ for (i = 0; win_headers[i] != NULL; i++) {
+ if (HeadCheck_check_header(win_headers[i])) {
+ S_keep(win_headers[i]);
+ }
+ }
+ }
+
+ /* One-offs. */
+ if (HeadCheck_check_header("pthread.h")) {
+ S_keep("pthread.h");
+ }
+
+ /* Append the config with every header detected so far. */
+ for (i = 0; keepers[i] != NULL; i++) {
+ S_encode_affirmation(keepers[i]);
+ ConfWriter_append_conf("#define CHY_%s\n", aff_buf);
+ }
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ if (has_posix) {
+ ConfWriter_shorten_macro("HAS_POSIX");
+ }
+ if (has_c89) {
+ ConfWriter_shorten_macro("HAS_C89");
+ ConfWriter_shorten_macro("HAS_C90");
+ }
+ for (i = 0; keepers[i] != NULL; i++) {
+ S_encode_affirmation(keepers[i]);
+ ConfWriter_shorten_macro(aff_buf);
+ }
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+static void
+S_keep(const char *header_name) {
+ if (keeper_count >= MAX_KEEPER_COUNT) {
+ Util_die("Too many keepers -- increase MAX_KEEPER_COUNT");
+ }
+ keepers[keeper_count++] = header_name;
+ keepers[keeper_count] = NULL;
+}
+
+static void
+S_encode_affirmation(const char *header_name) {
+ char *buf, *buf_end;
+ size_t len = strlen(header_name) + sizeof("HAS_");
+
+ /* Grow buffer and start off with "HAS_". */
+ if (aff_buf_size < len + 1) {
+ free(aff_buf);
+ aff_buf_size = len + 1;
+ aff_buf = (char*)malloc(aff_buf_size);
+ }
+ strcpy(aff_buf, "HAS_");
+
+ /* Transform one char at a time. */
+ for (buf = aff_buf + sizeof("HAS_") - 1, buf_end = aff_buf + len;
+ buf < buf_end;
+ header_name++, buf++
+ ) {
+ if (*header_name == '\0') {
+ *buf = '\0';
+ break;
+ }
+ else if (isalnum(*header_name)) {
+ *buf = toupper(*header_name);
+ }
+ else {
+ *buf = '_';
+ }
+ }
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Headers.h b/charmonizer/src/Charmonizer/Probe/Headers.h
new file mode 100644
index 0000000..9beb38b
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Headers.h
@@ -0,0 +1,103 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/Headers.h
+ */
+
+#ifndef H_CHAZ_HEADERS
+#define H_CHAZ_HEADERS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include <stdio.h>
+#include "Charmonizer/Core/Defines.h"
+
+/* Check whether a particular header file is available. The test-compile is
+ * only run the first time a given request is made.
+ */
+chaz_bool_t
+chaz_Headers_check(const char *header_name);
+
+/* Run the Headers module.
+ *
+ * Exported symbols:
+ *
+ * If HAS_C89 is declared, this system has all the header files described in
+ * Ansi C 1989. HAS_C90 is a synonym. (It would be surprising if they are
+ * not defined, because Charmonizer itself assumes C89.)
+ *
+ * HAS_C89
+ * HAS_C90
+ *
+ * One symbol is exported for each C89 header file:
+ *
+ * HAS_ASSERT_H
+ * HAS_CTYPE_H
+ * HAS_ERRNO_H
+ * HAS_FLOAT_H
+ * HAS_LIMITS_H
+ * HAS_LOCALE_H
+ * HAS_MATH_H
+ * HAS_SETJMP_H
+ * HAS_SIGNAL_H
+ * HAS_STDARG_H
+ * HAS_STDDEF_H
+ * HAS_STDIO_H
+ * HAS_STDLIB_H
+ * HAS_STRING_H
+ * HAS_TIME_H
+ *
+ * One symbol is exported for every POSIX header present, and HAS_POSIX is
+ * exported if they're all there.
+ *
+ * HAS_POSIX
+ *
+ * HAS_CPIO_H
+ * HAS_DIRENT_H
+ * HAS_FCNTL_H
+ * HAS_GRP_H
+ * HAS_PWD_H
+ * HAS_SYS_STAT_H
+ * HAS_SYS_TIMES_H
+ * HAS_SYS_TYPES_H
+ * HAS_SYS_UTSNAME_H
+ * HAS_WAIT_H
+ * HAS_TAR_H
+ * HAS_TERMIOS_H
+ * HAS_UNISTD_H
+ * HAS_UTIME_H
+ *
+ * If pthread.h is available, this will be exported:
+ *
+ * HAS_PTHREAD_H
+ */
+void
+chaz_Headers_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Headers_run chaz_Headers_run
+ #define Headers_check chaz_Headers_check
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_HEADERS */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Integers.c b/charmonizer/src/Charmonizer/Probe/Integers.c
new file mode 100644
index 0000000..7f0e0a0
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Integers.c
@@ -0,0 +1,479 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/Integers.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+/* Determine endian-ness of this machine.
+ */
+static chaz_bool_t
+S_machine_is_big_endian(void);
+
+static const char sizes_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main () { )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("%d ", (int)sizeof(char)); )
+ QUOTE( printf("%d ", (int)sizeof(short)); )
+ QUOTE( printf("%d ", (int)sizeof(int)); )
+ QUOTE( printf("%d ", (int)sizeof(long)); )
+ QUOTE( printf("%d ", (int)sizeof(void*)); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static const char type64_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() )
+ QUOTE( { )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("%%d", (int)sizeof(%s)); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static const char literal64_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( #define big 9000000000000000000%s )
+ QUOTE( int main() )
+ QUOTE( { )
+ QUOTE( Charm_Setup; )
+ QUOTE( int truncated = (int)big; )
+ QUOTE( printf("%%d\n", truncated); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+void
+Integers_run(void) {
+ char *output;
+ size_t output_len;
+ int sizeof_char = -1;
+ int sizeof_short = -1;
+ int sizeof_int = -1;
+ int sizeof_ptr = -1;
+ int sizeof_long = -1;
+ int sizeof_long_long = -1;
+ int sizeof___int64 = -1;
+ chaz_bool_t has_8 = false;
+ chaz_bool_t has_16 = false;
+ chaz_bool_t has_32 = false;
+ chaz_bool_t has_64 = false;
+ chaz_bool_t has_long_long = false;
+ chaz_bool_t has___int64 = false;
+ chaz_bool_t has_inttypes = HeadCheck_check_header("inttypes.h");
+ chaz_bool_t has_stdint = HeadCheck_check_header("stdint.h");
+ char i32_t_type[10];
+ char i32_t_postfix[10];
+ char u32_t_postfix[10];
+ char i64_t_type[10];
+ char i64_t_postfix[10];
+ char u64_t_postfix[10];
+ char code_buf[1000];
+
+ ConfWriter_start_module("Integers");
+
+ /* Document endian-ness. */
+ if (S_machine_is_big_endian()) {
+ ConfWriter_append_conf("#define CHY_BIG_END\n");
+ }
+ else {
+ ConfWriter_append_conf("#define CHY_LITTLE_END\n");
+ }
+
+ /* Record sizeof() for several common integer types. */
+ output = CC_capture_output(sizes_code, strlen(sizes_code), &output_len);
+ if (output != NULL) {
+ char *end_ptr = output;
+
+ sizeof_char = strtol(output, &end_ptr, 10);
+ output = end_ptr;
+ sizeof_short = strtol(output, &end_ptr, 10);
+ output = end_ptr;
+ sizeof_int = strtol(output, &end_ptr, 10);
+ output = end_ptr;
+ sizeof_long = strtol(output, &end_ptr, 10);
+ output = end_ptr;
+ sizeof_ptr = strtol(output, &end_ptr, 10);
+ }
+
+ /* Determine whether long longs are available. */
+ sprintf(code_buf, type64_code, "long long");
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ has_long_long = true;
+ sizeof_long_long = strtol(output, NULL, 10);
+ }
+
+ /* Determine whether the __int64 type is available. */
+ sprintf(code_buf, type64_code, "__int64");
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ has___int64 = true;
+ sizeof___int64 = strtol(output, NULL, 10);
+ }
+
+ /* Figure out which integer types are available. */
+ if (sizeof_char == 1) {
+ has_8 = true;
+ }
+ if (sizeof_short == 2) {
+ has_16 = true;
+ }
+ if (sizeof_int == 4) {
+ has_32 = true;
+ strcpy(i32_t_type, "int");
+ strcpy(i32_t_postfix, "");
+ strcpy(u32_t_postfix, "U");
+ }
+ else if (sizeof_long == 4) {
+ has_32 = true;
+ strcpy(i32_t_type, "long");
+ strcpy(i32_t_postfix, "L");
+ strcpy(u32_t_postfix, "UL");
+ }
+ if (sizeof_long == 8) {
+ has_64 = true;
+ strcpy(i64_t_type, "long");
+ }
+ else if (sizeof_long_long == 8) {
+ has_64 = true;
+ strcpy(i64_t_type, "long long");
+ }
+ else if (sizeof___int64 == 8) {
+ has_64 = true;
+ strcpy(i64_t_type, "__int64");
+ }
+
+ /* Probe for 64-bit literal syntax. */
+ if (has_64 && sizeof_long == 8) {
+ strcpy(i64_t_postfix, "L");
+ strcpy(u64_t_postfix, "UL");
+ }
+ else if (has_64) {
+ sprintf(code_buf, literal64_code, "LL");
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ strcpy(i64_t_postfix, "LL");
+ }
+ else {
+ sprintf(code_buf, literal64_code, "i64");
+ output = CC_capture_output(code_buf, strlen(code_buf),
+ &output_len);
+ if (output != NULL) {
+ strcpy(i64_t_postfix, "i64");
+ }
+ else {
+ Util_die("64-bit types, but no literal syntax found");
+ }
+ }
+ sprintf(code_buf, literal64_code, "ULL");
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ strcpy(u64_t_postfix, "ULL");
+ }
+ else {
+ sprintf(code_buf, literal64_code, "Ui64");
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ strcpy(u64_t_postfix, "Ui64");
+ }
+ else {
+ Util_die("64-bit types, but no literal syntax found");
+ }
+ }
+ }
+
+ /* Write out some conditional defines. */
+ if (has_inttypes) {
+ ConfWriter_append_conf("#define CHY_HAS_INTTYPES_H\n");
+ }
+ if (has_stdint) {
+ ConfWriter_append_conf("#define CHY_HAS_STDINT_H\n");
+ }
+ if (has_long_long) {
+ ConfWriter_append_conf("#define CHY_HAS_LONG_LONG\n");
+ }
+ if (has___int64) {
+ ConfWriter_append_conf("#define CHY_HAS___INT64\n");
+ }
+
+ /* Write out sizes. */
+ ConfWriter_append_conf("#define CHY_SIZEOF_CHAR %d\n", sizeof_char);
+ ConfWriter_append_conf("#define CHY_SIZEOF_SHORT %d\n", sizeof_short);
+ ConfWriter_append_conf("#define CHY_SIZEOF_INT %d\n", sizeof_int);
+ ConfWriter_append_conf("#define CHY_SIZEOF_LONG %d\n", sizeof_long);
+ ConfWriter_append_conf("#define CHY_SIZEOF_PTR %d\n", sizeof_ptr);
+ if (has_long_long) {
+ ConfWriter_append_conf("#define CHY_SIZEOF_LONG_LONG %d\n",
+ sizeof_long_long);
+ }
+ if (has___int64) {
+ ConfWriter_append_conf("#define CHY_SIZEOF___INT64 %d\n",
+ sizeof___int64);
+ }
+
+ /* Write affirmations, typedefs and maximums/minimums. */
+ ConfWriter_append_conf("typedef int chy_bool_t;\n");
+ if (has_stdint) {
+ ConfWriter_append_conf("#include <stdint.h>\n");
+ }
+ else {
+ /* we support only the following subset of stdint.h
+ * int8_t
+ * int16_t
+ * int32_t
+ * int64_t
+ * uint8_t
+ * uint16_t
+ * uint32_t
+ * uint64_t
+ */
+ if (has_8) {
+ ConfWriter_append_conf(
+ "typedef signed char int8_t;\n"
+ "typedef unsigned char uint8_t;\n"
+ );
+ }
+ if (has_16) {
+ ConfWriter_append_conf(
+ "typedef short int16_t;\n"
+ "typedef unsigned short uint16_t;\n"
+ );
+ }
+ if (has_32) {
+ ConfWriter_append_conf(
+ "typedef %s int32_t;\n", i32_t_type
+ );
+ ConfWriter_append_conf(
+ "typedef unsigned %s uint32_t;\n", i32_t_type
+ );
+ }
+ if (has_64) {
+ ConfWriter_append_conf(
+ "typedef %s int64_t;\n", i64_t_type
+ );
+ ConfWriter_append_conf(
+ "typedef unsigned %s uint64_t;\n", i64_t_type
+ );
+ }
+ }
+ if (has_8) {
+ ConfWriter_append_conf(
+ "#define CHY_HAS_I8_T\n"
+ "typedef signed char chy_i8_t;\n"
+ "typedef unsigned char chy_u8_t;\n"
+ "#define CHY_I8_MAX 0x7F\n"
+ "#define CHY_I8_MIN (-I8_MAX - 1)\n"
+ "#define CHY_U8_MAX (I8_MAX * 2 + 1)\n"
+ );
+ }
+ if (has_16) {
+ ConfWriter_append_conf(
+ "#define CHY_HAS_I16_T\n"
+ "typedef short chy_i16_t;\n"
+ "typedef unsigned short chy_u16_t;\n"
+ "#define CHY_I16_MAX 0x7FFF\n"
+ "#define CHY_I16_MIN (-I16_MAX - 1)\n"
+ "#define CHY_U16_MAX (I16_MAX * 2 + 1)\n"
+ );
+ }
+ if (has_32) {
+ ConfWriter_append_conf("#define CHY_HAS_I32_T\n");
+ ConfWriter_append_conf("typedef %s chy_i32_t;\n", i32_t_type);
+ ConfWriter_append_conf("typedef unsigned %s chy_u32_t;\n",
+ i32_t_type);
+ ConfWriter_append_conf("#define CHY_I32_MAX 0x7FFFFFFF%s\n",
+ i32_t_postfix);
+ ConfWriter_append_conf("#define CHY_I32_MIN (-I32_MAX - 1)\n");
+ ConfWriter_append_conf("#define CHY_U32_MAX (I32_MAX * 2%s + 1%s)\n",
+ u32_t_postfix, u32_t_postfix);
+ }
+ if (has_64) {
+ ConfWriter_append_conf("#define CHY_HAS_I64_T\n");
+ ConfWriter_append_conf("typedef %s chy_i64_t;\n", i64_t_type);
+ ConfWriter_append_conf("typedef unsigned %s chy_u64_t;\n",
+ i64_t_type);
+ ConfWriter_append_conf("#define CHY_I64_MAX 0x7FFFFFFFFFFFFFFF%s\n",
+ i64_t_postfix);
+ ConfWriter_append_conf("#define CHY_I64_MIN (-I64_MAX - 1%s)\n",
+ i64_t_postfix);
+ ConfWriter_append_conf("#define CHY_U64_MAX (I64_MAX * 2%s + 1%s)\n",
+ u64_t_postfix, u64_t_postfix);
+ }
+
+ /* Create the I64P and U64P printf macros. */
+ if (has_64) {
+ int i;
+ const char *options[] = {
+ "ll",
+ "l",
+ "L",
+ "q", /* Some *BSDs */
+ "I64", /* Microsoft */
+ NULL,
+ };
+
+ /* Buffer to hold the code, and its start and end. */
+ static const char format_64_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("%%%su", 18446744073709551615%s); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+ for (i = 0; options[i] != NULL; i++) {
+ /* Try to print 2**64-1, and see if we get it back intact. */
+ sprintf(code_buf, format_64_code, options[i], u64_t_postfix);
+ output = CC_capture_output(code_buf, strlen(code_buf),
+ &output_len);
+
+ if (output_len != 0
+ && strcmp(output, "18446744073709551615") == 0
+ ) {
+ ConfWriter_append_conf("#define CHY_I64P \"%sd\"\n",
+ options[i]);
+ ConfWriter_append_conf("#define CHY_U64P \"%su\"\n",
+ options[i]);
+ break;
+ }
+ }
+
+ }
+
+ /* Write out the 32-bit and 64-bit literal macros. */
+ if (has_32) {
+ if (strcmp(i32_t_postfix, "") == 0) {
+ ConfWriter_append_conf("#define CHY_I32_C(n) n\n");
+ ConfWriter_append_conf("#define CHY_U32_C(n) n##%s\n",
+ u32_t_postfix);
+ }
+ else {
+ ConfWriter_append_conf("#define CHY_I32_C(n) n##%s\n",
+ i32_t_postfix);
+ ConfWriter_append_conf("#define CHY_U32_C(n) n##%s\n",
+ u32_t_postfix);
+ }
+ }
+ if (has_64) {
+ ConfWriter_append_conf("#define CHY_I64_C(n) n##%s\n", i64_t_postfix);
+ ConfWriter_append_conf("#define CHY_U64_C(n) n##%s\n", u64_t_postfix);
+ }
+
+ /* Create macro for promoting pointers to integers. */
+ if (has_64) {
+ if (sizeof_ptr == 8) {
+ ConfWriter_append_conf("#define CHY_PTR_TO_I64(ptr) "
+ "((chy_i64_t)(chy_u64_t)(ptr))\n");
+ }
+ else {
+ ConfWriter_append_conf("#define CHY_PTR_TO_I64(ptr) "
+ "((chy_i64_t)(chy_u32_t)(ptr))\n");
+ }
+ }
+
+ /* True and false. */
+ ConfWriter_append_conf(
+ "#ifndef true\n"
+ " #define true 1\n"
+ "#endif\n"
+ "#ifndef false\n"
+ " #define false 0\n"
+ "#endif\n"
+ );
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ if (S_machine_is_big_endian()) {
+ ConfWriter_shorten_macro("BIG_END");
+ }
+ else {
+ ConfWriter_shorten_macro("LITTLE_END");
+ }
+ ConfWriter_shorten_macro("SIZEOF_CHAR");
+ ConfWriter_shorten_macro("SIZEOF_SHORT");
+ ConfWriter_shorten_macro("SIZEOF_LONG");
+ ConfWriter_shorten_macro("SIZEOF_INT");
+ ConfWriter_shorten_macro("SIZEOF_PTR");
+ if (has_long_long) {
+ ConfWriter_shorten_macro("HAS_LONG_LONG");
+ ConfWriter_shorten_macro("SIZEOF_LONG_LONG");
+ }
+ if (has___int64) {
+ ConfWriter_shorten_macro("HAS___INT64");
+ ConfWriter_shorten_macro("SIZEOF___INT64");
+ }
+ if (has_inttypes) {
+ ConfWriter_shorten_macro("HAS_INTTYPES_H");
+ }
+ ConfWriter_shorten_typedef("bool_t");
+ if (has_8) {
+ ConfWriter_shorten_macro("HAS_I8_T");
+ ConfWriter_shorten_typedef("i8_t");
+ ConfWriter_shorten_typedef("u8_t");
+ ConfWriter_shorten_macro("I8_MAX");
+ ConfWriter_shorten_macro("I8_MIN");
+ ConfWriter_shorten_macro("U8_MAX");
+ }
+ if (has_16) {
+ ConfWriter_shorten_macro("HAS_I16_T");
+ ConfWriter_shorten_typedef("i16_t");
+ ConfWriter_shorten_typedef("u16_t");
+ ConfWriter_shorten_macro("I16_MAX");
+ ConfWriter_shorten_macro("I16_MIN");
+ ConfWriter_shorten_macro("U16_MAX");
+ }
+ if (has_32) {
+ ConfWriter_shorten_macro("HAS_I32_T");
+ ConfWriter_shorten_typedef("i32_t");
+ ConfWriter_shorten_typedef("u32_t");
+ ConfWriter_shorten_macro("I32_MAX");
+ ConfWriter_shorten_macro("I32_MIN");
+ ConfWriter_shorten_macro("U32_MAX");
+ ConfWriter_shorten_macro("I32_C");
+ ConfWriter_shorten_macro("U32_C");
+ }
+ if (has_64) {
+ ConfWriter_shorten_macro("HAS_I64_T");
+ ConfWriter_shorten_typedef("i64_t");
+ ConfWriter_shorten_typedef("u64_t");
+ ConfWriter_shorten_macro("I64_MAX");
+ ConfWriter_shorten_macro("I64_MIN");
+ ConfWriter_shorten_macro("U64_MAX");
+ ConfWriter_shorten_macro("I64P");
+ ConfWriter_shorten_macro("U64P");
+ ConfWriter_shorten_macro("I64_C");
+ ConfWriter_shorten_macro("U64_C");
+ ConfWriter_shorten_macro("PTR_TO_I64");
+ }
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+static chaz_bool_t
+S_machine_is_big_endian(void) {
+ long one = 1;
+ return !(*((char*)(&one)));
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Integers.h b/charmonizer/src/Charmonizer/Probe/Integers.h
new file mode 100644
index 0000000..701ffa0
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Integers.h
@@ -0,0 +1,150 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/Integers.h -- info about integer types and sizes.
+ *
+ * One or the other of these will be defined, depending on whether the
+ * processor is big-endian or little-endian.
+ *
+ * BIG_END
+ * LITTLE_END
+ *
+ * These will always be defined:
+ *
+ * SIZEOF_CHAR
+ * SIZEOF_SHORT
+ * SIZEOF_INT
+ * SIZEOF_LONG
+ * SIZEOF_PTR
+ *
+ * If long longs are available these symbols will be defined:
+ *
+ * HAS_LONG_LONG
+ * SIZEOF_LONG_LONG
+ *
+ * Similarly, with the __int64 type (the sizeof is included for completeness):
+ *
+ * HAS___INT64
+ * SIZEOF___INT64
+ *
+ * If the inttypes.h or stdint.h header files are available, these may be
+ * defined:
+ *
+ * HAS_INTTYPES_H
+ * HAS_STDINT_H
+ *
+ * If stdint.h is is available, it will be pound-included in the configuration
+ * header. If it is not, the following typedefs will be defined if possible:
+ *
+ * int8_t
+ * int16_t
+ * int32_t
+ * int64_t
+ * uint8_t
+ * uint16_t
+ * uint32_t
+ * uint64_t
+ *
+ * The following typedefs will be created if a suitable integer type exists,
+ * as will most often be the case. However, if for example a char is 64 bits
+ * (as on certain Crays), no 8-bit types will be defined, or if no 64-bit
+ * integer type is available, no 64-bit types will be defined, etc.
+ *
+ * bool_t
+ * i8_t
+ * u8_t
+ * i16_t
+ * u16_t
+ * i32_t
+ * u32_t
+ * i64_t
+ * u64_t
+ *
+ * Availability of the preceding integer typedefs is indicated by which of
+ * these are defined:
+ *
+ * HAS_I8_T
+ * HAS_I16_T
+ * HAS_I32_T
+ * HAS_I64_T
+ *
+ * Maximums will be defined for all available integer types (save bool_t), and
+ * minimums for all available signed types.
+ *
+ * I8_MAX
+ * U8_MAX
+ * I16_MAX
+ * U16_MAX
+ * I32_MAX
+ * U32_MAX
+ * I64_MAX
+ * U64_MAX
+ * I8_MIN
+ * I16_MIN
+ * I32_MIN
+ * I64_MIN
+ *
+ * If 64-bit integers are available, this macro will promote pointers to i64_t
+ * safely.
+ *
+ * PTR_TO_I64(ptr)
+ *
+ * If 64-bit integers are available, these macros will expand to the printf
+ * conversion specification for signed and unsigned versions (most commonly
+ * "lld" and "llu").
+ *
+ * I64P
+ * U64P
+ *
+ * 32-bit and 64-bit literals can be spec'd via these macros, which append the
+ * appropriate postfix:
+ *
+ * I32_C(n)
+ * U32_C(n)
+ * I64_C(n)
+ * U64_C(n)
+ *
+ * These symbols will be defined if they are not already:
+ *
+ * true
+ * false
+ */
+
+#ifndef H_CHAZ_INTEGERS
+#define H_CHAZ_INTEGERS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+
+/* Run the Integers module.
+ */
+void chaz_Integers_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Integers_run chaz_Integers_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_INTEGERS */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/LargeFiles.c b/charmonizer/src/Charmonizer/Probe/LargeFiles.c
new file mode 100644
index 0000000..f1f800e
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/LargeFiles.c
@@ -0,0 +1,431 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Stat.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/LargeFiles.h"
+#include <errno.h>
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+/* Sets of symbols which might provide large file support. */
+typedef struct off64_combo {
+ const char *includes;
+ const char *fopen_command;
+ const char *ftell_command;
+ const char *fseek_command;
+ const char *offset64_type;
+} off64_combo;
+static off64_combo off64_combos[] = {
+ { "#include <sys/types.h>\n", "fopen64", "ftello64", "fseeko64", "off64_t" },
+ { "#include <sys/types.h>\n", "fopen", "ftello64", "fseeko64", "off64_t" },
+ { "#include <sys/types.h>\n", "fopen", "ftello", "fseeko", "off_t" },
+ { "", "fopen", "ftell", "fseek", "off_t" },
+ { "", "fopen", "_ftelli64", "_fseeki64", "__int64" },
+ { "", "fopen", "ftell", "fseek", "long" },
+ { NULL, NULL, NULL, NULL, NULL }
+};
+
+typedef struct unbuff_combo {
+ const char *includes;
+ const char *lseek_command;
+ const char *pread64_command;
+} unbuff_combo;
+static unbuff_combo unbuff_combos[] = {
+ { "#include <unistd.h>\n#include <fcntl.h>\n", "lseek64", "pread64" },
+ { "#include <unistd.h>\n#include <fcntl.h>\n", "lseek", "pread" },
+ { "#include <io.h>\n#include <fcntl.h>\n", "_lseeki64", "NO_PREAD64" },
+ { NULL, NULL, NULL }
+};
+
+/* Check what name 64-bit ftell, fseek go by.
+ */
+static chaz_bool_t
+S_probe_off64(off64_combo *combo);
+
+/* Check for a 64-bit lseek.
+ */
+static chaz_bool_t
+S_probe_lseek(unbuff_combo *combo);
+
+/* Check for a 64-bit pread.
+ */
+static chaz_bool_t
+S_probe_pread64(unbuff_combo *combo);
+
+/* Determine whether we can use sparse files.
+ */
+static chaz_bool_t
+S_check_sparse_files(void);
+
+/* Helper for check_sparse_files().
+ */
+static void
+S_test_sparse_file(long offset, Stat *st);
+
+/* See if trying to write a 5 GB file in a subprocess bombs out. If it
+ * doesn't, then the test suite can safely verify large file support.
+ */
+static chaz_bool_t
+S_can_create_big_files(void);
+
+/* Vars for holding lfs commands, once they're discovered. */
+static char fopen_command[10];
+static char fseek_command[10];
+static char ftell_command[10];
+static char lseek_command[10];
+static char pread64_command[10];
+static char off64_type[10];
+
+void
+LargeFiles_run(void) {
+ chaz_bool_t success = false;
+ chaz_bool_t found_lseek = false;
+ chaz_bool_t found_pread64 = false;
+ unsigned i;
+
+ ConfWriter_start_module("LargeFiles");
+
+ /* See if off64_t and friends exist or have synonyms. */
+ for (i = 0; off64_combos[i].includes != NULL; i++) {
+ off64_combo combo = off64_combos[i];
+ success = S_probe_off64(&combo);
+ if (success) {
+ strcpy(fopen_command, combo.fopen_command);
+ strcpy(fseek_command, combo.fseek_command);
+ strcpy(ftell_command, combo.ftell_command);
+ strcpy(off64_type, combo.offset64_type);
+ break;
+ }
+ }
+
+ /* Write the affirmations/definitions. */
+ if (success) {
+ ConfWriter_append_conf("#define CHY_HAS_LARGE_FILE_SUPPORT\n");
+ /* Alias these only if they're not already provided and correct. */
+ if (strcmp(off64_type, "off64_t") != 0) {
+ ConfWriter_append_conf("#define chy_off64_t %s\n", off64_type);
+ ConfWriter_append_conf("#define chy_fopen64 %s\n", fopen_command);
+ ConfWriter_append_conf("#define chy_ftello64 %s\n", ftell_command);
+ ConfWriter_append_conf("#define chy_fseeko64 %s\n", fseek_command);
+ }
+ }
+
+ /* Probe for 64-bit versions of lseek and pread (if we have an off64_t). */
+ if (success) {
+ for (i = 0; unbuff_combos[i].lseek_command != NULL; i++) {
+ unbuff_combo combo = unbuff_combos[i];
+ found_lseek = S_probe_lseek(&combo);
+ if (found_lseek) {
+ strcpy(lseek_command, combo.lseek_command);
+ ConfWriter_append_conf("#define chy_lseek64 %s\n",
+ lseek_command);
+ break;
+ }
+ }
+ for (i = 0; unbuff_combos[i].pread64_command != NULL; i++) {
+ unbuff_combo combo = unbuff_combos[i];
+ found_pread64 = S_probe_pread64(&combo);
+ if (found_pread64) {
+ strcpy(pread64_command, combo.pread64_command);
+ ConfWriter_append_conf("#define chy_pread64 %s\n",
+ pread64_command);
+ found_pread64 = true;
+ break;
+ }
+ }
+ }
+
+ /* Check for sparse files. */
+ if (S_check_sparse_files()) {
+ ConfWriter_append_conf("#define CHAZ_HAS_SPARSE_FILES\n");
+ /* See if we can create a 5 GB file without crashing. */
+ if (success && S_can_create_big_files()) {
+ ConfWriter_append_conf("#define CHAZ_CAN_CREATE_BIG_FILES\n");
+ }
+ }
+ else {
+ ConfWriter_append_conf("#define CHAZ_NO_SPARSE_FILES\n");
+ }
+
+ /* Short names. */
+ if (success) {
+ ConfWriter_start_short_names();
+ ConfWriter_shorten_macro("HAS_LARGE_FILE_SUPPORT");
+
+ /* Alias these only if they're not already provided and correct. */
+ if (strcmp(off64_type, "off64_t") != 0) {
+ ConfWriter_shorten_typedef("off64_t");
+ ConfWriter_shorten_function("fopen64");
+ ConfWriter_shorten_function("ftello64");
+ ConfWriter_shorten_function("fseeko64");
+ }
+ if (found_lseek && strcmp(lseek_command, "lseek64") != 0) {
+ ConfWriter_shorten_function("lseek64");
+ }
+ if (found_pread64 && strcmp(pread64_command, "pread64") != 0) {
+ ConfWriter_shorten_function("pread64");
+ }
+ ConfWriter_end_short_names();
+ }
+
+ ConfWriter_end_module();
+}
+
+/* Code for checking ftello64 and friends. */
+static const char off64_code[] =
+ QUOTE( %s )
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( %s pos; )
+ QUOTE( FILE *f; )
+ QUOTE( Charm_Setup; )
+ QUOTE( f = %s("_charm_off64", "w"); )
+ QUOTE( if (f == NULL) return -1; )
+ QUOTE( printf("%%d", (int)sizeof(%s)); )
+ QUOTE( pos = %s(stdout); )
+ QUOTE( %s(stdout, 0, SEEK_SET); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+
+static chaz_bool_t
+S_probe_off64(off64_combo *combo) {
+ char *output = NULL;
+ size_t output_len;
+ size_t needed = sizeof(off64_code)
+ + (2 * strlen(combo->offset64_type))
+ + strlen(combo->fopen_command)
+ + strlen(combo->ftell_command)
+ + strlen(combo->fseek_command)
+ + 20;
+ char *code_buf = (char*)malloc(needed);
+ chaz_bool_t success = false;
+
+ /* Prepare the source code. */
+ sprintf(code_buf, off64_code, combo->includes, combo->offset64_type,
+ combo->fopen_command, combo->offset64_type, combo->ftell_command,
+ combo->fseek_command);
+
+ /* Verify compilation and that the offset type has 8 bytes. */
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ long size = strtol(output, NULL, 10);
+ if (size == 8) {
+ success = true;
+ }
+ free(output);
+ }
+
+ if (!Util_remove_and_verify("_charm_off64")) {
+ Util_die("Failed to remove '_charm_off64'");
+ }
+
+ return success;
+}
+
+/* Code for checking 64-bit lseek. */
+static const char lseek_code[] =
+ QUOTE( %s )
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( int fd; )
+ QUOTE( Charm_Setup; )
+ QUOTE( fd = open("_charm_lseek", O_WRONLY | O_CREAT, 0666); )
+ QUOTE( if (fd == -1) { return -1; } )
+ QUOTE( %s(fd, 0, SEEK_SET); )
+ QUOTE( printf("%%d", 1); )
+ QUOTE( if (close(fd)) { return -1; } )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static chaz_bool_t
+S_probe_lseek(unbuff_combo *combo) {
+ char *output = NULL;
+ size_t output_len;
+ size_t needed = sizeof(lseek_code)
+ + strlen(combo->includes)
+ + strlen(combo->lseek_command)
+ + 20;
+ char *code_buf = (char*)malloc(needed);
+ chaz_bool_t success = false;
+
+ /* Verify compilation. */
+ sprintf(code_buf, lseek_code, combo->includes, combo->lseek_command);
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ success = true;
+ free(output);
+ }
+
+ if (!Util_remove_and_verify("_charm_lseek")) {
+ Util_die("Failed to remove '_charm_lseek'");
+ }
+
+ free(code_buf);
+ return success;
+}
+
+/* Code for checking 64-bit pread. The pread call will fail, but that's fine
+ * as long as it compiles. */
+static const char pread64_code[] =
+ QUOTE( %s )
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( int fd = 20; )
+ QUOTE( char buf[1]; )
+ QUOTE( Charm_Setup; )
+ QUOTE( printf("1"); )
+ QUOTE( %s(fd, buf, 1, 1); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static chaz_bool_t
+S_probe_pread64(unbuff_combo *combo) {
+ char *output = NULL;
+ size_t output_len;
+ size_t needed = sizeof(pread64_code)
+ + strlen(combo->includes)
+ + strlen(combo->pread64_command)
+ + 20;
+ char *code_buf = (char*)malloc(needed);
+ chaz_bool_t success = false;
+
+ /* Verify compilation. */
+ sprintf(code_buf, pread64_code, combo->includes, combo->pread64_command);
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+ if (output != NULL) {
+ success = true;
+ free(output);
+ }
+
+ free(code_buf);
+ return success;
+}
+
+static chaz_bool_t
+S_check_sparse_files(void) {
+ Stat st_a, st_b;
+
+ /* Bail out if we can't stat() a file. */
+ if (!HeadCheck_check_header("sys/stat.h")) {
+ return false;
+ }
+
+ /* Write and stat a 1 MB file and a 2 MB file, both of them sparse. */
+ S_test_sparse_file(1000000, &st_a);
+ S_test_sparse_file(2000000, &st_b);
+ if (!(st_a.valid && st_b.valid)) {
+ return false;
+ }
+ if (st_a.size != 1000001) {
+ Util_die("Expected size of 1000001 but got %ld", (long)st_a.size);
+ }
+ if (st_b.size != 2000001) {
+ Util_die("Expected size of 2000001 but got %ld", (long)st_b.size);
+ }
+
+ /* See if two files with very different lengths have the same block size. */
+ if (st_a.blocks == st_b.blocks) {
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+static void
+S_test_sparse_file(long offset, Stat *st) {
+ FILE *sparse_fh;
+
+ /* Make sure the file's not there, then open. */
+ Util_remove_and_verify("_charm_sparse");
+ if ((sparse_fh = fopen("_charm_sparse", "w+")) == NULL) {
+ Util_die("Couldn't open file '_charm_sparse'");
+ }
+
+ /* Seek fh to [offset], write a byte, close file. */
+ if ((fseek(sparse_fh, offset, SEEK_SET)) == -1) {
+ Util_die("seek failed: %s", strerror(errno));
+ }
+ if ((fprintf(sparse_fh, "X")) != 1) {
+ Util_die("fprintf failed");
+ }
+ if (fclose(sparse_fh)) {
+ Util_die("Error closing file '_charm_sparse': %s", strerror(errno));
+ }
+
+ /* Stat the file. */
+ Stat_stat("_charm_sparse", st);
+
+ remove("_charm_sparse");
+}
+
+/* Open a file, seek to a loc, print a char, and communicate success. */
+static const char create_bigfile_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( int main() { )
+ QUOTE( FILE *fh = fopen("_charm_large_file_test", "w+"); )
+ QUOTE( int check_seek; )
+ QUOTE( Charm_Setup; )
+ /* Bail unless seek succeeds. */
+ QUOTE( check_seek = %s(fh, 5000000000, SEEK_SET); )
+ QUOTE( if (check_seek == -1) )
+ QUOTE( exit(1); )
+ /* Bail unless we write successfully. */
+ QUOTE( if (fprintf(fh, "X") != 1) )
+ QUOTE( exit(1); )
+ QUOTE( if (fclose(fh)) )
+ QUOTE( exit(1); )
+ /* Communicate success to Charmonizer. */
+ QUOTE( printf("1"); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+static chaz_bool_t
+S_can_create_big_files(void) {
+ char *output;
+ size_t output_len;
+ FILE *truncating_fh;
+ size_t needed = strlen(create_bigfile_code)
+ + strlen(fseek_command)
+ + 10;
+ char *code_buf = (char*)malloc(needed);
+
+ /* Concat the source strings, compile the file, capture output. */
+ sprintf(code_buf, create_bigfile_code, fseek_command);
+ output = CC_capture_output(code_buf, strlen(code_buf), &output_len);
+
+ /* Truncate, just in case the call to remove fails. */
+ truncating_fh = fopen("_charm_large_file_test", "w");
+ if (truncating_fh != NULL) {
+ fclose(truncating_fh);
+ }
+ Util_remove_and_verify("_charm_large_file_test");
+
+ /* Return true if the test app made it to the finish line. */
+ free(code_buf);
+ return output == NULL ? false : true;
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/LargeFiles.h b/charmonizer/src/Charmonizer/Probe/LargeFiles.h
new file mode 100644
index 0000000..5425580
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/LargeFiles.h
@@ -0,0 +1,60 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/LargeFiles.h
+ */
+
+#ifndef H_CHAZ_LARGE_FILES
+#define H_CHAZ_LARGE_FILES
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+
+/* The LargeFiles module attempts to detect these symbols or alias them to
+ * synonyms:
+ *
+ * off64_t
+ * fopen64
+ * ftello64
+ * fseeko64
+ *
+ * If the attempt succeeds, this will be defined:
+ *
+ * HAS_LARGE_FILE_SUPPORT
+ *
+ * Additionally, 64-bit versions of lseek and pread may be detected/aliased:
+ *
+ * lseek64
+ * pread64
+ *
+ * Use of the off64_t symbol may require sys/types.h.
+ */
+void chaz_LargeFiles_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define LargeFiles_run chaz_LargeFiles_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_LARGE_FILES */
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Memory.c b/charmonizer/src/Charmonizer/Probe/Memory.c
new file mode 100644
index 0000000..10ef020
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Memory.c
@@ -0,0 +1,134 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Probe/Memory.h"
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/HeaderChecker.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+static const char alloca_code[] =
+ "#include <%s>\n"
+ QUOTE( int main() { )
+ QUOTE( void *foo = %s(1); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+void
+Memory_run(void) {
+ chaz_bool_t has_sys_mman_h = false;
+ chaz_bool_t has_alloca_h = false;
+ chaz_bool_t has_malloc_h = false;
+ chaz_bool_t need_stdlib_h = false;
+ chaz_bool_t has_alloca = false;
+ chaz_bool_t has_builtin_alloca = false;
+ chaz_bool_t has_underscore_alloca = false;
+ char code_buf[sizeof(alloca_code) + 100];
+
+ ConfWriter_start_module("Memory");
+
+ {
+ /* OpenBSD needs sys/types.h for sys/mman.h to work and mmap() to be
+ * available. Everybody else that has sys/mman.h should have
+ * sys/types.h as well. */
+ const char *mman_headers[] = {
+ "sys/types.h",
+ "sys/mman.h",
+ NULL
+ };
+ if (chaz_HeadCheck_check_many_headers((const char**)mman_headers)) {
+ has_sys_mman_h = true;
+ chaz_ConfWriter_append_conf("#define CHY_HAS_SYS_MMAN_H\n\n");
+ }
+ }
+
+ /* Unixen. */
+ sprintf(code_buf, alloca_code, "alloca.h", "alloca");
+ if (CC_test_compile(code_buf, strlen(code_buf))) {
+ has_alloca_h = true;
+ has_alloca = true;
+ ConfWriter_append_conf("#define CHY_HAS_ALLOCA_H\n");
+ ConfWriter_append_conf("#define chy_alloca alloca\n");
+ }
+ if (!has_alloca) {
+ sprintf(code_buf, alloca_code, "stdlib.h", "alloca");
+ if (CC_test_compile(code_buf, strlen(code_buf))) {
+ has_alloca = true;
+ need_stdlib_h = true;
+ ConfWriter_append_conf("#define CHY_ALLOCA_IN_STDLIB_H\n");
+ ConfWriter_append_conf("#define chy_alloca alloca\n");
+ }
+ }
+ if (!has_alloca) {
+ sprintf(code_buf, alloca_code, "stdio.h", /* stdio.h is filler */
+ "__builtin_alloca");
+ if (CC_test_compile(code_buf, strlen(code_buf))) {
+ has_builtin_alloca = true;
+ ConfWriter_append_conf("#define chy_alloca __builtin_alloca\n");
+ }
+ }
+
+ /* Windows. */
+ if (!(has_alloca || has_builtin_alloca)) {
+ sprintf(code_buf, alloca_code, "malloc.h", "alloca");
+ if (CC_test_compile(code_buf, strlen(code_buf))) {
+ has_malloc_h = true;
+ has_alloca = true;
+ ConfWriter_append_conf("#define CHY_HAS_MALLOC_H\n");
+ ConfWriter_append_conf("#define chy_alloca alloca\n");
+ }
+ }
+ if (!(has_alloca || has_builtin_alloca)) {
+ sprintf(code_buf, alloca_code, "malloc.h", "_alloca");
+ if (CC_test_compile(code_buf, strlen(code_buf))) {
+ has_malloc_h = true;
+ has_underscore_alloca = true;
+ ConfWriter_append_conf("#define CHY_HAS_MALLOC_H\n");
+ ConfWriter_append_conf("#define chy_alloca _alloca\n");
+ }
+ }
+
+ /* Shorten */
+ ConfWriter_start_short_names();
+ if (has_sys_mman_h) {
+ ConfWriter_shorten_macro("HAS_SYS_MMAN_H");
+ }
+ if (has_alloca_h) {
+ ConfWriter_shorten_macro("HAS_ALLOCA_H");
+ }
+ if (has_malloc_h) {
+ ConfWriter_shorten_macro("HAS_MALLOC_H");
+ if (!has_alloca && has_underscore_alloca) {
+ ConfWriter_shorten_function("alloca");
+ }
+ }
+ if (need_stdlib_h) {
+ ConfWriter_shorten_macro("ALLOCA_IN_STDLIB_H");
+ }
+ if (!has_alloca && has_builtin_alloca) {
+ ConfWriter_shorten_function("alloca");
+ }
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/Memory.h b/charmonizer/src/Charmonizer/Probe/Memory.h
new file mode 100644
index 0000000..b17487e
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/Memory.h
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/Memory.h
+ */
+
+#ifndef H_CHAZ_MEMORY
+#define H_CHAZ_MEMORY
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* The Memory module attempts to detect these symbols or alias them to
+ * synonyms:
+ *
+ * alloca
+ *
+ * These following symbols will be defined if the associated headers are
+ * available:
+ *
+ * HAS_SYS_MMAN_H <sys/mman.h>
+ * HAS_ALLOCA_H <alloca.h>
+ * HAS_MALLOC_H <malloc.h>
+ *
+ * Defined if alloca() is available via stdlib.h:
+ *
+ * ALLOCA_IN_STDLIB_H
+ */
+void chaz_Memory_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define Memory_run chaz_Memory_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_MEMORY */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/UnusedVars.c b/charmonizer/src/Charmonizer/Probe/UnusedVars.c
new file mode 100644
index 0000000..b9c7617
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/UnusedVars.c
@@ -0,0 +1,44 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/UnusedVars.h"
+#include <string.h>
+#include <stdio.h>
+
+
+void
+UnusedVars_run(void) {
+ ConfWriter_start_module("UnusedVars");
+
+ /* Write the macros (no test, these are the same everywhere). */
+ ConfWriter_append_conf("#define CHY_UNUSED_VAR(x) ((void)x)\n");
+ ConfWriter_append_conf("#define CHY_UNREACHABLE_RETURN(type) return (type)0\n");
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ ConfWriter_shorten_macro("UNUSED_VAR");
+ ConfWriter_shorten_macro("UNREACHABLE_RETURN");
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/UnusedVars.h b/charmonizer/src/Charmonizer/Probe/UnusedVars.h
new file mode 100644
index 0000000..cb1020b
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/UnusedVars.h
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/UnusedVars.h
+ */
+
+#ifndef H_CHAZ_UNUSED_VARS
+#define H_CHAZ_UNUSED_VARS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+
+/* Run the UnusedVars module.
+ *
+ * These symbols are exported:
+ *
+ * UNUSED_VAR(var)
+ * UNREACHABLE_RETURN(type)
+ *
+ */
+void chaz_UnusedVars_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define UnusedVars_run chaz_UnusedVars_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_UNUSED_VARS */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/VariadicMacros.c b/charmonizer/src/Charmonizer/Probe/VariadicMacros.c
new file mode 100644
index 0000000..8c76134
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/VariadicMacros.c
@@ -0,0 +1,95 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Core/Compiler.h"
+#include "Charmonizer/Core/ConfWriter.h"
+#include "Charmonizer/Core/Util.h"
+#include "Charmonizer/Probe/VariadicMacros.h"
+#include <string.h>
+#include <stdio.h>
+
+
+/* Code for verifying ISO-style variadic macros. */
+static const char iso_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( #define ISO_TEST(fmt, ...) \\ )
+ " printf(fmt, __VA_ARGS__) \n"
+ QUOTE( int main() { )
+ QUOTE( Charm_Setup; )
+ QUOTE( ISO_TEST("%d %d", 1, 1); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+/* Code for verifying GNU-style variadic macros. */
+static const char gnuc_code[] =
+ QUOTE( #include "_charm.h" )
+ QUOTE( #define GNU_TEST(fmt, args...) printf(fmt, ##args) )
+ QUOTE( int main() { )
+ QUOTE( Charm_Setup; )
+ QUOTE( GNU_TEST("%d %d", 1, 1); )
+ QUOTE( return 0; )
+ QUOTE( } );
+
+void
+VariadicMacros_run(void) {
+ char *output;
+ size_t output_len;
+ chaz_bool_t has_varmacros = false;
+ chaz_bool_t has_iso_varmacros = false;
+ chaz_bool_t has_gnuc_varmacros = false;
+
+ ConfWriter_start_module("VariadicMacros");
+
+ /* Test for ISO-style variadic macros. */
+ output = CC_capture_output(iso_code, strlen(iso_code), &output_len);
+ if (output != NULL) {
+ has_varmacros = true;
+ has_iso_varmacros = true;
+ ConfWriter_append_conf("#define CHY_HAS_VARIADIC_MACROS\n");
+ ConfWriter_append_conf("#define CHY_HAS_ISO_VARIADIC_MACROS\n");
+ }
+
+ /* Test for GNU-style variadic macros. */
+ output = CC_capture_output(gnuc_code, strlen(gnuc_code), &output_len);
+ if (output != NULL) {
+ has_gnuc_varmacros = true;
+ if (has_varmacros == false) {
+ has_varmacros = true;
+ ConfWriter_append_conf("#define CHY_HAS_VARIADIC_MACROS\n");
+ }
+ ConfWriter_append_conf("#define CHY_HAS_GNUC_VARIADIC_MACROS\n");
+ }
+
+ /* Shorten. */
+ ConfWriter_start_short_names();
+ if (has_varmacros) {
+ ConfWriter_shorten_macro("HAS_VARIADIC_MACROS");
+ }
+ if (has_iso_varmacros) {
+ ConfWriter_shorten_macro("HAS_ISO_VARIADIC_MACROS");
+ }
+ if (has_gnuc_varmacros) {
+ ConfWriter_shorten_macro("HAS_GNUC_VARIADIC_MACROS");
+ }
+ ConfWriter_end_short_names();
+
+ ConfWriter_end_module();
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Probe/VariadicMacros.h b/charmonizer/src/Charmonizer/Probe/VariadicMacros.h
new file mode 100644
index 0000000..d65cf38
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Probe/VariadicMacros.h
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Probe/VariadicMacros.h
+ */
+
+#ifndef H_CHAZ_VARIADIC_MACROS
+#define H_CHAZ_VARIADIC_MACROS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+
+/* Run the VariadicMacros module.
+ *
+ * If your compiler supports ISO-style variadic macros, this will be defined:
+ *
+ * HAS_ISO_VARIADIC_MACROS
+ *
+ * If your compiler supports GNU-style variadic macros, this will be defined:
+ *
+ * HAS_GNUC_VARIADIC_MACROS
+ *
+ * If you have at least one of the above, this will be defined:
+ *
+ * HAS_VARIADIC_MACROS
+ */
+void chaz_VariadicMacros_run(void);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define VariadicMacros_run chaz_VariadicMacros_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_VARIADIC_MACROS */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Test.c b/charmonizer/src/Charmonizer/Test.c
new file mode 100644
index 0000000..a708ff8
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test.c
@@ -0,0 +1,249 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <stdarg.h>
+#include <string.h>
+#include "Charmonizer/Test.h"
+
+static void
+S_TestBatch_destroy(TestBatch *batch);
+
+static void
+S_TestBatch_run_test(TestBatch *batch);
+
+#define PRINT_SUPPLIED_MESS(_pattern, _args) \
+ va_start(_args, _pattern); \
+ vprintf(_pattern, _args); \
+ va_end(_args); \
+ printf("\n");
+
+void
+Test_init(void) {
+ /* Unbuffer stdout. */
+ int check_val = setvbuf(stdout, NULL, _IONBF, 0);
+ if (check_val != 0) {
+ fprintf(stderr, "Failed when trying to unbuffer stdout\n");
+ }
+}
+
+TestBatch*
+Test_new_batch(const char *batch_name, unsigned num_tests,
+ TestBatch_test_func_t test_func) {
+ TestBatch *batch = (TestBatch*)malloc(sizeof(TestBatch));
+
+ /* Assign. */
+ batch->num_tests = num_tests;
+ batch->name = strdup(batch_name);
+ batch->test_func = test_func;
+
+ /* Initialize. */
+ batch->test_num = 0;
+ batch->num_passed = 0;
+ batch->num_failed = 0;
+ batch->num_skipped = 0;
+ batch->destroy = S_TestBatch_destroy;
+ batch->run_test = S_TestBatch_run_test;
+
+ return batch;
+}
+
+void
+Test_plan(TestBatch *batch) {
+ printf("1..%u\n", batch->num_tests);
+}
+
+static void
+S_TestBatch_destroy(TestBatch *batch) {
+ free(batch->name);
+ free(batch);
+}
+
+static void
+S_TestBatch_run_test(TestBatch *batch) {
+ /* Print start. */
+ PLAN(batch);
+
+ /* Run the batch. */
+ batch->test_func(batch);
+}
+
+void
+Test_test_true(TestBatch *batch, int value, const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Test condition and pass or fail. */
+ if (value) {
+ printf("ok %u - ", batch->test_num);
+ batch->num_passed++;
+ }
+ else {
+ printf("not ok %u - ", batch->test_num);
+ batch->num_failed++;
+ }
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_test_false(TestBatch *batch, int value, const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Test condition and pass or fail. */
+ if (value == 0) {
+ printf("ok %u - ", batch->test_num);
+ batch->num_passed++;
+ }
+ else {
+ printf("not ok %u - ", batch->test_num);
+ batch->num_failed++;
+ }
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_test_str_eq(TestBatch *batch, const char *got, const char *expected,
+ const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Test condition and pass or fail. */
+ if (strcmp(expected, got) == 0) {
+ printf("ok %u - ", batch->test_num);
+ batch->num_passed++;
+ }
+ else {
+ printf("not ok %u - Expected '%s', got '%s'\n ", batch->test_num,
+ expected, got);
+ batch->num_failed++;
+ }
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+
+void
+Test_pass(TestBatch *batch, const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Indicate pass, update pass counter. */
+ printf("ok %u - ", batch->test_num);
+ batch->num_passed++;
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_fail(TestBatch *batch, const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Indicate failure, update pass counter. */
+ printf("not ok %u - ", batch->test_num);
+ batch->num_failed++;
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_test_int_eq(TestBatch *batch, long got, long expected,
+ const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ if (expected == got) {
+ printf("ok %u - ", batch->test_num);
+ batch->num_passed++;
+ }
+ else {
+ printf("not ok %u - Expected '%ld', got '%ld'\n ", batch->test_num,
+ expected, got);
+ batch->num_failed++;
+ }
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_test_float_eq(TestBatch *batch, double got, double expected,
+ const char *pat, ...) {
+ va_list args;
+ double diff = expected / got;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Evaluate condition and pass or fail. */
+ if (diff > 0.00001) {
+ printf("ok %u - ", batch->test_num);
+ batch->num_passed++;
+ }
+ else {
+ printf("not ok %u - Expected '%f', got '%f'\n ", batch->test_num,
+ expected, got);
+ batch->num_failed++;
+ }
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_skip(TestBatch *batch, const char *pat, ...) {
+ va_list args;
+
+ /* Increment test number. */
+ batch->test_num++;
+
+ /* Indicate that test is being skipped, update pass counter. */
+ printf("ok %u # SKIP ", batch->test_num);
+ batch->num_skipped++;
+
+ PRINT_SUPPLIED_MESS(pat, args);
+}
+
+void
+Test_report_skip_remaining(TestBatch *batch, const char *pat, ...) {
+ va_list args;
+ unsigned remaining = batch->num_tests - batch->test_num;
+
+ /* Indicate that tests are being skipped, update skip counter. */
+ printf("# Skipping all %u remaining tests: ", remaining);
+ PRINT_SUPPLIED_MESS(pat, args);
+ while (batch->test_num < batch->num_tests) {
+ SKIP(batch, "");
+ }
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Test.h b/charmonizer/src/Charmonizer/Test.h
new file mode 100644
index 0000000..495fd53
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test.h
@@ -0,0 +1,167 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Test.h - test Charmonizer's output.
+ */
+
+#ifndef H_CHAZ_TEST
+#define H_CHAZ_TEST
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Charmonizer/Core/Defines.h"
+
+typedef struct chaz_TestBatch chaz_TestBatch;
+
+/* Destructor.
+ */
+typedef void
+(*chaz_TestBatch_destroy_t)(chaz_TestBatch *batch);
+
+/* This function, which is unique to each TestBatch, actually runs the test
+ * sequence.
+ */
+typedef void
+(*chaz_TestBatch_test_func_t)(chaz_TestBatch *batch);
+
+/* Print a header, execute the test sequence, print a report.
+ */
+typedef void
+(*chaz_TestBatch_run_test_t)(chaz_TestBatch *batch);
+
+struct chaz_TestBatch {
+ char *name;
+ unsigned test_num;
+ unsigned num_tests;
+ unsigned num_passed;
+ unsigned num_failed;
+ unsigned num_skipped;
+ chaz_TestBatch_destroy_t destroy;
+ chaz_TestBatch_test_func_t test_func;
+ chaz_TestBatch_run_test_t run_test;
+};
+
+/* Unbuffer stdout. Perform any other setup needed.
+ */
+void
+chaz_Test_init(void);
+
+/* Constructor for TestBatch.
+ */
+chaz_TestBatch*
+chaz_Test_new_batch(const char *batch_name, unsigned num_tests,
+ chaz_TestBatch_test_func_t test_func);
+
+/* Note: maybe add line numbers later.
+ */
+#define CHAZ_TEST_PLAN chaz_Test_plan
+#define CHAZ_TEST_TEST_TRUE chaz_Test_test_true
+#define CHAZ_TEST_TEST_FALSE chaz_Test_test_false
+#define CHAZ_TEST_TEST_STR_EQ chaz_Test_test_str_eq
+#define CHAZ_TEST_PASS chaz_Test_pass
+#define CHAZ_TEST_FAIL chaz_Test_fail
+#define CHAZ_TEST_TEST_INT_EQ chaz_Test_test_int_eq
+#define CHAZ_TEST_TEST_FLOAT_EQ chaz_Test_test_float_eq
+
+/* Print a message indicating that a test was skipped and update batch.
+ */
+#define CHAZ_TEST_SKIP(batch, message) \
+ chaz_Test_skip(batch, message)
+
+/* Print a message indicating that all remaining tests will be skipped and
+ * return.
+ */
+#define CHAZ_TEST_SKIP_REMAINING(batch, message) \
+ do { \
+ chaz_Test_report_skip_remaining(batch, message); \
+ return; \
+ } while (0)
+
+void
+chaz_Test_plan(chaz_TestBatch *batch);
+
+void
+chaz_Test_test_true(chaz_TestBatch *batch, int expression,
+ const char *pat, ...);
+
+void
+chaz_Test_test_false(chaz_TestBatch *batch, int expression,
+ const char *pat, ...);
+
+void
+chaz_Test_test_str_eq(chaz_TestBatch *batch, const char *got,
+ const char *expected, const char *pat, ...);
+
+void
+chaz_Test_pass(chaz_TestBatch *batch, const char *pat, ...);
+
+void
+chaz_Test_fail(chaz_TestBatch *batch, const char *pat, ...);
+
+void
+chaz_Test_test_int_eq(chaz_TestBatch *batch, long got, long expected,
+ const char *pat, ...);
+
+void
+chaz_Test_test_float_eq(chaz_TestBatch *batch, double got,
+ double expected, const char *pat, ...);
+
+void
+chaz_Test_skip(chaz_TestBatch *batch, const char *pat, ...);
+
+void
+chaz_Test_report_skip_remaining(chaz_TestBatch* batch,
+ const char *pat, ...);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define TestBatch_destroy_t chaz_TestBatch_destroy_t
+ #define TestBatch_test_func_t chaz_TestBatch_test_func_t
+ #define TestBatch_run_test_t chaz_TestBatch_run_test_t
+ #define TestBatch chaz_TestBatch
+ #define Test_init chaz_Test_init
+ #define Test_new_batch chaz_Test_new_batch
+ #define Test_plan chaz_Test_plan
+ #define PLAN CHAZ_TEST_PLAN
+ #define Test_test_true chaz_Test_test_true
+ #define TEST_TRUE CHAZ_TEST_TEST_TRUE
+ #define Test_test_false chaz_Test_test_false
+ #define TEST_FALSE CHAZ_TEST_TEST_FALSE
+ #define Test_test_str_eq chaz_Test_test_str_eq
+ #define TEST_STR_EQ CHAZ_TEST_TEST_STR_EQ
+ #define Test_pass chaz_Test_pass
+ #define PASS CHAZ_TEST_PASS
+ #define Test_fail chaz_Test_fail
+ #define FAIL CHAZ_TEST_FAIL
+ #define Test_test_int_eq chaz_Test_test_int_eq
+ #define TEST_INT_EQ CHAZ_TEST_TEST_INT_EQ
+ #define Test_test_float_eq chaz_Test_test_float_eq
+ #define TEST_FLOAT_EQ CHAZ_TEST_TEST_FLOAT_EQ
+ #define Test_skip chaz_Test_skip
+ #define SKIP CHAZ_TEST_SKIP
+ #define Test_report_skip_remaining chaz_Test_report_skip_remaining
+ #define SKIP_REMAINING CHAZ_TEST_SKIP_REMAINING
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_TEST */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Test/AllTests.c b/charmonizer/src/Charmonizer/Test/AllTests.c
new file mode 100644
index 0000000..9e944c9
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/AllTests.c
@@ -0,0 +1,74 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "Charmonizer/Test/AllTests.h"
+
+static TestBatch **batches = NULL;
+
+void
+AllTests_init() {
+ Test_init();
+
+ /* Create a null-terminated array of test batches to iterate over. */
+ batches = (TestBatch**)malloc(8 * sizeof(TestBatch*));
+ batches[0] = TestDirManip_prepare();
+ batches[1] = TestFuncMacro_prepare();
+ batches[2] = TestHeaders_prepare();
+ batches[3] = TestIntegers_prepare();
+ batches[4] = TestLargeFiles_prepare();
+ batches[5] = TestUnusedVars_prepare();
+ batches[6] = TestVariadicMacros_prepare();
+ batches[7] = NULL;
+}
+
+void
+AllTests_run() {
+ int total_tests = 0;
+ int total_passed = 0;
+ int total_failed = 0;
+ int total_skipped = 0;
+ int i;
+
+ /* Sanity check. */
+ if (batches == NULL) {
+ fprintf(stderr, "Must call AllTests_init() first.");
+ exit(1);
+ }
+
+ /* Loop through test functions, accumulating results. */
+ for (i = 0; batches[i] != NULL; i++) {
+ TestBatch *batch = batches[i];
+ batch->run_test(batch);
+ total_tests += batch->num_tests;
+ total_passed += batch->num_passed;
+ total_failed += batch->num_failed;
+ total_skipped += batch->num_skipped;
+ batch->destroy(batch);
+ }
+
+ /* Print totals. */
+ printf("=============================\n");
+ printf("TOTAL TESTS: %d\nTOTAL PASSED: %d\nTOTAL FAILED: %d\n"
+ "TOTAL SKIPPED: %d\n",
+ total_tests, total_passed, total_failed, total_skipped);
+}
+
+
+
diff --git a/charmonizer/src/Charmonizer/Test/AllTests.h b/charmonizer/src/Charmonizer/Test/AllTests.h
new file mode 100644
index 0000000..23b85a7
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/AllTests.h
@@ -0,0 +1,118 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Charmonizer/Test.h - test Charmonizer's output.
+ */
+
+#ifndef H_CHAZ_ALL_TESTS
+#define H_CHAZ_ALL_TESTS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "Charmonizer/Test.h"
+
+/* Initialize the AllTests module.
+ */
+void
+chaz_AllTests_init();
+
+/* Run all tests.
+ */
+void
+chaz_AllTests_run();
+
+/* These tests all require the file charmony.h.
+ *
+ * Since Charmonizer conditionally defines many symbols, it can be difficult
+ * to tell whether a symbol is missing because it should not have been
+ * generated, or whether it is missing because an error occurred. These test
+ * functions make the assumption that any missing symbols have a good excuse
+ * for their absence, and test only defined symbols. This may result in
+ * undetected failure some of the time. However, missing symbols required by
+ * your application will trigger compile-time errors, so the theoretical
+ * problem of silent failure is less severe than it appears, affecting only
+ * fallbacks.
+ */
+
+chaz_TestBatch*
+chaz_TestDirManip_prepare();
+
+chaz_TestBatch*
+chaz_TestFuncMacro_prepare();
+
+chaz_TestBatch*
+chaz_TestHeaders_prepare();
+
+chaz_TestBatch*
+chaz_TestIntegers_prepare();
+
+chaz_TestBatch*
+chaz_TestLargeFiles_prepare();
+
+chaz_TestBatch*
+chaz_TestUnusedVars_prepare();
+
+chaz_TestBatch*
+chaz_TestVariadicMacros_prepare();
+
+void
+chaz_TestDirManip_run(chaz_TestBatch *batch);
+
+void
+chaz_TestFuncMacro_run(chaz_TestBatch *batch);
+
+void
+chaz_TestHeaders_run(chaz_TestBatch *batch);
+
+void
+chaz_TestIntegers_run(chaz_TestBatch *batch);
+
+void
+chaz_TestLargeFiles_run(chaz_TestBatch *batch);
+
+void
+chaz_TestUnusedVars_run(chaz_TestBatch *batch);
+
+void
+chaz_TestVariadicMacros_run(chaz_TestBatch *batch);
+
+#ifdef CHAZ_USE_SHORT_NAMES
+ #define TestDirManip_prepare chaz_TestDirManip_prepare
+ #define TestDirManip_run chaz_TestDirManip_run
+ #define TestFuncMacro_prepare chaz_TestFuncMacro_prepare
+ #define TestHeaders_prepare chaz_TestHeaders_prepare
+ #define TestIntegers_prepare chaz_TestIntegers_prepare
+ #define TestLargeFiles_prepare chaz_TestLargeFiles_prepare
+ #define TestUnusedVars_prepare chaz_TestUnusedVars_prepare
+ #define TestVariadicMacros_prepare chaz_TestVariadicMacros_prepare
+ #define TestFuncMacro_run chaz_TestFuncMacro_run
+ #define TestHeaders_run chaz_TestHeaders_run
+ #define TestIntegers_run chaz_TestIntegers_run
+ #define TestLargeFiles_run chaz_TestLargeFiles_run
+ #define TestUnusedVars_run chaz_TestUnusedVars_run
+ #define TestVariadicMacros_run chaz_TestVariadicMacros_run
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CHAZ_TEST */
+
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestDirManip.c b/charmonizer/src/Charmonizer/Test/TestDirManip.c
new file mode 100644
index 0000000..9d73b8c
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestDirManip.c
@@ -0,0 +1,73 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+#include <stdio.h>
+#include <string.h>
+#include "charmony.h"
+#ifdef HAS_DIRENT_H
+ #include <dirent.h>
+#endif
+#ifdef HAS_SYS_STAT_H
+ #include <sys/stat.h>
+#endif
+#ifdef HAS_SYS_TYPES_H
+ #include <sys/stat.h>
+#endif
+#ifdef HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+#ifdef HAS_DIRECT_H
+ #include <direct.h>
+#endif
+
+TestBatch*
+TestDirManip_prepare() {
+ return Test_new_batch("Integers", 6, TestDirManip_run);
+}
+
+void
+TestDirManip_run(TestBatch *batch) {
+ TEST_INT_EQ(batch, 0, makedir("_chaz_test_dir", 0777), "makedir");
+ TEST_INT_EQ(batch, 0, makedir("_chaz_test_dir" DIR_SEP "deep", 0777),
+ "makedir with DIR_SEP");
+ TEST_INT_EQ(batch, 0, rmdir("_chaz_test_dir" DIR_SEP "deep"),
+ "rmdir with DIR_SEP");
+ TEST_INT_EQ(batch, 0, rmdir("_chaz_test_dir"), "rmdir");
+#ifdef CHY_HAS_DIRENT_D_NAMLEN
+ {
+ struct dirent entry;
+ entry.d_namlen = 5;
+ TEST_INT_EQ(batch, 5, entry.d_namlen, "d_namlen");
+ }
+#else
+ SKIP(batch, "no d_namlen member on this platform");
+#endif
+#ifdef CHY_HAS_DIRENT_D_TYPE
+ {
+ struct dirent entry;
+ entry.d_type = 5;
+ TEST_INT_EQ(batch, 5, entry.d_type, "d_type");
+ }
+#else
+ SKIP(batch, "no d_type member on this platform");
+#endif
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestFuncMacro.c b/charmonizer/src/Charmonizer/Test/TestFuncMacro.c
new file mode 100644
index 0000000..1e7d06c
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestFuncMacro.c
@@ -0,0 +1,66 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "charmony.h"
+#include <string.h>
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+
+TestBatch*
+TestFuncMacro_prepare() {
+ return Test_new_batch("FuncMacro", 4, TestFuncMacro_run);
+}
+
+#ifdef INLINE
+static INLINE const char* S_inline_function() {
+ return "inline works";
+}
+#endif
+
+void
+TestFuncMacro_run(TestBatch *batch) {
+
+#ifdef HAS_FUNC_MACRO
+ TEST_STR_EQ(batch, FUNC_MACRO, "chaz_TestFuncMacro_run",
+ "FUNC_MACRO");
+#else
+ SKIP(batch, "no FUNC_MACRO");
+#endif
+
+#ifdef HAS_ISO_FUNC_MACRO
+ TEST_STR_EQ(batch, __func__, "chaz_TestFuncMacro_run",
+ "HAS_ISO_FUNC_MACRO");
+#else
+ SKIP(batch, "no ISO_FUNC_MACRO");
+#endif
+
+#ifdef HAS_GNUC_FUNC_MACRO
+ TEST_STR_EQ(batch, __FUNCTION__, "chaz_TestFuncMacro_run",
+ "HAS_GNUC_FUNC_MACRO");
+#else
+ SKIP(batch, "no GNUC_FUNC_MACRO");
+#endif
+
+#ifdef INLINE
+ PASS(batch, S_inline_function());
+#else
+ SKIP(batch, "no INLINE functions");
+#endif
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestHeaders.c b/charmonizer/src/Charmonizer/Test/TestHeaders.c
new file mode 100644
index 0000000..3f9bf75
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestHeaders.c
@@ -0,0 +1,165 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "charmony.h"
+#include <string.h>
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+
+#ifdef HAS_ASSERT_H
+ #include <assert.h>
+#endif
+#ifdef HAS_CTYPE_H
+ #include <ctype.h>
+#endif
+#ifdef HAS_ERRNO_H
+ #include <errno.h>
+#endif
+#ifdef HAS_FLOAT_H
+ #include <float.h>
+#endif
+#ifdef HAS_LIMITS_H
+ #include <limits.h>
+#endif
+#ifdef HAS_LOCALE_H
+ #include <locale.h>
+#endif
+#ifdef HAS_MATH_H
+ #include <math.h>
+#endif
+#ifdef HAS_SETJMP_H
+ #include <setjmp.h>
+#endif
+#ifdef HAS_SIGNAL_H
+ #include <signal.h>
+#endif
+#ifdef HAS_STDARG_H
+ #include <stdarg.h>
+#endif
+#ifdef HAS_STDDEF_H
+ #include <stddef.h>
+#endif
+#ifdef HAS_STDIO_H
+ #include <stdio.h>
+#endif
+#ifdef HAS_STDLIB_H
+ #include <stdlib.h>
+#endif
+#ifdef HAS_STRING_H
+ #include <string.h>
+#endif
+#ifdef HAS_TIME_H
+ #include <time.h>
+#endif
+
+#ifdef HAS_CPIO_H
+ #include <cpio.h>
+#endif
+#ifdef HAS_DIRENT_H
+ #include <dirent.h>
+#endif
+#ifdef HAS_FCNTL_H
+ #include <fcntl.h>
+#endif
+#ifdef HAS_GRP_H
+ #include <grp.h>
+#endif
+#ifdef HAS_PWD_H
+ #include <pwd.h>
+#endif
+#ifdef HAS_SYS_STAT_H
+ #include <sys/stat.h>
+#endif
+#ifdef HAS_SYS_TIMES_H
+ #include <sys/times.h>
+#endif
+#ifdef HAS_SYS_TYPES_H
+ #include <sys/types.h>
+#endif
+#ifdef HAS_SYS_UTSNAME_H
+ #include <sys/utsname.h>
+#endif
+#ifdef HAS_WAIT_H
+ #include <wait.h>
+#endif
+#ifdef HAS_TAR_H
+ #include <tar.h>
+#endif
+#ifdef HAS_TERMIOS_H
+ #include <termios.h>
+#endif
+#ifdef HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+#ifdef HAS_UTIME_H
+ #include <utime.h>
+#endif
+
+#if defined(HAS_C89) || defined(HAS_C90)
+ #include <assert.h>
+ #include <ctype.h>
+ #include <errno.h>
+ #include <float.h>
+ #include <limits.h>
+ #include <locale.h>
+ #include <math.h>
+ #include <setjmp.h>
+ #include <signal.h>
+ #include <stdarg.h>
+ #include <stddef.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <time.h>
+#endif
+
+#ifdef HAS_POSIX
+ #include <cpio.h>
+ #include <dirent.h>
+ #include <fcntl.h>
+ #include <grp.h>
+ #include <pwd.h>
+ #include <sys/stat.h>
+ #include <sys/times.h>
+ #include <sys/types.h>
+ #include <sys/utsname.h>
+ #include <sys/wait.h>
+ #include <tar.h>
+ #include <termios.h>
+ #include <unistd.h>
+ #include <utime.h>
+#endif
+
+TestBatch*
+TestHeaders_prepare() {
+ return Test_new_batch("Headers", 2, TestHeaders_run);
+}
+
+void
+TestHeaders_run(TestBatch *batch) {
+ PASS(batch, "Compiled successfully with all detected headers");
+
+ /* Don't bother checking all -- just use stdio as an example. */
+#ifdef HAS_STDIO_H
+ PASS(batch, "stdio.h should have been detected");
+#else
+ FAIL(batch, "stdio.h should have been detected");
+#endif
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestIntegers.c b/charmonizer/src/Charmonizer/Test/TestIntegers.c
new file mode 100644
index 0000000..c55915f
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestIntegers.c
@@ -0,0 +1,129 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+#include "charmony.h"
+#include <stdio.h>
+#include <string.h>
+#ifdef HAS_INTTYPES_H
+ #include <inttypes.h>
+#endif
+
+TestBatch*
+TestIntegers_prepare() {
+ return Test_new_batch("Integers", 37, TestIntegers_run);
+}
+
+void
+TestIntegers_run(TestBatch *batch) {
+ {
+ long one = 1;
+ long big_endian = !(*((char *)(&one)));
+#ifdef BIG_END
+ TEST_TRUE(batch, big_endian, "BIG_END");
+#else
+ #if defined(LITTLE_END)
+ TEST_TRUE(batch, !big_endian, "LITTLE_END");
+ #else
+ FAIL(batch, "Either BIG_END or LITTLE_END should be defined");
+ #endif
+#endif
+ }
+
+ TEST_INT_EQ(batch, SIZEOF_CHAR, sizeof(char), "SIZEOF_CHAR");
+ TEST_INT_EQ(batch, SIZEOF_SHORT, sizeof(short), "SIZEOF_SHORT");
+ TEST_INT_EQ(batch, SIZEOF_INT, sizeof(int), "SIZEOF_INT");
+ TEST_INT_EQ(batch, SIZEOF_LONG, sizeof(long), "SIZEOF_LONG");
+ TEST_INT_EQ(batch, SIZEOF_PTR, sizeof(void*), "SIZEOF_PTR");
+
+#ifdef HAS_LONG_LONG
+ TEST_INT_EQ(batch, SIZEOF_LONG_LONG, sizeof(long long),
+ "HAS_LONG_LONG and SIZEOF_LONG_LONG");
+#endif
+
+#ifdef HAS_INTTYPES_H
+ TEST_INT_EQ(batch, sizeof(int8_t), 1, "HAS_INTTYPES_H");
+#else
+ SKIP(batch, "No inttypes.h");
+#endif
+
+ {
+ bool_t the_truth = true;
+ TEST_TRUE(batch, the_truth, "bool_t true");
+ TEST_FALSE(batch, false, "false is false");
+ }
+#ifdef HAS_I8_T
+ {
+ int8_t foo = -100;
+ uint8_t bar = 200;
+ TEST_INT_EQ(batch, foo, -100, "int8_t is signed");
+ TEST_INT_EQ(batch, bar, 200, "uint8_t is unsigned");
+ TEST_INT_EQ(batch, sizeof(int8_t), 1, "i8_t is 1 byte");
+ TEST_INT_EQ(batch, sizeof(uint8_t), 1, "u8_t is 1 byte");
+ TEST_INT_EQ(batch, I8_MAX, 127, "I8_MAX");
+ TEST_INT_EQ(batch, I8_MIN, -128, "I8_MIN");
+ TEST_INT_EQ(batch, U8_MAX, 255, "U8_MAX");
+ }
+#endif
+#ifdef HAS_I16_T
+ {
+ int16_t foo = -100;
+ uint16_t bar = 30000;
+ TEST_INT_EQ(batch, foo, -100, "int16_t is signed");
+ TEST_INT_EQ(batch, bar, 30000, "uint16_t is unsigned");
+ TEST_INT_EQ(batch, sizeof(int16_t), 2, "int16_t is 2 bytes");
+ TEST_INT_EQ(batch, sizeof(uint16_t), 2, "uint16_t is 2 bytes");
+ TEST_INT_EQ(batch, I16_MAX, 32767, "I16_MAX");
+ TEST_INT_EQ(batch, I16_MIN, -32768, "I16_MIN");
+ TEST_INT_EQ(batch, U16_MAX, 65535, "U16_MAX");
+ }
+#endif
+#ifdef HAS_I32_T
+ {
+ int32_t foo = -100;
+ uint32_t bar = 4000000000UL;
+ TEST_TRUE(batch, (foo == -100), "int32_t is signed");
+ TEST_TRUE(batch, (bar == 4000000000UL), "uint32_t is unsigned");
+ TEST_TRUE(batch, (sizeof(int32_t) == 4), "int32_t is 4 bytes");
+ TEST_TRUE(batch, (sizeof(uint32_t) == 4), "uint32_t is 4 bytes");
+ TEST_TRUE(batch, (I32_MAX == I32_C(2147483647)), "I32_MAX");
+ /* The (-2147483647 - 1) avoids a compiler warning. */
+ TEST_TRUE(batch, (I32_MIN == I32_C(-2147483647 - 1)), "I32_MIN");
+ TEST_TRUE(batch, (U32_MAX == U32_C(4294967295)), "U32_MAX");
+ }
+#endif
+#ifdef HAS_I64_T
+ {
+ char buf[100];
+ int64_t foo = -100;
+ uint64_t bar = U64_C(18000000000000000000);
+ TEST_TRUE(batch, (foo == -100), "int64_t is signed");
+ TEST_TRUE(batch, (bar == U64_C(18000000000000000000)),
+ "uint64_t is unsigned");
+ TEST_TRUE(batch, (sizeof(int64_t) == 8), "int64_t is 8 bytes");
+ TEST_TRUE(batch, (sizeof(uint64_t) == 8), "uint64_t is 8 bytes");
+ sprintf(buf, "%"I64P, foo);
+ TEST_STR_EQ(batch, buf, "-100", "I64P");
+ sprintf(buf, "%"U64P, bar);
+ TEST_STR_EQ(batch, buf, "18000000000000000000", "U64P");
+ }
+#endif
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestLargeFiles.c b/charmonizer/src/Charmonizer/Test/TestLargeFiles.c
new file mode 100644
index 0000000..ead6186
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestLargeFiles.c
@@ -0,0 +1,104 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "charmony.h"
+
+#ifdef HAS_SYS_TYPES_H
+ #include <sys/types.h>
+#endif
+
+#include <stdio.h>
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+
+TestBatch*
+TestLargeFiles_prepare() {
+ return Test_new_batch("LargeFiles", 10, TestLargeFiles_run);
+}
+
+void
+TestLargeFiles_run(TestBatch *batch) {
+ FILE *fh;
+ off64_t offset;
+ int check_val;
+ char check_char;
+
+ /* A little over 4 GB, and a little over 2 GB. */
+ off64_t gb4_plus = ((off64_t)0x7FFFFFFF << 1) + 100;
+ off64_t gb2_plus = (off64_t)0x7FFFFFFF + 200;
+
+ /* Gb4_plus modulo 4 GB (wrap is intentional). */
+ i32_t wrap_gb4 = (i32_t)gb4_plus;
+
+ TEST_INT_EQ(batch, sizeof(off64_t), 8, "off64_t type has 8 bytes");
+
+#ifndef HAS_LARGE_FILE_SUPPORT
+ SKIP_REMAINING(batch, "No large file support");
+#endif
+#ifndef CHAZ_HAS_SPARSE_FILES
+ SKIP_REMAINING(batch,
+ "Can't verify large file support without sparse files");
+#endif
+#ifndef CHAZ_CAN_CREATE_BIG_FILES
+ SKIP_REMAINING(batch, "Unsafe to create 5GB sparse files on this system");
+#endif
+
+ fh = fopen64("_charm_large_file_test", "w+");
+ if (fh == NULL) {
+ SKIP_REMAINING(batch, "Failed to open file");
+ }
+
+ check_val = fseeko64(fh, gb4_plus, SEEK_SET);
+ TEST_INT_EQ(batch, check_val, 0, "fseeko64 above 4 GB");
+
+ offset = ftello64(fh);
+ TEST_TRUE(batch, (offset == gb4_plus), "ftello64 above 4 GB");
+
+ check_val = fprintf(fh, "X");
+ TEST_INT_EQ(batch, check_val, 1, "print above 4 GB");
+
+ check_val = fseeko64(fh, gb2_plus, SEEK_SET);
+ TEST_INT_EQ(batch, check_val, 0, "fseeko64 above 2 GB");
+
+ offset = ftello64(fh);
+ TEST_TRUE(batch, (offset == gb2_plus), "ftello64 above 2 GB");
+
+ check_val = fseeko64(fh, -1, SEEK_END);
+ TEST_INT_EQ(batch, check_val, 0, "seek to near end");
+
+ check_char = fgetc(fh);
+ TEST_INT_EQ(batch, check_char, 'X', "read value after multiple seeks");
+
+ fseeko64(fh, wrap_gb4, SEEK_SET);
+ check_char = fgetc(fh);
+ TEST_INT_EQ(batch, check_char, '\0', "No wraparound");
+
+ check_val = fclose(fh);
+ TEST_INT_EQ(batch, check_val, 0, "fclose succeeds after all that");
+
+ /* Truncate, just in case the call to remove fails. */
+ fh = fopen64("_charm_large_file_test", "w+");
+ if (fh != NULL) {
+ fclose(fh);
+ }
+ remove("_charm_large_file_test");
+}
+
+
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestUnusedVars.c b/charmonizer/src/Charmonizer/Test/TestUnusedVars.c
new file mode 100644
index 0000000..219af4a
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestUnusedVars.c
@@ -0,0 +1,42 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "charmony.h"
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+
+TestBatch*
+TestUnusedVars_prepare() {
+ return Test_new_batch("UnusedVars", 2, TestUnusedVars_run);
+}
+void
+TestUnusedVars_run(TestBatch *batch) {
+#ifdef UNUSED_VAR
+ PASS(batch, "UNUSED_VAR macro is defined");
+#else
+ FAIL(batch, "UNUSED_VAR macro is defined");
+#endif
+
+#ifdef UNREACHABLE_RETURN
+ PASS(batch, "UNREACHABLE_RETURN macro is defined");
+#else
+ FAIL(batch, "UNREACHABLE_RETURN macro is defined");
+#endif
+}
+
+
diff --git a/charmonizer/src/Charmonizer/Test/TestVariadicMacros.c b/charmonizer/src/Charmonizer/Test/TestVariadicMacros.c
new file mode 100644
index 0000000..9593aad
--- /dev/null
+++ b/charmonizer/src/Charmonizer/Test/TestVariadicMacros.c
@@ -0,0 +1,69 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define CHAZ_USE_SHORT_NAMES
+
+#include "charmony.h"
+#include <string.h>
+#include <stdio.h>
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+
+TestBatch*
+TestVariadicMacros_prepare() {
+ return Test_new_batch("VariadicMacros", 4, TestVariadicMacros_run);
+}
+
+void
+TestVariadicMacros_run(TestBatch *batch) {
+ char buf[10];
+ chaz_bool_t really_has_var_macs = false;
+
+#if defined(HAS_ISO_VARIADIC_MACROS) || defined(HAS_GNUC_VARIADIC_MACROS)
+ #ifdef HAS_VARIADIC_MACROS
+ PASS(batch, "#defines agree");
+ #else
+ FAIL(batch, 0, "#defines agree");
+ #endif
+#else
+ SKIP_REMAINING(batch, "No variadic macro support");
+#endif
+
+
+#ifdef HAS_ISO_VARIADIC_MACROS
+ #define ISO_TEST(buffer, fmt, ...) \
+ sprintf(buffer, fmt, __VA_ARGS__)
+ really_has_var_macs = true;
+ ISO_TEST(buf, "%s", "iso");
+ TEST_STR_EQ(batch, buf, "iso", "ISO variadic macros work");
+#else
+ SKIP(batch, "No ISO variadic macros");
+#endif
+
+#ifdef HAS_GNUC_VARIADIC_MACROS
+ #define GNU_TEST(buffer, fmt, args...) \
+ sprintf(buffer, fmt, ##args )
+ really_has_var_macs = true;
+ GNU_TEST(buf, "%s", "gnu");
+ TEST_STR_EQ(batch, buf, "gnu", "GNUC variadic macros work");
+#else
+ SKIP(batch, "No GNUC variadic macros");
+#endif
+
+ TEST_TRUE(batch, really_has_var_macs, "either ISO or GNUC");
+}
+
+
diff --git a/clownfish/Build.PL b/clownfish/Build.PL
new file mode 100644
index 0000000..bcbca44
--- /dev/null
+++ b/clownfish/Build.PL
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use 5.008003;
+use strict;
+use warnings;
+use lib 'buildlib';
+use Clownfish::Build;
+
+my $builder = Clownfish::Build->new(
+ module_name => 'Clownfish',
+ license => 'apache',
+ dist_author =>
+ 'The Apache Lucy Project <lucy-dev at incubator dot apache dot org>',
+ dist_version_from => 'lib/Clownfish.pm',
+ requires => { 'Parse::RecDescent' => 0, },
+ build_requires => {
+ 'ExtUtils::CBuilder' => 0.18,
+ 'ExtUtils::ParseXS' => 2.16,
+ 'Devel::PPPort' => 3.13,
+ },
+ include_dirs => ['include'],
+ c_source => 'src',
+ add_to_cleanup => [
+ qw(
+ MANIFEST.bak
+ perltidy.ERR
+ *.pdb
+ *.manifest
+ ),
+ ],
+);
+
+$builder->create_build_script();
+
diff --git a/clownfish/MANIFEST b/clownfish/MANIFEST
new file mode 100644
index 0000000..c52cf6a
--- /dev/null
+++ b/clownfish/MANIFEST
@@ -0,0 +1,92 @@
+Build.PL
+include/CFC.h
+lib/Clownfish.pm
+lib/Clownfish.xs
+lib/Clownfish/Base.pm
+lib/Clownfish/Binding/Core.pm
+lib/Clownfish/Binding/Core/Aliases.pm
+lib/Clownfish/Binding/Core/Class.pm
+lib/Clownfish/Binding/Core/File.pm
+lib/Clownfish/Binding/Core/Function.pm
+lib/Clownfish/Binding/Core/Method.pm
+lib/Clownfish/Binding/Perl.pm
+lib/Clownfish/Binding/Perl/Class.pm
+lib/Clownfish/Binding/Perl/Constructor.pm
+lib/Clownfish/Binding/Perl/Method.pm
+lib/Clownfish/Binding/Perl/Subroutine.pm
+lib/Clownfish/Binding/Perl/TypeMap.pm
+lib/Clownfish/CBlock.pm
+lib/Clownfish/Class.pm
+lib/Clownfish/DocuComment.pm
+lib/Clownfish/Dumpable.pm
+lib/Clownfish/File.pm
+lib/Clownfish/Function.pm
+lib/Clownfish/Hierarchy.pm
+lib/Clownfish/Method.pm
+lib/Clownfish/ParamList.pm
+lib/Clownfish/Parcel.pm
+lib/Clownfish/Parser.pm
+lib/Clownfish/Symbol.pm
+lib/Clownfish/Type.pm
+lib/Clownfish/Util.pm
+lib/Clownfish/Variable.pm
+MANIFEST This list of files
+src/CFCBase.c
+src/CFCBase.h
+src/CFCCBlock.c
+src/CFCCBlock.h
+src/CFCClass.c
+src/CFCClass.h
+src/CFCDocuComment.c
+src/CFCDocuComment.h
+src/CFCDumpable.c
+src/CFCDumpable.h
+src/CFCFile.c
+src/CFCFile.h
+src/CFCFunction.c
+src/CFCFunction.h
+src/CFCHierarchy.c
+src/CFCHierarchy.h
+src/CFCMethod.c
+src/CFCMethod.h
+src/CFCParamList.c
+src/CFCParamList.h
+src/CFCParcel.c
+src/CFCParcel.h
+src/CFCSymbol.c
+src/CFCSymbol.h
+src/CFCType.c
+src/CFCType.h
+src/CFCUtil.c
+src/CFCUtil.h
+src/CFCVariable.c
+src/CFCVariable.h
+t/000-load.t
+t/001-util.t
+t/050-docucomment.t
+t/051-symbol.t
+t/100-type.t
+t/101-primitive_type.t
+t/102-integer_type.t
+t/103-float_type.t
+t/104-void_type.t
+t/105-object_type.t
+t/106-va_list_type.t
+t/107-arbitrary_type.t
+t/108-composite_type.t
+t/200-function.t
+t/201-method.t
+t/202-overridden_method.t
+t/203-final_method.t
+t/300-variable.t
+t/301-param_list.t
+t/400-class.t
+t/401-c_block.t
+t/402-parcel.t
+t/403-file.t
+t/500-hierarchy.t
+t/600-parser.t
+t/cfsource/Animal.cfh
+t/cfsource/Animal/Dog.cfh
+t/cfsource/Animal/Util.cfh
+typemap
diff --git a/clownfish/buildlib/Clownfish/Build.pm b/clownfish/buildlib/Clownfish/Build.pm
new file mode 100644
index 0000000..bdc0e06
--- /dev/null
+++ b/clownfish/buildlib/Clownfish/Build.pm
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Build;
+use base qw( Module::Build );
+
+use File::Spec::Functions qw( catfile );
+
+my $PPPORT_H_PATH = catfile(qw( include ppport.h ));
+
+sub extra_ccflags {
+ my $self = shift;
+ my $extra_ccflags = defined $ENV{CFLAGS} ? "$ENV{CFLAGS} " : "";
+ my $gcc_version
+ = $ENV{REAL_GCC_VERSION}
+ || $self->config('gccversion')
+ || undef;
+ if ( defined $gcc_version ) {
+ $gcc_version =~ /^(\d+(\.\d+))/
+ or die "Invalid GCC version: $gcc_version";
+ $gcc_version = $1;
+ }
+
+ if ( defined $ENV{LUCY_DEBUG} ) {
+ if ( defined $gcc_version ) {
+ $extra_ccflags .= "-DLUCY_DEBUG ";
+ $extra_ccflags
+ .= "-DPERL_GCC_PEDANTIC -std=gnu99 -pedantic -Wall ";
+ $extra_ccflags .= "-Wextra " if $gcc_version >= 3.4; # correct
+ $extra_ccflags .= "-Wno-variadic-macros "
+ if $gcc_version > 3.4; # at least not on gcc 3.4
+ }
+ }
+
+ if ( $ENV{LUCY_VALGRIND} and defined $gcc_version ) {
+ $extra_ccflags .= "-fno-inline-functions ";
+ }
+
+ # Compile as C++ under MSVC. Turn off stupid warnings, too.
+ if ( $self->config('cc') =~ /^cl\b/ ) {
+ $extra_ccflags .= '/TP -D_CRT_SECURE_NO_WARNINGS ';
+ }
+
+ if ( defined $gcc_version ) {
+ # Tell GCC explicitly to run with maximum options.
+ if ( $extra_ccflags !~ m/-std=/ ) {
+ $extra_ccflags .= "-std=gnu99 ";
+ }
+ if ( $extra_ccflags !~ m/-D_GNU_SOURCE/ ) {
+ $extra_ccflags .= "-D_GNU_SOURCE ";
+ }
+ }
+
+ return $extra_ccflags;
+}
+
+sub new {
+ my ( $class, %args ) = @_;
+ return $class->SUPER::new(
+ %args,
+ recursive_test_files => 1,
+ extra_compiler_flags => __PACKAGE__->extra_ccflags,
+ );
+}
+
+# Write ppport.h, which supplies some XS routines not found in older Perls and
+# allows us to use more up-to-date XS API while still supporting Perls back to
+# 5.8.3.
+#
+# The Devel::PPPort docs recommend that we distribute ppport.h rather than
+# require Devel::PPPort itself, but ppport.h isn't compatible with the Apache
+# license.
+sub ACTION_ppport {
+ my $self = shift;
+ if ( !-e $PPPORT_H_PATH ) {
+ require Devel::PPPort;
+ $self->add_to_cleanup($PPPORT_H_PATH);
+ Devel::PPPort::WriteFile($PPPORT_H_PATH);
+ }
+}
+
+sub ACTION_code {
+ my $self = shift;
+ $self->dispatch('ppport');
+ $self->SUPER::ACTION_code;
+}
+
+1;
+
diff --git a/clownfish/include/CFC.h b/clownfish/include/CFC.h
new file mode 100644
index 0000000..634776b
--- /dev/null
+++ b/clownfish/include/CFC.h
@@ -0,0 +1,32 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "CFCBase.h"
+#include "CFCCBlock.h"
+#include "CFCClass.h"
+#include "CFCDocuComment.h"
+#include "CFCDumpable.h"
+#include "CFCFile.h"
+#include "CFCFunction.h"
+#include "CFCHierarchy.h"
+#include "CFCMethod.h"
+#include "CFCParamList.h"
+#include "CFCParcel.h"
+#include "CFCSymbol.h"
+#include "CFCType.h"
+#include "CFCUtil.h"
+#include "CFCVariable.h"
+
diff --git a/clownfish/lib/Clownfish.pm b/clownfish/lib/Clownfish.pm
new file mode 100644
index 0000000..ecb5302
--- /dev/null
+++ b/clownfish/lib/Clownfish.pm
@@ -0,0 +1,215 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish;
+use Clownfish::Base;
+our $VERSION = '0.01';
+
+BEGIN {
+ push @Clownfish::CBlock::ISA, "Clownfish::Base";
+ push @Clownfish::DocuComment::ISA, "Clownfish::Base";
+ push @Clownfish::File::ISA, "Clownfish::Base";
+ push @Clownfish::Parcel::ISA, "Clownfish::Base";
+ push @Clownfish::Symbol::ISA, "Clownfish::Base";
+ push @Clownfish::Type::ISA, "Clownfish::Base";
+}
+
+use XSLoader;
+BEGIN { XSLoader::load( 'Clownfish', '0.01' ) }
+
+1;
+
+=head1 NAME
+
+Clownfish - A small OO language that forms symbiotic relationships with "host"
+languages.
+
+=head1 PRIVATE API
+
+Clownfish is an Apache Lucy implementation detail. This documentation is partial --
+enough for the curious hacker, but not a full API.
+
+=head1 DESCRIPTION
+
+=head2 Overview.
+
+Clownfish is a small language for declaring an object oriented interface and a
+compiler which allows classes to be implemented either in C, in a "host"
+language, or a combination of both.
+
+=head2 Why use Clownfish?
+
+=over
+
+=item *
+
+Clownfish-based projects give users the ability to write full subclasses
+in any "host" language for which a binding has been prepared.
+
+=item *
+
+Pure C Clownfish class implementations are very fast.
+
+=item *
+
+Users can perform rapid prototyping in their language of choice, then port
+their classes to C either for speed or to make them available across multiple
+language platforms.
+
+=item *
+
+=back
+
+=head2 Object Model
+
+Clownfish is single-inheritance and class based -- a minimalist design which
+makes it as compatible as possible with a broad range of hosts.
+
+Subclasses may be created either at compile time or at run time.
+
+=back
+
+=head2 C method invocation syntax.
+
+Methods are differentiated from functions via capitalization:
+Boat_capsize() is a function, Boat_Capsize() is a method.
+
+ // Base method.
+ void
+ Boat_capsize(Boat *self)
+ {
+ self->upside_down = true;
+ }
+
+ // Implementing function, in Boat/Battleship.c
+ void
+ Battleship_capsize(Battleship *self)
+ {
+ // Superclass method invocation.
+ Boat_capsize_t capsize = (Boat_capsize_t)SUPER_METHOD(
+ BATTLESHIP, Battleship, Capsize);
+ capsize((Boat*)self);
+
+ // Subclass-specific behavior.
+ Battleship_Sink(self);
+ }
+
+ // Implementing function, in Boat/RubberDinghy.c
+ void
+ RubDing_capsize(RubberDinghy *self)
+ {
+ // Superclass method invocation.
+ Boat_capsize_t capsize = (Boat_capsize_t)SUPER_METHOD(
+ RUBBERDINGHY, RubDing, Capsize);
+ capsize((Boat*)self);
+
+ // Subclass-specific behavior.
+ RubDing_Drift(self);
+ }
+
+=head2 Class declaration syntax
+
+ [final] [inert] class CLASSNAME [cnick CNICK]
+ [inherits PARENT] [ : ATTRIBUTE ]* {
+
+ [declarations]
+
+ }
+
+Example:
+
+ class Boat::RubberDinghy cnick RubDing inherits Boat {
+
+ public inert incremented RubberDinghy*
+ new();
+
+ void
+ Capsize(RubberDinghy *self);
+ }
+
+=over
+
+=item * B<CLASSNAME> - The name of this class. The last string of characters
+will be used as the object's C struct name.
+
+=item * B<CNICK> - A recognizable abbreviation of the class name, used as a
+prefix for every function and method.
+
+=item * B<PARENT> - The full name of the parent class.
+
+=item * B<ATTRIBUTE> - An arbitrary attribute, e.g. "dumpable", or perhaps
+"serializable". A class may have multiple attributes, each preceded by a
+colon.
+
+=back
+
+=head2 Memory management
+
+At present, memory is managed via a reference counting scheme, but this is not
+inherently part of Clownfish.
+
+=head2 Namespaces, parcels, prefixes, and "short names"
+
+There are two levels of namespacing in Clownfish: parcels and classes.
+
+Clownfish classes intended to be published as a single unit may be grouped
+together using a "parcel". Parcel directives need to go at the top of each
+class file.
+
+ parcel Crustacean cnick Crust;
+
+All symbols generated by Clownfish for classes within a parcel will be
+prefixed by varying capitalizations of the parcel's C-nickname or "cnick" in
+order to avoid namespace collisions with other projects.
+
+Within a parcel, the last part of each class name must be unique.
+
+ class Crustacean::Lobster::Claw { ... }
+ class Crustacean::Crab::Claw { ... } // Illegal, "Claw" already used
+
+"Short names" -- names minus the parcel prefix -- will be auto-generated for
+all class symbols. When there is no danger of namespace collision, typically
+because no third-party non-system libraries are being pound-included, the
+short names can be used after a USE_SHORT_NAMES directive:
+
+ #define CRUST_USE_SHORT_NAMES
+
+The USE_SHORT_NAMES directives do not affect class prefixes, only parcel
+prefixes.
+
+ // No short names.
+ crust_LobsterClaw *claw = crust_LobClaw_new();
+
+ // With short names.
+ #define CRUST_USE_SHORT_NAMES
+ LobsterClaw *claw = LobClaw_new();
+
+=head2 Inclusion
+
+C header code generated by the Clownfish compiler is written to a file with
+whose name is the same as the .cfh file, but with an extension of ".h". C
+code should pound-include "Crustacean/Lobster.h" for a class defined in
+"Crustacean/Lobster.cfh".
+
+=head1 COPYRIGHT
+
+Clownfish is distributed under the Apache License, Version 2.0, as
+described in the file C<LICENSE> included with the distribution.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish.xs b/clownfish/lib/Clownfish.xs
new file mode 100644
index 0000000..a4773c7
--- /dev/null
+++ b/clownfish/lib/Clownfish.xs
@@ -0,0 +1,1668 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+
+#include "CFC.h"
+
+/* Rather than provide an XSUB for each accessor, we can use one multipath
+ * accessor function per class, with several Perl-space aliases. All set
+ * functions have odd-numbered aliases, and all get functions have
+ * even-numbered aliases. These two macros serve as bookends for the switch
+ * function.
+ */
+#define START_SET_OR_GET_SWITCH \
+ SV *retval = &PL_sv_undef; \
+ /* If called as a setter, make sure the extra arg is there. */ \
+ if (ix % 2 == 1) { \
+ if (items != 2) { croak("usage: $object->set_xxxxxx($val)"); } \
+ } \
+ else { \
+ if (items != 1) { croak("usage: $object->get_xxxxx()"); } \
+ } \
+ switch (ix) {
+
+#define END_SET_OR_GET_SWITCH \
+ default: croak("Internal error. ix: %d", ix); \
+ } \
+ if (ix % 2 == 0) { \
+ XPUSHs(sv_2mortal(retval)); \
+ XSRETURN(1); \
+ } \
+ else { \
+ XSRETURN(0); \
+ }
+
+static SV*
+S_cfcbase_to_perlref(void *thing) {
+ if (thing) {
+ SV *perl_obj = (SV*)CFCBase_get_perl_obj((CFCBase*)thing);
+ return newRV(perl_obj);
+ }
+ else {
+ return newSV(0);
+ }
+}
+
+// Transform a NULL-terminated array of CFCBase* into a Perl arrayref.
+static SV*
+S_array_of_cfcbase_to_av(CFCBase **things) {
+ AV *av = newAV();
+ size_t i;
+ for (i = 0; things[i] != NULL; i++) {
+ SV *val = S_cfcbase_to_perlref(things[i]);
+ av_store(av, i, val);
+ }
+ SV *retval = newRV((SV*)av);
+ SvREFCNT_dec(av);
+ return retval;
+}
+
+MODULE = Clownfish PACKAGE = Clownfish::CBlock
+
+SV*
+_new(klass, contents)
+ const char *klass;
+ const char *contents;
+CODE:
+ CFCCBlock *self = CFCCBlock_new(contents);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCCBlock *self;
+PPCODE:
+ CFCCBlock_destroy(self);
+
+void
+_set_or_get(self, ...)
+ CFCCBlock *self;
+ALIAS:
+ get_contents = 2
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ const char *contents = CFCCBlock_get_contents(self);
+ retval = newSVpvn(contents, strlen(contents));
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+MODULE = Clownfish PACKAGE = Clownfish::Class
+
+SV*
+_create(klass, parcel, exposure_sv, class_name_sv, cnick_sv, micro_sym_sv, docucomment, source_class_sv, parent_class_name_sv, is_final, is_inert)
+ const char *klass;
+ CFCParcel *parcel;
+ SV *exposure_sv;
+ SV *class_name_sv;
+ SV *cnick_sv;
+ SV *micro_sym_sv;
+ CFCDocuComment *docucomment;
+ SV *source_class_sv;
+ SV *parent_class_name_sv;
+ bool is_final;
+ bool is_inert;
+CODE:
+ const char *exposure =
+ SvOK(exposure_sv) ? SvPV_nolen(exposure_sv) : NULL;
+ const char *class_name =
+ SvOK(class_name_sv) ? SvPV_nolen(class_name_sv) : NULL;
+ const char *cnick =
+ SvOK(cnick_sv) ? SvPV_nolen(cnick_sv) : NULL;
+ const char *micro_sym =
+ SvOK(micro_sym_sv) ? SvPV_nolen(micro_sym_sv) : NULL;
+ const char *source_class =
+ SvOK(source_class_sv) ? SvPV_nolen(source_class_sv) : NULL;
+ const char *parent_class_name =
+ SvOK(parent_class_name_sv) ? SvPV_nolen(parent_class_name_sv) : NULL;
+ CFCClass *self = CFCClass_create(parcel, exposure, class_name, cnick,
+ micro_sym, docucomment, source_class,
+ parent_class_name, is_final, is_inert);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCClass *self;
+PPCODE:
+ CFCClass_destroy(self);
+
+SV*
+_fetch_singleton(parcel, class_name)
+ CFCParcel *parcel;
+ const char *class_name;
+CODE:
+ CFCClass *klass = CFCClass_fetch_singleton(parcel, class_name);
+ RETVAL = S_cfcbase_to_perlref(klass);
+OUTPUT: RETVAL
+
+void
+_register(self)
+ CFCClass *self;
+PPCODE:
+ CFCClass_register(self);
+
+void
+_clear_registry(...)
+PPCODE:
+ CFCClass_clear_registry();
+
+void
+append_autocode(self, autocode)
+ CFCClass *self;
+ const char *autocode;
+PPCODE:
+ CFCClass_append_autocode(self, autocode);
+
+void
+add_child(self, child)
+ CFCClass *self;
+ CFCClass *child;
+PPCODE:
+ CFCClass_add_child(self, child);
+
+void
+add_member_var(self, var)
+ CFCClass *self;
+ CFCVariable *var;
+PPCODE:
+ CFCClass_add_member_var(self, var);
+
+void
+add_function(self, func)
+ CFCClass *self;
+ CFCFunction *func;
+PPCODE:
+ CFCClass_add_function(self, func);
+
+void
+add_method(self, method)
+ CFCClass *self;
+ CFCMethod *method;
+PPCODE:
+ CFCClass_add_method(self, method);
+
+void
+add_attribute(self, name, value_sv)
+ CFCClass *self;
+ const char *name;
+ SV *value_sv;
+PPCODE:
+ char *value = SvOK(value_sv) ? SvPV_nolen(value_sv) : NULL;
+ CFCClass_add_attribute(self, name, value);
+
+int
+has_attribute(self, name)
+ CFCClass *self;
+ const char *name;
+CODE:
+ RETVAL = CFCClass_has_attribute(self, name);
+OUTPUT: RETVAL
+
+void
+grow_tree(self)
+ CFCClass *self;
+PPCODE:
+ CFCClass_grow_tree(self);
+
+void
+add_inert_var(self, var)
+ CFCClass *self;
+ CFCVariable *var;
+PPCODE:
+ CFCClass_add_inert_var(self, var);
+
+SV*
+function(self, sym)
+ CFCClass *self;
+ const char *sym;
+CODE:
+ CFCFunction *func = CFCClass_function(self, sym);
+ RETVAL = S_cfcbase_to_perlref(func);
+OUTPUT: RETVAL
+
+SV*
+method(self, sym)
+ CFCClass *self;
+ const char *sym;
+CODE:
+ CFCMethod *method = CFCClass_method(self, sym);
+ RETVAL = S_cfcbase_to_perlref(method);
+OUTPUT: RETVAL
+
+SV*
+novel_method(self, sym)
+ CFCClass *self;
+ const char *sym;
+CODE:
+ CFCMethod *method = CFCClass_novel_method(self, sym);
+ RETVAL = S_cfcbase_to_perlref(method);
+OUTPUT: RETVAL
+
+void
+_set_or_get(self, ...)
+ CFCClass *self;
+ALIAS:
+ get_cnick = 2
+ set_parent = 5
+ get_parent = 6
+ get_autocode = 8
+ get_source_class = 10
+ get_parent_class_name = 12
+ final = 14
+ inert = 16
+ get_struct_sym = 18
+ full_struct_sym = 20
+ short_vtable_var = 22
+ full_vtable_var = 24
+ full_vtable_type = 26
+ include_h = 28
+ get_docucomment = 30
+ children = 32
+ functions = 34
+ methods = 36
+ member_vars = 38
+ inert_vars = 40
+ tree_to_ladder = 42
+ novel_methods = 44
+ novel_member_vars = 46
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ const char *value = CFCClass_get_cnick(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 5: {
+ CFCClass *parent = NULL;
+ if (SvOK(ST(1))
+ && sv_derived_from(ST(1), "Clownfish::Class")
+ ) {
+ IV objint = SvIV((SV*)SvRV(ST(1)));
+ parent = INT2PTR(CFCClass*, objint);
+ }
+ CFCClass_set_parent(self, parent);
+ break;
+ }
+ case 6: {
+ CFCClass *parent = CFCClass_get_parent(self);
+ retval = S_cfcbase_to_perlref(parent);
+ break;
+ }
+ case 8: {
+ const char *value = CFCClass_get_autocode(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 10: {
+ const char *value = CFCClass_get_source_class(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 12: {
+ const char *value = CFCClass_get_parent_class_name(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 14:
+ retval = newSViv(CFCClass_final(self));
+ break;
+ case 16:
+ retval = newSViv(CFCClass_inert(self));
+ break;
+ case 18: {
+ const char *value = CFCClass_get_struct_sym(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 20: {
+ const char *value = CFCClass_full_struct_sym(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 22: {
+ const char *value = CFCClass_short_vtable_var(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 24: {
+ const char *value = CFCClass_full_vtable_var(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 26: {
+ const char *value = CFCClass_full_vtable_type(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 28: {
+ const char *value = CFCClass_include_h(self);
+ retval = value ? newSVpvn(value, strlen(value)) : newSV(0);
+ }
+ break;
+ case 30: {
+ CFCDocuComment *docucomment = CFCClass_get_docucomment(self);
+ retval = S_cfcbase_to_perlref(docucomment);
+ }
+ break;
+ case 32:
+ retval = S_array_of_cfcbase_to_av(
+ (CFCBase**)CFCClass_children(self));
+ break;
+ case 34:
+ retval = S_array_of_cfcbase_to_av((CFCBase**)CFCClass_functions(self));
+ break;
+ case 36:
+ retval = S_array_of_cfcbase_to_av((CFCBase**)CFCClass_methods(self));
+ break;
+ case 38:
+ retval = S_array_of_cfcbase_to_av((CFCBase**)CFCClass_member_vars(self));
+ break;
+ case 40:
+ retval = S_array_of_cfcbase_to_av((CFCBase**)CFCClass_inert_vars(self));
+ break;
+ case 42: {
+ CFCClass **ladder = CFCClass_tree_to_ladder(self);
+ retval = S_array_of_cfcbase_to_av((CFCBase**)ladder);
+ FREEMEM(ladder);
+ break;
+ }
+ case 44: {
+ CFCMethod **novel = CFCClass_novel_methods(self);
+ retval = S_array_of_cfcbase_to_av((CFCBase**)novel);
+ FREEMEM(novel);
+ break;
+ }
+ case 46: {
+ CFCVariable **novel = CFCClass_novel_member_vars(self);
+ retval = S_array_of_cfcbase_to_av((CFCBase**)novel);
+ FREEMEM(novel);
+ break;
+ }
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::DocuComment
+
+SV*
+parse(klass, text)
+ const char *klass;
+ const char *text;
+CODE:
+ CFCDocuComment *self = CFCDocuComment_parse(text);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCDocuComment *self;
+PPCODE:
+ CFCDocuComment_destroy(self);
+
+void
+_set_or_get(self, ...)
+ CFCDocuComment *self;
+ALIAS:
+ get_description = 2
+ get_brief = 4
+ get_long = 6
+ get_param_names = 8
+ get_param_docs = 10
+ get_retval = 12
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ const char *description = CFCDocuComment_get_description(self);
+ retval = newSVpvn(description, strlen(description));
+ }
+ break;
+ case 4: {
+ const char *brief = CFCDocuComment_get_brief(self);
+ retval = newSVpvn(brief, strlen(brief));
+ }
+ break;
+ case 6: {
+ const char *long_description = CFCDocuComment_get_long(self);
+ retval = newSVpvn(long_description, strlen(long_description));
+ }
+ break;
+ case 8: {
+ AV *av = newAV();
+ const char **names = CFCDocuComment_get_param_names(self);
+ size_t i;
+ for (i = 0; names[i] != NULL; i++) {
+ SV *val_sv = newSVpvn(names[i], strlen(names[i]));
+ av_store(av, i, val_sv);
+ }
+ retval = newRV((SV*)av);
+ SvREFCNT_dec(av);
+ break;
+ }
+ case 10: {
+ AV *av = newAV();
+ const char **docs = CFCDocuComment_get_param_docs(self);
+ size_t i;
+ for (i = 0; docs[i] != NULL; i++) {
+ SV *val_sv = newSVpvn(docs[i], strlen(docs[i]));
+ av_store(av, i, val_sv);
+ }
+ retval = newRV((SV*)av);
+ SvREFCNT_dec(av);
+ break;
+ }
+ case 12: {
+ const char *rv = CFCDocuComment_get_retval(self);
+ retval = rv ? newSVpvn(rv, strlen(rv)) : newSV(0);
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+MODULE = Clownfish PACKAGE = Clownfish::Dumpable
+
+SV*
+_new(klass)
+ const char *klass;
+CODE:
+ CFCDumpable *self = CFCDumpable_new();
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCDumpable *self;
+PPCODE:
+ CFCDumpable_destroy(self);
+
+void
+add_dumpables(self, klass)
+ CFCDumpable *self;
+ CFCClass *klass;
+PPCODE:
+ CFCDumpable_add_dumpables(self, klass);
+
+
+MODULE = Clownfish PACKAGE = Clownfish::File
+
+SV*
+_new(klass, source_class)
+ const char *klass;
+ const char *source_class;
+CODE:
+ CFCFile *self = CFCFile_new(source_class);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+_destroy(self)
+ CFCFile *self;
+PPCODE:
+ CFCFile_destroy(self);
+
+void
+add_block(self, block)
+ CFCFile *self;
+ CFCBase *block;
+PPCODE:
+ CFCFile_add_block(self, block);
+
+void
+_set_or_get(self, ...)
+ CFCFile *self;
+ALIAS:
+ set_modified = 1
+ get_modified = 2
+ get_source_class = 4
+ guard_name = 6
+ guard_start = 8
+ guard_close = 10
+ blocks = 12
+ classes = 14
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 1:
+ CFCFile_set_modified(self, !!SvTRUE(ST(1)));
+ break;
+ case 2:
+ retval = newSViv(CFCFile_get_modified(self));
+ break;
+ case 4: {
+ const char *value = CFCFile_get_source_class(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 6: {
+ const char *value = CFCFile_guard_name(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 8: {
+ const char *value = CFCFile_guard_start(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 10: {
+ const char *value = CFCFile_guard_close(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 12:
+ retval = S_array_of_cfcbase_to_av(CFCFile_blocks(self));
+ break;
+ case 14:
+ retval = S_array_of_cfcbase_to_av(
+ (CFCBase**)CFCFile_classes(self));
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+SV*
+_gen_path(self, base_dir = NULL)
+ CFCFile *self;
+ const char *base_dir;
+ALIAS:
+ c_path = 1
+ h_path = 2
+ cfh_path = 3
+CODE:
+{
+ size_t buf_size = CFCFile_path_buf_size(self, base_dir);
+ RETVAL = newSV(buf_size);
+ SvPOK_on(RETVAL);
+ char *buf = SvPVX(RETVAL);
+ switch (ix) {
+ case 1:
+ CFCFile_c_path(self, buf, buf_size, base_dir);
+ break;
+ case 2:
+ CFCFile_h_path(self, buf, buf_size, base_dir);
+ break;
+ case 3:
+ CFCFile_cfh_path(self, buf, buf_size, base_dir);
+ break;
+ default:
+ croak("unexpected ix value: %d", ix);
+ }
+ SvCUR_set(RETVAL, strlen(buf));
+}
+OUTPUT: RETVAL
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Function
+
+SV*
+_new(klass, parcel, exposure_sv, class_name_sv, class_cnick_sv, micro_sym_sv, return_type, param_list, docucomment, is_inline)
+ const char *klass;
+ CFCParcel *parcel;
+ SV *exposure_sv;
+ SV *class_name_sv;
+ SV *class_cnick_sv;
+ SV *micro_sym_sv;
+ CFCType *return_type;
+ CFCParamList *param_list;
+ CFCDocuComment *docucomment;
+ int is_inline;
+CODE:
+ const char *exposure =
+ SvOK(exposure_sv) ? SvPV_nolen(exposure_sv) : NULL;
+ const char *class_name =
+ SvOK(class_name_sv) ? SvPV_nolen(class_name_sv) : NULL;
+ const char *class_cnick =
+ SvOK(class_cnick_sv) ? SvPV_nolen(class_cnick_sv) : NULL;
+ const char *micro_sym =
+ SvOK(micro_sym_sv) ? SvPV_nolen(micro_sym_sv) : NULL;
+ CFCFunction *self = CFCFunction_new(parcel, exposure, class_name,
+ class_cnick, micro_sym, return_type,
+ param_list, docucomment, is_inline);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCFunction *self;
+PPCODE:
+ CFCFunction_destroy(self);
+
+void
+_set_or_get(self, ...)
+ CFCFunction *self;
+ALIAS:
+ get_return_type = 2
+ get_param_list = 4
+ get_docucomment = 6
+ inline = 8
+ void = 10
+ full_func_sym = 12
+ short_func_sym = 14
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ CFCType *type = CFCFunction_get_return_type(self);
+ retval = S_cfcbase_to_perlref(type);
+ }
+ break;
+ case 4: {
+ CFCParamList *param_list = CFCFunction_get_param_list(self);
+ retval = S_cfcbase_to_perlref(param_list);
+ }
+ break;
+ case 6: {
+ CFCDocuComment *docucomment
+ = CFCFunction_get_docucomment(self);
+ retval = S_cfcbase_to_perlref(docucomment);
+ }
+ break;
+ case 8:
+ retval = newSViv(CFCFunction_inline(self));
+ break;
+ case 10:
+ retval = newSViv(CFCFunction_void(self));
+ break;
+ case 12: {
+ const char *full_sym = CFCFunction_full_func_sym(self);
+ retval = newSVpv(full_sym, strlen(full_sym));
+ }
+ break;
+ case 14: {
+ const char *short_sym = CFCFunction_short_func_sym(self);
+ retval = newSVpv(short_sym, strlen(short_sym));
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+MODULE = Clownfish PACKAGE = Clownfish::Hierarchy
+
+SV*
+_new(klass, source, dest, parser)
+ const char *klass;
+ const char *source;
+ const char *dest;
+ SV *parser;
+CODE:
+ CFCHierarchy *self = CFCHierarchy_new(source, dest, parser);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCHierarchy *self;
+PPCODE:
+ CFCHierarchy_destroy(self);
+
+void
+build(self)
+ CFCHierarchy *self;
+PPCODE:
+ CFCHierarchy_build(self);
+
+int
+propagate_modified(self, ...)
+ CFCHierarchy *self;
+CODE:
+ int modified = items > 1 ? !!SvTRUE(ST(1)) : 0;
+ RETVAL = CFCHierarchy_propagate_modified(self, modified);
+OUTPUT: RETVAL
+
+void
+_set_or_get(self, ...)
+ CFCHierarchy *self;
+ALIAS:
+ get_source = 2
+ get_dest = 4
+ files = 8
+ ordered_classes = 10
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ const char *value = CFCHierarchy_get_source(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 4: {
+ const char *value = CFCHierarchy_get_dest(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 8:
+ retval = S_array_of_cfcbase_to_av(
+ (CFCBase**)CFCHierarchy_files(self));
+ break;
+ case 10: {
+ CFCClass **ladder = CFCHierarchy_ordered_classes(self);
+ retval = S_array_of_cfcbase_to_av((CFCBase**)ladder);
+ FREEMEM(ladder);
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Method
+
+SV*
+_new(klass, parcel, exposure_sv, class_name_sv, class_cnick_sv, macro_sym, return_type, param_list, docucomment, is_final, is_abstract)
+ const char *klass;
+ CFCParcel *parcel;
+ SV *exposure_sv;
+ SV *class_name_sv;
+ SV *class_cnick_sv;
+ const char *macro_sym;
+ CFCType *return_type;
+ CFCParamList *param_list;
+ CFCDocuComment *docucomment;
+ int is_final;
+ int is_abstract;
+CODE:
+ const char *exposure =
+ SvOK(exposure_sv) ? SvPV_nolen(exposure_sv) : NULL;
+ const char *class_name =
+ SvOK(class_name_sv) ? SvPV_nolen(class_name_sv) : NULL;
+ const char *class_cnick =
+ SvOK(class_cnick_sv) ? SvPV_nolen(class_cnick_sv) : NULL;
+ CFCMethod *self = CFCMethod_new(parcel, exposure, class_name, class_cnick,
+ macro_sym, return_type, param_list,
+ docucomment, is_final, is_abstract);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCMethod *self;
+PPCODE:
+ CFCMethod_destroy(self);
+
+int
+compatible(self, other)
+ CFCMethod *self;
+ CFCMethod *other;
+CODE:
+ RETVAL = CFCMethod_compatible(self, other);
+OUTPUT: RETVAL
+
+void
+override(self, other)
+ CFCMethod *self;
+ CFCMethod *other;
+PPCODE:
+ CFCMethod_override(self, other);
+
+SV*
+finalize(self)
+ CFCMethod *self;
+CODE:
+ CFCMethod *finalized = CFCMethod_finalize(self);
+ RETVAL = S_cfcbase_to_perlref(finalized);
+ CFCBase_decref((CFCBase*)finalized);
+OUTPUT: RETVAL
+
+SV*
+_various_method_syms(self, invoker)
+ CFCMethod *self;
+ const char *invoker;
+ALIAS:
+ short_method_sym = 1
+ full_method_sym = 2
+ full_offset_sym = 3
+CODE:
+ size_t size = 0;
+ switch (ix) {
+ case 1:
+ size = CFCMethod_short_method_sym(self, invoker, NULL, 0);
+ break;
+ case 2:
+ size = CFCMethod_full_method_sym(self, invoker, NULL, 0);
+ break;
+ case 3:
+ size = CFCMethod_full_offset_sym(self, invoker, NULL, 0);
+ break;
+ default: croak("Unexpected ix: %d", ix);
+ }
+ RETVAL = newSV(size);
+ SvPOK_on(RETVAL);
+ char *buf = SvPVX(RETVAL);
+ switch (ix) {
+ case 1:
+ CFCMethod_short_method_sym(self, invoker, buf, size);
+ break;
+ case 2:
+ CFCMethod_full_method_sym(self, invoker, buf, size);
+ break;
+ case 3:
+ CFCMethod_full_offset_sym(self, invoker, buf, size);
+ break;
+ default: croak("Unexpected ix: %d", ix);
+ }
+ SvCUR_set(RETVAL, strlen(buf));
+OUTPUT: RETVAL
+
+void
+_set_or_get(self, ...)
+ CFCMethod *self;
+ALIAS:
+ get_macro_sym = 2
+ short_typedef = 4
+ full_typedef = 6
+ full_callback_sym = 8
+ full_override_sym = 10
+ abstract = 12
+ novel = 14
+ final = 16
+ self_type = 18
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ const char *macro_sym = CFCMethod_get_macro_sym(self);
+ retval = newSVpvn(macro_sym, strlen(macro_sym));
+ }
+ break;
+ case 4: {
+ const char *short_typedef = CFCMethod_short_typedef(self);
+ retval = newSVpvn(short_typedef, strlen(short_typedef));
+ }
+ break;
+ case 6: {
+ const char *value = CFCMethod_full_typedef(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 8: {
+ const char *value = CFCMethod_full_callback_sym(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 10: {
+ const char *value = CFCMethod_full_override_sym(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 12:
+ retval = newSViv(CFCMethod_abstract(self));
+ break;
+ case 14:
+ retval = newSViv(CFCMethod_novel(self));
+ break;
+ case 16:
+ retval = newSViv(CFCMethod_final(self));
+ break;
+ case 18: {
+ CFCType *type = CFCMethod_self_type(self);
+ retval = S_cfcbase_to_perlref(type);
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::ParamList
+
+SV*
+_new(klass, variadic)
+ const char *klass;
+ int variadic;
+CODE:
+ CFCParamList *self = CFCParamList_new(variadic);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCParamList *self;
+PPCODE:
+ CFCParamList_destroy(self);
+
+void
+add_param(self, variable, value_sv)
+ CFCParamList *self;
+ CFCVariable *variable;
+ SV *value_sv;
+PPCODE:
+ const char *value = SvOK(value_sv) ? SvPV_nolen(value_sv) : NULL;
+ CFCParamList_add_param(self, variable, value);
+
+void
+_set_or_get(self, ...)
+ CFCParamList *self;
+ALIAS:
+ get_variables = 2
+ get_initial_values = 4
+ variadic = 6
+ num_vars = 8
+ to_c = 10
+ name_list = 12
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ AV *av = newAV();
+ CFCVariable **vars = CFCParamList_get_variables(self);
+ size_t i;
+ size_t num_vars = CFCParamList_num_vars(self);
+ for (i = 0; i < num_vars; i++) {
+ SV *ref = S_cfcbase_to_perlref(vars[i]);
+ av_store(av, i, ref);
+ }
+ retval = newRV((SV*)av);
+ SvREFCNT_dec(av);
+ break;
+ }
+ case 4: {
+ AV *av = newAV();
+ const char **values = CFCParamList_get_initial_values(self);
+ size_t i;
+ size_t num_vars = CFCParamList_num_vars(self);
+ for (i = 0; i < num_vars; i++) {
+ if (values[i] != NULL) {
+ SV *val_sv = newSVpvn(values[i], strlen(values[i]));
+ av_store(av, i, val_sv);
+ }
+ else {
+ av_store(av, i, newSV(0));
+ }
+ }
+ retval = newRV((SV*)av);
+ SvREFCNT_dec(av);
+ break;
+ }
+ case 6:
+ retval = newSViv(CFCParamList_variadic(self));
+ break;
+ case 8:
+ retval = newSViv(CFCParamList_num_vars(self));
+ break;
+ case 10: {
+ const char *value = CFCParamList_to_c(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ case 12: {
+ const char *value = CFCParamList_name_list(self);
+ retval = newSVpv(value, strlen(value));
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Parcel
+
+SV*
+_singleton(klass, name_sv, cnick_sv)
+ const char *klass;
+ SV *name_sv;
+ SV *cnick_sv;
+CODE:
+ const char *name = SvOK(name_sv) ? SvPV_nolen(name_sv) : NULL;
+ const char *cnick = SvOK(cnick_sv) ? SvPV_nolen(cnick_sv) : NULL;
+ CFCParcel *self = CFCParcel_singleton(name, cnick);
+ RETVAL = S_cfcbase_to_perlref(self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCParcel *self;
+PPCODE:
+ CFCParcel_destroy(self);
+
+int
+equals(self, other)
+ CFCParcel *self;
+ CFCParcel *other;
+CODE:
+ RETVAL = CFCParcel_equals(self, other);
+OUTPUT: RETVAL
+
+SV*
+default_parcel(...)
+CODE:
+ CFCParcel *default_parcel = CFCParcel_default_parcel();
+ RETVAL = S_cfcbase_to_perlref(default_parcel);
+OUTPUT: RETVAL
+
+void
+reap_singletons(...)
+PPCODE:
+ CFCParcel_reap_singletons();
+
+void
+_set_or_get(self, ...)
+ CFCParcel *self;
+ALIAS:
+ get_name = 2
+ get_cnick = 4
+ get_prefix = 6
+ get_Prefix = 8
+ get_PREFIX = 10
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ const char *name = CFCParcel_get_name(self);
+ retval = newSVpvn(name, strlen(name));
+ }
+ break;
+ case 4: {
+ const char *cnick = CFCParcel_get_cnick(self);
+ retval = newSVpvn(cnick, strlen(cnick));
+ }
+ break;
+ case 6: {
+ const char *value = CFCParcel_get_prefix(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 8: {
+ const char *value = CFCParcel_get_Prefix(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 10: {
+ const char *value = CFCParcel_get_PREFIX(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Symbol
+
+SV*
+_new(klass, parcel, exposure, class_name_sv, class_cnick_sv, micro_sym_sv)
+ const char *klass;
+ CFCParcel *parcel;
+ const char *exposure;
+ SV *class_name_sv;
+ SV *class_cnick_sv;
+ SV *micro_sym_sv;
+CODE:
+ const char *class_name = SvOK(class_name_sv)
+ ? SvPV_nolen(class_name_sv)
+ : NULL;
+ const char *class_cnick = SvOK(class_cnick_sv)
+ ? SvPV_nolen(class_cnick_sv)
+ : NULL;
+ const char *micro_sym = SvOK(micro_sym_sv)
+ ? SvPV_nolen(micro_sym_sv)
+ : NULL;
+ CFCSymbol *self = CFCSymbol_new(parcel, exposure, class_name, class_cnick,
+ micro_sym);
+ RETVAL = newSV(0);
+ sv_setref_pv(RETVAL, klass, (void*)self);
+OUTPUT: RETVAL
+
+int
+equals(self, other)
+ CFCSymbol *self;
+ CFCSymbol *other;
+CODE:
+ RETVAL = CFCSymbol_equals(self, other);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCSymbol *self;
+PPCODE:
+ CFCSymbol_destroy(self);
+
+void
+_set_or_get(self, ...)
+ CFCSymbol *self;
+ALIAS:
+ get_parcel = 2
+ get_class_name = 4
+ get_class_cnick = 6
+ get_exposure = 8
+ micro_sym = 10
+ get_prefix = 12
+ get_Prefix = 14
+ get_PREFIX = 16
+ public = 18
+ private = 20
+ parcel = 22
+ local = 24
+ short_sym = 26
+ full_sym = 28
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ struct CFCParcel *parcel = CFCSymbol_get_parcel(self);
+ retval = S_cfcbase_to_perlref(parcel);
+ }
+ break;
+ case 4: {
+ const char *class_name = CFCSymbol_get_class_name(self);
+ retval = class_name
+ ? newSVpvn(class_name, strlen(class_name))
+ : newSV(0);
+ }
+ break;
+ case 6: {
+ const char *class_cnick = CFCSymbol_get_class_cnick(self);
+ retval = class_cnick
+ ? newSVpvn(class_cnick, strlen(class_cnick))
+ : newSV(0);
+ }
+ break;
+ case 8: {
+ const char *exposure = CFCSymbol_get_exposure(self);
+ retval = newSVpvn(exposure, strlen(exposure));
+ }
+ break;
+ case 10: {
+ const char *micro_sym = CFCSymbol_micro_sym(self);
+ retval = newSVpvn(micro_sym, strlen(micro_sym));
+ }
+ break;
+ case 12: {
+ const char *value = CFCSymbol_get_prefix(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 14: {
+ const char *value = CFCSymbol_get_Prefix(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 16: {
+ const char *value = CFCSymbol_get_PREFIX(self);
+ retval = newSVpvn(value, strlen(value));
+ }
+ break;
+ case 18:
+ retval = newSViv(CFCSymbol_public(self));
+ break;
+ case 20:
+ retval = newSViv(CFCSymbol_private(self));
+ break;
+ case 22:
+ retval = newSViv(CFCSymbol_parcel(self));
+ break;
+ case 24:
+ retval = newSViv(CFCSymbol_local(self));
+ break;
+ case 26: {
+ const char *short_sym = CFCSymbol_short_sym(self);
+ retval = newSVpvn(short_sym, strlen(short_sym));
+ }
+ break;
+ case 28: {
+ const char *full_sym = CFCSymbol_full_sym(self);
+ retval = newSVpvn(full_sym, strlen(full_sym));
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Type
+
+SV*
+_new(klass, flags, parcel, specifier, indirection, c_string)
+ const char *klass;
+ int flags;
+ CFCParcel *parcel;
+ const char *specifier;
+ int indirection;
+ const char *c_string;
+CODE:
+ CFCType *self = CFCType_new(flags, parcel, specifier, indirection,
+ c_string);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_integer(klass, flags, specifier)
+ const char *klass;
+ int flags;
+ const char *specifier;
+CODE:
+ CFCType *self = CFCType_new_integer(flags, specifier);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_float(klass, flags, specifier)
+ const char *klass;
+ int flags;
+ const char *specifier;
+CODE:
+ CFCType *self = CFCType_new_float(flags, specifier);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_object(klass, flags, parcel, specifier, indirection)
+ const char *klass;
+ int flags;
+ CFCParcel *parcel;
+ const char *specifier;
+ int indirection;
+CODE:
+ CFCType *self = CFCType_new_object(flags, parcel, specifier, indirection);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_composite(klass, flags, child_sv, indirection, array)
+ const char *klass;
+ int flags;
+ SV *child_sv;
+ int indirection;
+ const char *array;
+CODE:
+ CFCType *child = NULL;
+ if (SvOK(child_sv) && sv_derived_from(child_sv, "Clownfish::Type")) {
+ IV objint = SvIV((SV*)SvRV(child_sv));
+ child = INT2PTR(CFCType*, objint);
+ }
+ else {
+ croak("Param 'child' not a Clownfish::Type");
+ }
+ CFCType *self = CFCType_new_composite(flags, child, indirection, array);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_void(klass, is_const)
+ const char *klass;
+ int is_const;
+CODE:
+ CFCType *self = CFCType_new_void(is_const);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_va_list(klass)
+ const char *klass;
+CODE:
+ CFCType *self = CFCType_new_va_list();
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+SV*
+_new_arbitrary(klass, parcel, specifier)
+ const char *klass;
+ CFCParcel *parcel;
+ const char *specifier;
+CODE:
+ CFCType *self = CFCType_new_arbitrary(parcel, specifier);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCType *self;
+PPCODE:
+ CFCType_destroy(self);
+
+int
+equals(self, other)
+ CFCType *self;
+ CFCType *other;
+CODE:
+ RETVAL = CFCType_equals(self, other);
+OUTPUT: RETVAL
+
+int
+similar(self, other)
+ CFCType *self;
+ CFCType *other;
+CODE:
+ RETVAL = CFCType_similar(self, other);
+OUTPUT: RETVAL
+
+unsigned
+CONST(...)
+CODE:
+ RETVAL = CFCTYPE_CONST;
+OUTPUT: RETVAL
+
+unsigned
+NULLABLE(...)
+CODE:
+ RETVAL = CFCTYPE_NULLABLE;
+OUTPUT: RETVAL
+
+unsigned
+INCREMENTED(...)
+CODE:
+ RETVAL = CFCTYPE_INCREMENTED;
+OUTPUT: RETVAL
+
+unsigned
+DECREMENTED(...)
+CODE:
+ RETVAL = CFCTYPE_DECREMENTED;
+OUTPUT: RETVAL
+
+unsigned
+VOID(...)
+CODE:
+ RETVAL = CFCTYPE_VOID;
+OUTPUT: RETVAL
+
+unsigned
+OBJECT(...)
+CODE:
+ RETVAL = CFCTYPE_OBJECT;
+OUTPUT: RETVAL
+
+unsigned
+PRIMITIVE(...)
+CODE:
+ RETVAL = CFCTYPE_PRIMITIVE;
+OUTPUT: RETVAL
+
+unsigned
+INTEGER(...)
+CODE:
+ RETVAL = CFCTYPE_INTEGER;
+OUTPUT: RETVAL
+
+unsigned
+FLOATING(...)
+CODE:
+ RETVAL = CFCTYPE_FLOATING;
+OUTPUT: RETVAL
+
+unsigned
+STRING_TYPE(...)
+CODE:
+ RETVAL = CFCTYPE_STRING_TYPE;
+OUTPUT: RETVAL
+
+unsigned
+VA_LIST(...)
+CODE:
+ RETVAL = CFCTYPE_VA_LIST;
+OUTPUT: RETVAL
+
+unsigned
+ARBITRARY(...)
+CODE:
+ RETVAL = CFCTYPE_ARBITRARY;
+OUTPUT: RETVAL
+
+unsigned
+COMPOSITE(...)
+CODE:
+ RETVAL = CFCTYPE_COMPOSITE;
+OUTPUT: RETVAL
+
+void
+_set_or_get(self, ...)
+ CFCType *self;
+ALIAS:
+ set_specifier = 1
+ get_specifier = 2
+ get_parcel = 4
+ get_indirection = 6
+ set_c_string = 7
+ to_c = 8
+ const = 10
+ set_nullable = 11
+ nullable = 12
+ is_void = 14
+ is_object = 16
+ is_primitive = 18
+ is_integer = 20
+ is_floating = 22
+ is_string_type = 24
+ is_va_list = 26
+ is_arbitrary = 28
+ is_composite = 30
+ get_width = 32
+ incremented = 34
+ decremented = 36
+ get_array = 38
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 1:
+ CFCType_set_specifier(self, SvPV_nolen(ST(1)));
+ break;
+ case 2: {
+ const char *specifier = CFCType_get_specifier(self);
+ retval = newSVpvn(specifier, strlen(specifier));
+ }
+ break;
+ case 4: {
+ CFCParcel *parcel = CFCType_get_parcel(self);
+ retval = S_cfcbase_to_perlref(parcel);
+ }
+ break;
+ case 6:
+ retval = newSViv(CFCType_get_indirection(self));
+ break;
+ case 7:
+ CFCType_set_c_string(self, SvPV_nolen(ST(1)));
+ case 8: {
+ const char *c_string = CFCType_to_c(self);
+ retval = newSVpvn(c_string, strlen(c_string));
+ }
+ break;
+ case 10:
+ retval = newSViv(CFCType_const(self));
+ break;
+ case 11:
+ CFCType_set_nullable(self, !!SvTRUE(ST(1)));
+ break;
+ case 12:
+ retval = newSViv(CFCType_nullable(self));
+ break;
+ case 14:
+ retval = newSViv(CFCType_is_void(self));
+ break;
+ case 16:
+ retval = newSViv(CFCType_is_object(self));
+ break;
+ case 18:
+ retval = newSViv(CFCType_is_primitive(self));
+ break;
+ case 20:
+ retval = newSViv(CFCType_is_integer(self));
+ break;
+ case 22:
+ retval = newSViv(CFCType_is_floating(self));
+ break;
+ case 24:
+ retval = newSViv(CFCType_is_string_type(self));
+ break;
+ case 26:
+ retval = newSViv(CFCType_is_va_list(self));
+ break;
+ case 28:
+ retval = newSViv(CFCType_is_arbitrary(self));
+ break;
+ case 30:
+ retval = newSViv(CFCType_is_composite(self));
+ break;
+ case 32:
+ retval = newSVuv(CFCType_get_width(self));
+ break;
+ case 34:
+ retval = newSVuv(CFCType_incremented(self));
+ break;
+ case 36:
+ retval = newSVuv(CFCType_decremented(self));
+ break;
+ case 38: {
+ const char *array = CFCType_get_array(self);
+ retval = array
+ ? newSVpvn(array, strlen(array))
+ : newSV(0);
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Util
+
+SV*
+trim_whitespace(text)
+ SV *text;
+CODE:
+ RETVAL = newSVsv(text);
+ STRLEN len;
+ char *ptr = SvPV(RETVAL, len);
+ CFCUtil_trim_whitespace(ptr);
+ SvCUR_set(RETVAL, strlen(ptr));
+OUTPUT: RETVAL
+
+SV*
+slurp_text(path)
+ const char *path;
+CODE:
+ size_t len;
+ char *contents = CFCUtil_slurp_text(path, &len);
+ RETVAL = newSVpvn(contents, len);
+ FREEMEM(contents);
+OUTPUT: RETVAL
+
+int
+current(orig, dest)
+ const char *orig;
+ const char *dest;
+CODE:
+ RETVAL = CFCUtil_current(orig, dest);
+OUTPUT: RETVAL
+
+void
+write_if_changed(path, content_sv)
+ const char *path;
+ SV *content_sv;
+PPCODE:
+ STRLEN len;
+ char *content = SvPV(content_sv, len);
+ CFCUtil_write_if_changed(path, content, len);
+
+
+MODULE = Clownfish PACKAGE = Clownfish::Variable
+
+SV*
+_new(klass, parcel, exposure, class_name_sv, class_cnick_sv, micro_sym_sv, type_sv)
+ const char *klass;
+ CFCParcel *parcel;
+ const char *exposure;
+ SV *class_name_sv;
+ SV *class_cnick_sv;
+ SV *micro_sym_sv;
+ SV *type_sv;
+CODE:
+ const char *class_name = SvOK(class_name_sv)
+ ? SvPV_nolen(class_name_sv)
+ : NULL;
+ const char *class_cnick = SvOK(class_cnick_sv)
+ ? SvPV_nolen(class_cnick_sv)
+ : NULL;
+ const char *micro_sym = SvOK(micro_sym_sv)
+ ? SvPV_nolen(micro_sym_sv)
+ : NULL;
+ CFCType *type = NULL;
+ if (SvOK(type_sv) && sv_derived_from(type_sv, "Clownfish::Type")) {
+ IV objint = SvIV((SV*)SvRV(type_sv));
+ type = INT2PTR(CFCType*, objint);
+ }
+ else {
+ croak("Param 'type' is not a Clownfish::Type");
+ }
+ CFCVariable *self = CFCVariable_new(parcel, exposure, class_name,
+ class_cnick, micro_sym, type);
+ RETVAL = S_cfcbase_to_perlref(self);
+ CFCBase_decref((CFCBase*)self);
+OUTPUT: RETVAL
+
+void
+DESTROY(self)
+ CFCVariable *self;
+PPCODE:
+ CFCVariable_destroy(self);
+
+int
+equals(self, other)
+ CFCVariable *self;
+ CFCVariable *other;
+CODE:
+ RETVAL = CFCVariable_equals(self, other);
+OUTPUT: RETVAL
+
+void
+_set_or_get(self, ...)
+ CFCVariable *self;
+ALIAS:
+ get_type = 2
+ local_c = 4
+ global_c = 6
+ local_declaration = 8
+PPCODE:
+{
+ START_SET_OR_GET_SWITCH
+ case 2: {
+ CFCType *type = CFCVariable_get_type(self);
+ retval = S_cfcbase_to_perlref(type);
+ }
+ break;
+ case 4: {
+ const char *local_c = CFCVariable_local_c(self);
+ retval = newSVpvn(local_c, strlen(local_c));
+ }
+ break;
+ case 6: {
+ const char *global_c = CFCVariable_global_c(self);
+ retval = newSVpvn(global_c, strlen(global_c));
+ }
+ break;
+ case 8: {
+ const char *local_dec = CFCVariable_local_declaration(self);
+ retval = newSVpvn(local_dec, strlen(local_dec));
+ }
+ break;
+ END_SET_OR_GET_SWITCH
+}
+
diff --git a/clownfish/lib/Clownfish/Base.pm b/clownfish/lib/Clownfish/Base.pm
new file mode 100644
index 0000000..4c17c12
--- /dev/null
+++ b/clownfish/lib/Clownfish/Base.pm
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Base;
+use Clownfish;
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Base - Base class for all CFC objects.
+
+=head1 DESCRIPTION
+
+ # TODO
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Binding/Core.pm b/clownfish/lib/Clownfish/Binding/Core.pm
new file mode 100644
index 0000000..97983bc
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Core.pm
@@ -0,0 +1,254 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Core;
+use Clownfish::Util qw( a_isa_b verify_args );
+use Clownfish::Binding::Core::File;
+use Clownfish::Binding::Core::Aliases;
+use File::Spec::Functions qw( catfile );
+use Fcntl;
+
+our %new_PARAMS = (
+ hierarchy => undef,
+ dest => undef,
+ header => undef,
+ footer => undef,
+);
+
+sub new {
+ my $either = shift;
+ verify_args( \%new_PARAMS, @_ ) or confess $@;
+ my $self = bless { %new_PARAMS, @_ }, ref($either) || $either;
+
+ # Validate.
+ for ( keys %new_PARAMS ) {
+ confess("Missing required param '$_'") unless defined $self->{$_};
+ }
+ confess("Not a Hierarchy")
+ unless a_isa_b( $self->{hierarchy}, "Clownfish::Hierarchy" );
+
+ return $self;
+}
+
+sub write_all_modified {
+ my ( $self, $modified ) = @_;
+ my $hierarchy = $self->{hierarchy};
+ my $header = $self->{header};
+ my $footer = $self->{footer};
+ my $dest = $self->{dest};
+
+ $modified = $hierarchy->propagate_modified($modified);
+
+ # Iterate over all File objects, writing out those which don't have
+ # up-to-date auto-generated files.
+ my %written;
+ for my $file ( @{ $hierarchy->files } ) {
+ next unless $file->get_modified;
+ my $source_class = $file->get_source_class;
+ next if $written{$source_class};
+ $written{$source_class} = 1;
+ Clownfish::Binding::Core::File->write_h(
+ file => $file,
+ dest => $dest,
+ header => $header,
+ footer => $footer,
+ );
+ Clownfish::Binding::Core::File->write_c(
+ file => $file,
+ dest => $dest,
+ header => $header,
+ footer => $footer,
+ );
+ }
+
+ # If any class definition changed, rewrite the boil.h file.
+ $self->_write_boil_h if $modified;
+
+ return $modified;
+}
+
+# Write the "boil.h" header file, which contains common symbols needed by all
+# classes, plus typedefs for all class structs.
+sub _write_boil_h {
+ my $self = shift;
+ my $ordered = $self->{hierarchy}->ordered_classes;
+ my $typedefs = "";
+
+ # Declare object structs for all instantiable classes.
+ for my $class (@$ordered) {
+ next if $class->inert;
+ my $full_struct = $class->full_struct_sym;
+ $typedefs .= "typedef struct $full_struct $full_struct;\n";
+ }
+
+ # Create Clownfish aliases if necessary.
+ my $aliases = Clownfish::Binding::Core::Aliases->c_aliases;
+
+ my $filepath = catfile( $self->{dest}, "boil.h" );
+ unlink $filepath;
+ sysopen( my $fh, $filepath, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$filepath': $!");
+ print $fh <<END_STUFF;
+$self->{header}
+#ifndef BOIL_H
+#define BOIL_H 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include "charmony.h"
+
+$aliases
+$typedefs
+
+/* Refcount / host object */
+typedef union {
+ size_t count;
+ void *host_obj;
+} cfish_ref_t;
+
+/* Generic method pointer.
+ */
+typedef void
+(*cfish_method_t)(const void *vself);
+
+/* Access the function pointer for a given method from the vtable.
+ */
+#define LUCY_METHOD(_vtable, _class_nick, _meth_name) \\
+ cfish_method(_vtable, \\
+ Lucy_ ## _class_nick ## _ ## _meth_name ## _OFFSET)
+
+static CHY_INLINE cfish_method_t
+cfish_method(const void *vtable, size_t offset) {
+ union { char *cptr; cfish_method_t *fptr; } ptr;
+ ptr.cptr = (char*)vtable + offset;
+ return ptr.fptr[0];
+}
+
+/* Access the function pointer for the given method in the superclass's
+ * vtable. */
+#define LUCY_SUPER_METHOD(_vtable, _class_nick, _meth_name) \\
+ cfish_super_method(_vtable, \\
+ Lucy_ ## _class_nick ## _ ## _meth_name ## _OFFSET)
+
+extern size_t cfish_VTable_offset_of_parent;
+static CHY_INLINE cfish_method_t
+cfish_super_method(const void *vtable, size_t offset) {
+ char *vt_as_char = (char*)vtable;
+ cfish_VTable **parent_ptr
+ = (cfish_VTable**)(vt_as_char + cfish_VTable_offset_of_parent);
+ return cfish_method(*parent_ptr, offset);
+}
+
+/* Return a boolean indicating whether a method has been overridden.
+ */
+#define LUCY_OVERRIDDEN(_self, _class_nick, _meth_name, _micro_name) \\
+ (cfish_method(*((cfish_VTable**)_self), \\
+ Lucy_ ## _class_nick ## _ ## _meth_name ## _OFFSET )\\
+ != (cfish_method_t)lucy_ ## _class_nick ## _ ## _micro_name )
+
+#ifdef CFISH_USE_SHORT_NAMES
+ #define METHOD LUCY_METHOD
+ #define SUPER_METHOD LUCY_SUPER_METHOD
+ #define OVERRIDDEN LUCY_OVERRIDDEN
+#endif
+
+typedef struct cfish_Callback {
+ const char *name;
+ size_t name_len;
+ cfish_method_t func;
+ size_t offset;
+} cfish_Callback;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* BOIL_H */
+
+$self->{footer}
+
+END_STUFF
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Core - Generate core C code for a Clownfish::Hierarchy.
+
+=head1 SYNOPSIS
+
+ my $hierarchy = Clownfish::Hierarchy->new(
+ source => '/path/to/clownfish/files',
+ dest => 'autogen',
+ );
+ $hierarchy->build;
+ my $core_binding = Clownfish::Binding::Core->new(
+ hierarchy => $hierarchy,
+ dest => 'autogen',
+ header => "/* Auto-generated file. */\n",
+ footer => $copyfoot,
+ );
+ my $modified = $core_binding->write_all_modified($modified);
+
+=head1 DESCRIPTION
+
+A Clownfish::Hierarchy describes an abstract specifiction for a class
+hierarchy; Clownfish::Binding::Core is responsible for auto-generating C
+code which implements that specification.
+
+=head1 METHODS
+
+=head2 new
+
+ my $binding = Clownfish::Binding::Core->new(
+ hierarchy => $hierarchy, # required
+ dest => '/path/to/autogen', # required
+ header => $header, # required
+ footer => $footer, # required
+ );
+
+=over
+
+=item * B<hierarchy> - A L<Clownfish::Hierarchy>.
+
+=item * B<dest> - The directory where C output files will be written.
+
+=item * B<header> - Text which will be prepended to each generated C file --
+typically, an "autogenerated file" warning.
+
+=item * B<footer> - Text to be appended to the end of each generated C file --
+typically copyright information.
+
+=back
+
+=head2 write_all_modified
+
+Call C<< $hierarchy->propagate_modified >> to establish which classes do not
+have up-to-date generated .c and .h files, then traverse the hierarchy writing
+all necessary files.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Binding/Core/Aliases.pm b/clownfish/lib/Clownfish/Binding/Core/Aliases.pm
new file mode 100644
index 0000000..c8191cf
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Core/Aliases.pm
@@ -0,0 +1,117 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Core::Aliases;
+
+our %aliases = (
+ cfish_ref_t => 'lucy_ref_t',
+ cfish_method_t => 'lucy_method_t',
+ cfish_method => 'lucy_method',
+ cfish_super_method => 'lucy_super_method',
+
+ cfish_Obj => 'lucy_Obj',
+ CFISH_OBJ => 'LUCY_OBJ',
+ Cfish_Obj_Dump => 'Lucy_Obj_Dump',
+ Cfish_Obj_Get_Class_Name => 'Lucy_Obj_Get_Class_Name',
+ Cfish_Obj_Is_A => 'Lucy_Obj_Is_A',
+ Cfish_Obj_Load => 'Lucy_Obj_Load',
+ Cfish_Obj_To_F64 => 'Lucy_Obj_To_F64',
+ Cfish_Obj_To_I64 => 'Lucy_Obj_To_I64',
+ Cfish_Obj_To_Host => 'Lucy_Obj_To_Host',
+ Cfish_Obj_Dec_RefCount => 'Lucy_Obj_Dec_RefCount',
+ Cfish_Obj_Inc_RefCount => 'Lucy_Obj_Inc_RefCount',
+
+ cfish_ByteBuf => 'lucy_ByteBuf',
+ CFISH_BYTEBUF => 'LUCY_BYTEBUF',
+ Cfish_BB_Get_Size => 'Lucy_BB_Get_Size',
+ Cfish_BB_Get_Buf => 'Lucy_BB_Get_Buf',
+
+ cfish_CharBuf => 'lucy_CharBuf',
+ CFISH_CHARBUF => 'LUCY_CHARBUF',
+ cfish_CB_newf => 'lucy_CB_newf',
+ cfish_CB_new_from_trusted_utf8 => 'lucy_CB_new_from_trusted_utf8',
+ Cfish_CB_Clone => 'Lucy_CB_Clone',
+ cfish_ZombieCharBuf => 'lucy_ZombieCharBuf',
+ CFISH_ZOMBIECHARBUF => 'LUCY_ZOMBIECHARBUF',
+ CFISH_VIEWCHARBUF => 'LUCY_VIEWCHARBUF',
+ cfish_ZCB_size => 'lucy_ZCB_size',
+ cfish_ZCB_wrap_str => 'lucy_ZCB_wrap_str',
+ Cfish_ZCB_Assign_Str => 'Lucy_ZCB_Assign_Str',
+ Cfish_ZCB_Assign_Trusted_Str => 'Lucy_ZCB_Assign_Trusted_Str',
+ Cfish_CB_Get_Ptr8 => 'Lucy_CB_Get_Ptr8',
+ Cfish_CB_Get_Size => 'Lucy_CB_Get_Size',
+
+ CFISH_FLOATNUM => 'LUCY_FLOATNUM',
+ CFISH_INTNUM => 'LUCY_INTNUM',
+ CFISH_INTEGER32 => 'LUCY_INTEGER32',
+ CFISH_INTEGER64 => 'LUCY_INTEGER64',
+ CFISH_FLOAT32 => 'LUCY_FLOAT32',
+ CFISH_FLOAT64 => 'LUCY_FLOAT64',
+
+ CFISH_ERR => 'LUCY_ERR',
+ cfish_Err_new => 'lucy_Err_new',
+ cfish_Err_set_error => 'lucy_Err_set_error',
+ cfish_Err_get_error => 'lucy_Err_get_error',
+
+ cfish_Hash => 'lucy_Hash',
+ CFISH_HASH => 'LUCY_HASH',
+ cfish_Hash_new => 'lucy_Hash_new',
+ Cfish_Hash_Iterate => 'Lucy_Hash_Iterate',
+ Cfish_Hash_Next => 'Lucy_Hash_Next',
+ Cfish_Hash_Fetch_Str => 'Lucy_Hash_Fetch_Str',
+ Cfish_Hash_Store_Str => 'Lucy_Hash_Store_Str',
+ Cfish_Hash_Store => 'Lucy_Hash_Store',
+
+ cfish_VArray => 'lucy_VArray',
+ CFISH_VARRAY => 'LUCY_VARRAY',
+ cfish_VA_new => 'lucy_VA_new',
+ Cfish_VA_Fetch => 'Lucy_VA_Fetch',
+ Cfish_VA_Get_Size => 'Lucy_VA_Get_Size',
+ Cfish_VA_Resize => 'Lucy_VA_Resize',
+ Cfish_VA_Store => 'Lucy_VA_Store',
+
+ cfish_VTable => 'lucy_VTable',
+ CFISH_VTABLE => 'LUCY_VTABLE',
+ cfish_VTable_add_to_registry => 'lucy_VTable_add_to_registry',
+ cfish_VTable_add_alias_to_registry => 'lucy_VTable_add_alias_to_registry',
+ cfish_VTable_offset_of_parent => 'lucy_VTable_offset_of_parent',
+ cfish_VTable_singleton => 'lucy_VTable_singleton',
+ Cfish_VTable_Get_Name => 'Lucy_VTable_Get_Name',
+ Cfish_VTable_Make_Obj => 'Lucy_VTable_Make_Obj',
+
+ cfish_Host_callback => 'lucy_Host_callback',
+ cfish_Host_callback_f64 => 'lucy_Host_callback_f64',
+ cfish_Host_callback_host => 'lucy_Host_callback_host',
+ cfish_Host_callback_i64 => 'lucy_Host_callback_i64',
+ cfish_Host_callback_obj => 'lucy_Host_callback_obj',
+ cfish_Host_callback_str => 'lucy_Host_callback_str',
+
+ CFISH_USE_SHORT_NAMES => 'LUCY_USE_SHORT_NAMES',
+);
+
+sub c_aliases {
+ my $content = "#ifndef CFISH_C_ALIASES\n#define CFISH_C_ALIASES\n\n";
+ for my $alias ( keys %aliases ) {
+ $content .= "#define $alias $aliases{$alias}\n";
+ }
+ $content .= "\n#endif /* CFISH_C_ALIASES */\n\n";
+ return $content;
+}
+
+1;
+
diff --git a/clownfish/lib/Clownfish/Binding/Core/Class.pm b/clownfish/lib/Clownfish/Binding/Core/Class.pm
new file mode 100644
index 0000000..fc52723
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Core/Class.pm
@@ -0,0 +1,407 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Core::Class;
+use Clownfish::Util qw( a_isa_b verify_args );
+use Clownfish::Binding::Core::Method;
+use Clownfish::Binding::Core::Function;
+use File::Spec::Functions qw( catfile );
+
+our %new_PARAMS = ( client => undef, );
+
+sub new {
+ my $either = shift;
+ verify_args( \%new_PARAMS, @_ ) or confess $@;
+ my $self = bless { %new_PARAMS, @_, }, ref($either) || $either;
+
+ # Validate.
+ my $client = $self->{client};
+ confess("Not a Clownfish::Class")
+ unless a_isa_b( $client, "Clownfish::Class" );
+
+ return $self;
+}
+
+sub _full_callbacks_var { shift->{client}->full_vtable_var . '_CALLBACKS' }
+sub _full_name_var { shift->{client}->full_vtable_var . '_CLASS_NAME' }
+sub _short_names_macro { shift->{client}->get_PREFIX . 'USE_SHORT_NAMES' }
+
+# C code defining the ZombieCharBuf which contains the class name for this
+# class.
+sub _name_var_definition {
+ my $self = shift;
+ my $full_var_name = _full_name_var($self);
+ my $class_name = $self->{client}->get_class_name;
+ my $class_name_len = length($class_name);
+ return <<END_STUFF;
+cfish_ZombieCharBuf $full_var_name = {
+ CFISH_ZOMBIECHARBUF,
+ {1}, /* ref.count */
+ "$class_name",
+ $class_name_len,
+ 0
+};
+
+END_STUFF
+}
+
+# Return C code defining the class's VTable.
+sub _vtable_definition {
+ my $self = shift;
+ my $client = $self->{client};
+ my $parent = $client->get_parent;
+ my $methods = $client->methods;
+ my $vt_type = $client->full_vtable_type;
+ my $cnick = $client->get_cnick;
+ my $vtable_var = $client->full_vtable_var;
+ my $struct_sym = $client->full_struct_sym;
+ my $vt = $vtable_var . "_vt";
+ my $name_var = _full_name_var($self);
+ my $cb_var = _full_callbacks_var($self);
+
+ # Create a pointer to the parent class's vtable.
+ my $parent_ref
+ = defined $parent
+ ? $parent->full_vtable_var
+ : "NULL"; # No parent, e.g. Obj or inert classes.
+
+ # Spec functions which implement the methods, casting to quiet compiler.
+ my @implementing_funcs
+ = map { "(cfish_method_t)" . $_->full_func_sym } @$methods;
+ my $method_string = join( ",\n ", @implementing_funcs );
+ my $num_methods = scalar @implementing_funcs;
+
+ return <<END_VTABLE
+
+$vt_type $vt = {
+ CFISH_VTABLE, /* vtable vtable */
+ {1}, /* ref.count */
+ $parent_ref, /* parent */
+ (cfish_CharBuf*)&$name_var,
+ 0, /* flags */
+ NULL, /* "void *x" member reserved for future use */
+ sizeof($struct_sym), /* obj_alloc_size */
+ offsetof(cfish_VTable, methods)
+ + $num_methods * sizeof(cfish_method_t), /* vt_alloc_size */
+ &$cb_var, /* callbacks */
+ {
+ $method_string
+ }
+};
+
+END_VTABLE
+}
+
+# Create the definition for the instantiable object struct.
+sub _struct_definition {
+ my $self = shift;
+ my $struct_sym = $self->{client}->full_struct_sym;
+ my $member_declarations = join( "\n ",
+ map { $_->local_declaration } @{ $self->{client}->member_vars } );
+ return <<END_STRUCT
+struct $struct_sym {
+ $member_declarations
+};
+END_STRUCT
+}
+
+sub to_c_header {
+ my $self = shift;
+ my $client = $self->{client};
+ my $cnick = $client->get_cnick;
+ my $functions = $client->functions;
+ my $methods = $client->methods;
+ my $novel_methods = $client->novel_methods;
+ my $inert_vars = $client->inert_vars;
+ my $vtable_var = $client->full_vtable_var;
+ my $short_vt_var = $client->short_vtable_var;
+ my $short_struct = $client->get_struct_sym;
+ my $full_struct = $client->full_struct_sym;
+ my $c_file_sym = "C_" . uc($full_struct);
+ my $struct_def = _struct_definition($self);
+
+ # If class inherits from something, include the parent class's header.
+ my $parent_include = "";
+ if ( my $parent = $client->get_parent ) {
+ $parent_include = $parent->include_h;
+ $parent_include = qq|#include "$parent_include"|;
+ }
+
+ # Add a C function definition for each method and each function.
+ my $sub_declarations = "";
+ for my $sub ( @$functions, @$novel_methods ) {
+ $sub_declarations
+ .= Clownfish::Binding::Core::Function->func_declaration($sub)
+ . "\n\n";
+ }
+
+ # Declare class (a.k.a. "inert") variables.
+ my $inert_var_defs = "";
+ for my $inert_var (@$inert_vars) {
+ $inert_var_defs .= "extern " . $inert_var->global_c . ";\n";
+ }
+
+ # Declare typedefs for novel methods, to ease casting.
+ my $method_typedefs = '';
+ for my $method (@$novel_methods) {
+ $method_typedefs
+ .= Clownfish::Binding::Core::Method->typedef_dec($method) . "\n";
+ }
+
+ # Define method invocation syntax.
+ my $method_defs = '';
+ for my $method (@$methods) {
+ $method_defs .= Clownfish::Binding::Core::Method->method_def(
+ method => $method,
+ class => $self->{client},
+ ) . "\n";
+ }
+
+ # Declare the virtual table singleton object.
+ my $vt_type = $self->{client}->full_vtable_type;
+ my $vt = "extern struct $vt_type ${vtable_var}_vt;";
+ my $vtable_object
+ = "#define $vtable_var ((cfish_VTable*)&${vtable_var}_vt)";
+ my $num_methods = scalar @$methods;
+
+ # Declare cfish_Callback objects.
+ my $callback_declarations = "";
+ for my $method (@$novel_methods) {
+ next unless $method->public || $method->abstract;
+ $callback_declarations
+ .= Clownfish::Binding::Core::Method->callback_dec($method);
+ }
+
+ # Define short names.
+ my $short_names = '';
+ my $short_names_macro = _short_names_macro($self);
+ for my $function (@$functions) {
+ my $short_func_sym = $function->short_sym;
+ my $full_func_sym = $function->full_sym;
+ $short_names .= " #define $short_func_sym $full_func_sym\n";
+ }
+ for my $inert_var (@$inert_vars) {
+ my $short_sym = $inert_var->short_sym;
+ my $full_sym = $inert_var->full_sym;
+ $short_names .= " #define $short_sym $full_sym\n";
+ }
+ if ( !$client->inert ) {
+ for my $method (@$novel_methods) {
+ if ( !$method->isa("Clownfish::Method::Overridden") ) {
+ my $short_typedef = $method->short_typedef;
+ my $full_typedef = $method->full_typedef;
+ $short_names .= " #define $short_typedef $full_typedef\n";
+ }
+ my $short_func_sym = $method->short_func_sym;
+ my $full_func_sym = $method->full_func_sym;
+ $short_names .= " #define $short_func_sym $full_func_sym\n";
+ }
+ for my $method (@$methods) {
+ my $short_method_sym = $method->short_method_sym($cnick);
+ my $full_method_sym = $method->full_method_sym($cnick);
+ $short_names .= " #define $short_method_sym $full_method_sym\n";
+ }
+ }
+
+ # Make the spacing in the file a little more elegant.
+ s/\s+$// for ( $method_typedefs, $method_defs, $short_names );
+
+ # Inert classes only output inert functions and member vars.
+ if ( $client->inert ) {
+ return <<END_INERT
+#include "charmony.h"
+#include "boil.h"
+$parent_include
+
+$inert_var_defs
+
+$sub_declarations
+
+#ifdef $short_names_macro
+$short_names
+#endif /* $short_names_macro */
+
+END_INERT
+ }
+
+ # Instantiable classes get everything.
+ return <<END_STUFF;
+
+#include "charmony.h"
+#include "boil.h"
+$parent_include
+
+#ifdef $c_file_sym
+$struct_def
+#endif /* $c_file_sym */
+
+$inert_var_defs
+
+$sub_declarations
+$callback_declarations
+
+$method_typedefs
+
+$method_defs
+
+typedef struct $vt_type {
+ cfish_VTable *vtable;
+ cfish_ref_t ref;
+ cfish_VTable *parent;
+ cfish_CharBuf *name;
+ uint32_t flags;
+ void *x;
+ size_t obj_alloc_size;
+ size_t vt_alloc_size;
+ void *callbacks;
+ cfish_method_t methods[$num_methods];
+} $vt_type;
+$vt
+$vtable_object
+
+#ifdef $short_names_macro
+ #define $short_struct $full_struct
+ #define $short_vt_var $vtable_var
+$short_names
+#endif /* $short_names_macro */
+
+END_STUFF
+}
+
+sub to_c {
+ my $self = shift;
+ my $client = $self->{client};
+
+ return $client->get_autocode if $client->inert;
+
+ my $include_h = $client->include_h;
+ my $autocode = $client->get_autocode;
+ my $offsets = '';
+ my $abstract_funcs = '';
+ my $callback_funcs = '';
+ my $callbacks = '';
+ my $vt_type = $client->full_vtable_type;
+ my $meth_num = 0;
+ my $cnick = $client->get_cnick;
+ my $class_name_def = _name_var_definition($self);
+ my $vtable_def = _vtable_definition($self);
+ my @class_callbacks;
+
+ # Prepare to identify novel methods.
+ my %novel = map { ( $_->micro_sym => $_ ) } @{ $client->novel_methods };
+
+ for my $method ( @{ $client->methods } ) {
+ my $var_name = $method->full_offset_sym($cnick);
+
+ # Create offset in bytes for the method from the top of the VTable
+ # object.
+ my $offset = "(offsetof($vt_type, methods)"
+ . " + $meth_num * sizeof(cfish_method_t))";
+ $offsets .= "size_t $var_name = $offset;\n";
+
+ # Create a default implementation for abstract methods.
+ if ( $method->abstract ) {
+ if ( $novel{ $method->micro_sym } ) {
+ $callback_funcs
+ .= Clownfish::Binding::Core::Method->abstract_method_def(
+ $method)
+ . "\n";
+ }
+ }
+
+ # Define callbacks for methods that can be overridden via the
+ # host.
+ if ( $method->public or $method->abstract ) {
+ my $callback_sym = $method->full_callback_sym;
+ if ( $novel{ $method->micro_sym } ) {
+ $callback_funcs
+ .= Clownfish::Binding::Core::Method->callback_def($method)
+ . "\n";
+ $callbacks
+ .= Clownfish::Binding::Core::Method->callback_obj_def(
+ method => $method,
+ offset => $offset,
+ );
+ }
+ push @class_callbacks, "&$callback_sym";
+ }
+ $meth_num++;
+ }
+
+ # Create a NULL-terminated array of cfish_Callback vars. Since C89
+ # doesn't allow us to initialize a pointer to an anonymous array inside a
+ # global struct, we have to give it a real symbol and then store a pointer
+ # to that symbol inside the VTable struct.
+ my $callbacks_var = _full_callbacks_var($self);
+ $callbacks .= "cfish_Callback *$callbacks_var" . "[] = {\n ";
+ $callbacks .= join( ",\n ", @class_callbacks, "NULL" );
+ $callbacks .= "\n};\n";
+
+ return <<END_STUFF;
+#include "$include_h"
+
+$offsets
+$callback_funcs
+$callbacks
+$class_name_def
+$vtable_def
+$autocode
+
+END_STUFF
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Core::Class - Generate core C code for a class.
+
+=head1 DESCRIPTION
+
+Clownfish::Class is an abstract specification for a class. This module
+autogenerates the C code with implements that specification.
+
+=head1 METHODS
+
+=head2 new
+
+ my $class_binding = Clownfish::Binding::Core::Class->new(
+ client => $class,
+ );
+
+=over
+
+=item * B<client> - A L<Clownfish::Class>.
+
+=back
+
+=head2 to_c_header
+
+Return the .h file which contains autogenerated C code defining the class's
+interface: all method invocation functions, etc...
+
+=head2 to_c
+
+Return the .c file which contains autogenerated C code necessary for the class
+to function properly.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Core/File.pm b/clownfish/lib/Clownfish/Binding/Core/File.pm
new file mode 100644
index 0000000..7a31509
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Core/File.pm
@@ -0,0 +1,214 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Core::File;
+use Clownfish::Util qw( a_isa_b verify_args );
+use Clownfish::Binding::Core::Class;
+use File::Spec::Functions qw( catfile splitpath );
+use File::Path qw( mkpath );
+use Scalar::Util qw( blessed );
+use Fcntl;
+use Carp;
+
+my %write_h_PARAMS = (
+ file => undef,
+ dest => undef,
+ header => undef,
+ footer => undef,
+);
+
+sub write_h {
+ my ( undef, %args ) = @_;
+ verify_args( \%write_h_PARAMS, %args ) or confess $@;
+ my $file = $args{file};
+ confess("Not a Clownfish::File")
+ unless a_isa_b( $file, "Clownfish::File" );
+ my $h_path = $file->h_path( $args{dest} );
+
+ # Unlink then open file.
+ my ( undef, $out_dir, undef ) = splitpath($h_path);
+ mkpath $out_dir unless -d $out_dir;
+ confess("Can't make dir '$out_dir'") unless -d $out_dir;
+ unlink $h_path;
+ sysopen( my $fh, $h_path, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$h_path' for writing");
+
+ # Create the include-guard strings.
+ my $include_guard_start = $file->guard_start;
+ my $include_guard_close = $file->guard_close;
+
+ # Aggregate block content.
+ my $content = "";
+ for my $block ( @{ $file->blocks } ) {
+ if ( a_isa_b( $block, 'Clownfish::Parcel' ) ) { }
+ elsif ( a_isa_b( $block, 'Clownfish::Class' ) ) {
+ my $class_binding
+ = Clownfish::Binding::Core::Class->new( client => $block, );
+ $content .= $class_binding->to_c_header . "\n";
+ }
+ elsif ( a_isa_b( $block, 'Clownfish::CBlock' ) ) {
+ $content .= $block->get_contents . "\n";
+ }
+ else {
+ confess("Invalid block: $block");
+ }
+ }
+
+ print $fh <<END_STUFF;
+$args{header}
+
+$include_guard_start
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+$content
+
+#ifdef __cplusplus
+}
+#endif
+
+$include_guard_close
+
+$args{footer}
+
+END_STUFF
+}
+
+my %write_c_PARAMS = (
+ file => undef,
+ dest => undef,
+ header => undef,
+ footer => undef,
+);
+
+sub write_c {
+ my ( undef, %args ) = @_;
+ verify_args( \%write_h_PARAMS, %args ) or confess $@;
+ my $file = $args{file};
+ confess("Not a Clownfish::File")
+ unless a_isa_b( $file, "Clownfish::File" );
+ my $c_path = $file->c_path( $args{dest} );
+
+ # Unlink then open file.
+ my ( undef, $out_dir, undef ) = splitpath($c_path);
+ mkpath $out_dir unless -d $out_dir;
+ confess("Can't make dir '$out_dir'") unless -d $out_dir;
+ unlink $c_path;
+ sysopen( my $fh, $c_path, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$c_path' for writing");
+
+ # Aggregate content.
+ my $content = "";
+ my $c_file_syms = "";
+ for my $block ( @{ $file->blocks } ) {
+ if ( blessed($block) ) {
+ if ( $block->isa('Clownfish::Class') ) {
+ my $bound
+ = Clownfish::Binding::Core::Class->new( client => $block,
+ );
+ $content .= $bound->to_c . "\n";
+ my $c_file_sym = "C_" . uc( $block->full_struct_sym );
+ $c_file_syms .= "#define $c_file_sym\n";
+ }
+ }
+ }
+
+ print $fh <<END_STUFF;
+$args{header}
+
+$c_file_syms
+#define C_LUCY_VTABLE
+#define C_LUCY_ZOMBIECHARBUF
+#include "boil.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/Hash.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Object/VArray.h"
+
+$content
+
+$args{footer}
+
+END_STUFF
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Core::File - Generate core C code for a Clownfish file.
+
+=head1 DESCRIPTION
+
+This module is the companion to Clownfish::File, generating the C code
+needed to implement the file's specification.
+
+There is a one-to-one mapping between Clownfish header files and autogenerated
+.h and .c files. If Foo.cfh includes both Foo and Foo::FooJr, then it is
+necessary to pound-include "Foo.h" in order to get FooJr's interface -- not
+"Foo/FooJr.h", which won't exist.
+
+=head1 CLASS METHODS
+
+=head2 write_h
+
+ Clownfish::Binding::Core::File->write_c(
+ file => $file, # required
+ dest => '/path/to/autogen_dir', # required
+ header => "/* Autogenerated file. */\n", # required
+ footer => $copyfoot, # required
+ );
+
+Generate a C header file containing all class declarations and literal C
+blocks.
+
+=over
+
+=item * B<file> - A L<Clownfish::File>.
+
+=item * B<dest> - The directory under which autogenerated files are being
+written.
+
+=item * B<header> - Text which will be prepended to each generated C file --
+typically, an "autogenerated file" warning.
+
+=item * B<footer> - Text to be appended to the end of each generated C file --
+typically copyright information.
+
+=back
+
+=head2 write_h
+
+ Clownfish::Binding::Core::File->write_c(
+ file => $file, # required
+ dest => '/path/to/autogen_dir', # required
+ header => "/* Autogenerated file. */\n", # required
+ footer => $copyfoot, # required
+ );
+
+Generate a C file containing code needed by the class implementations.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Core/Function.pm b/clownfish/lib/Clownfish/Binding/Core/Function.pm
new file mode 100644
index 0000000..436e247
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Core/Function.pm
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Core::Function;
+use Clownfish::Util qw( a_isa_b );
+
+sub func_declaration {
+ my ( undef, $function ) = @_;
+ confess("Not a Function")
+ unless a_isa_b( $function, "Clownfish::Function" );
+ my $return_type = $function->get_return_type;
+ my $param_list = $function->get_param_list;
+ my $dec = $function->inline ? 'static CHY_INLINE ' : '';
+ $dec .= $return_type->to_c . "\n";
+ $dec .= $function->full_func_sym;
+ $dec .= "(" . $param_list->to_c . ");";
+ return $dec;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Core::Function - Generate core C code for a function.
+
+=head1 CLASS METHODS
+
+=head2 func_declaration
+
+ my $declaration
+ = Clownfish::Binding::Core::Function->func_declaration($function);
+
+Return C code declaring the function's C implementation.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Core/Method.pm b/clownfish/lib/Clownfish/Binding/Core/Method.pm
new file mode 100644
index 0000000..6961f5e
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Core/Method.pm
@@ -0,0 +1,388 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Core::Method;
+use Clownfish::Util qw( a_isa_b );
+use Carp;
+
+sub method_def {
+ my ( undef, %args ) = @_;
+ my ( $method, $class ) = @args{qw( method class )};
+ confess("Not a Method")
+ unless a_isa_b( $method, "Clownfish::Method" );
+ confess("Not a Class")
+ unless a_isa_b( $class, "Clownfish::Class" );
+ if ( $method->final ) {
+ return _final_method_def( $method, $class );
+ }
+ else {
+ return _virtual_method_def( $method, $class );
+ }
+}
+
+sub _virtual_method_def {
+ my ( $method, $class ) = @_;
+ my $cnick = $class->get_cnick;
+ my $param_list = $method->get_param_list;
+ my $invoker_struct = $class->full_struct_sym;
+ my $common_struct = $method->self_type->get_specifier;
+ my $full_method_sym = $method->full_method_sym($cnick);
+ my $full_offset_sym = $method->full_offset_sym($cnick);
+ my $typedef = $method->full_typedef;
+ my $arg_names = $param_list->name_list;
+ $arg_names =~ s/\s*\w+/self/;
+
+ # Prepare the parameter list for the inline function.
+ my $params = $param_list->to_c;
+ $params =~ s/^.*?\*\s*\w+/const $invoker_struct *self/
+ or confess("no match: $params");
+
+ # Prepare a return statement... or not.
+ my $return_type = $method->get_return_type->to_c;
+ my $maybe_return = $method->get_return_type->is_void ? '' : 'return ';
+
+ return <<END_STUFF;
+extern size_t $full_offset_sym;
+static CHY_INLINE $return_type
+$full_method_sym($params) {
+ char *const method_address = *(char**)self + $full_offset_sym;
+ const $typedef method = *(($typedef*)method_address);
+ ${maybe_return}method(($common_struct*)$arg_names);
+}
+END_STUFF
+}
+
+# Create a macro definition that aliases to a function name directly, since
+# this method may not be overridden.
+sub _final_method_def {
+ my ( $method, $class ) = @_;
+ my $cnick = $class->get_cnick;
+ my $macro_sym = $method->get_macro_sym;
+ my $self_type = $method->self_type->to_c;
+ my $full_method_sym = $method->full_method_sym($cnick);
+ my $full_func_sym = $method->full_func_sym;
+ my $arg_names = $method->get_param_list->name_list;
+
+ return <<END_STUFF;
+#define $full_method_sym($arg_names) \\
+ $full_func_sym(($self_type)$arg_names)
+END_STUFF
+}
+
+sub typedef_dec {
+ my ( undef, $method ) = @_;
+ my $params = $method->get_param_list->to_c;
+ my $return_type = $method->get_return_type->to_c;
+ my $typedef = $method->full_typedef;
+ return <<END_STUFF;
+typedef $return_type
+(*$typedef)($params);
+END_STUFF
+}
+
+sub callback_dec {
+ my ( undef, $method ) = @_;
+ my $callback_sym = $method->full_callback_sym;
+ return qq|extern cfish_Callback $callback_sym;\n|;
+}
+
+sub callback_obj_def {
+ my ( undef, %args ) = @_;
+ my $method = $args{method};
+ my $offset = $args{offset};
+ my $macro_sym = $method->get_macro_sym;
+ my $len = length($macro_sym);
+ my $func_sym = $method->full_override_sym;
+ my $callback_sym = $method->full_callback_sym;
+ return qq|cfish_Callback $callback_sym = |
+ . qq|{"$macro_sym", $len, (cfish_method_t)$func_sym, $offset};\n|;
+}
+
+sub callback_def {
+ my ( undef, $method ) = @_;
+ my $return_type = $method->get_return_type;
+ my $params = _callback_params($method);
+ if ( !$params ) {
+ # Can't map vars, because there's at least one type in the argument
+ # list we don't yet support. Return a callback wrapper that throws an
+ # error error.
+ return _invalid_callback_def( $method, $params );
+ }
+ elsif ( $return_type->is_void ) {
+ return _void_callback_def( $method, $params );
+ }
+ elsif ( $return_type->is_object ) {
+ return _obj_callback_def( $method, $params );
+ }
+ else {
+ return _primitive_callback_def( $method, $params );
+ }
+}
+
+# Return a string which maps arguments to various arg wrappers conforming
+# to Host's callback interface. For instance, (int32_t foo, Obj *bar)
+# produces the following:
+#
+# CFISH_ARG_I32("foo", foo),
+# CFISH_ARG_OBJ("bar", bar)
+#
+sub _callback_params {
+ my $method = shift;
+ my $micro_sym = $method->micro_sym;
+ my $param_list = $method->get_param_list;
+ my $num_params = $param_list->num_vars - 1;
+ my $arg_vars = $param_list->get_variables;
+ my @params;
+
+ # Iterate over arguments, mapping them to various arg wrappers which
+ # conform to Host's callback interface.
+ for my $var ( @$arg_vars[ 1 .. $#$arg_vars ] ) {
+ my $name = $var->micro_sym;
+ my $type = $var->get_type;
+ my $param;
+ if ( $type->is_string_type ) {
+ $param = qq|CFISH_ARG_STR("$name", $name)|;
+ }
+ elsif ( $type->is_object ) {
+ $param = qq|CFISH_ARG_OBJ("$name", $name)|;
+ }
+ elsif ( $type->is_integer ) {
+ my $width = $type->get_width;
+ if ($width) {
+ if ( $width <= 4 ) {
+ $param = qq|CFISH_ARG_I32("$name", $name)|;
+ }
+ else {
+ $param = qq|CFISH_ARG_I64("$name", $name)|;
+ }
+ }
+ else {
+ my $c_type = $type->to_c;
+ $param = qq|CFISH_ARG_I($c_type, "$name", $name)|;
+ }
+ }
+ elsif ( $type->is_floating ) {
+ $param = qq|CFISH_ARG_F64("$name", $name)|;
+ }
+ else {
+ # Can't map variable type. Signal to caller.
+ return undef;
+ }
+ push @params, $param;
+ }
+ return join( ', ', 'self', qq|"$micro_sym"|, $num_params, @params );
+}
+
+# Return a function which throws a runtime error indicating which variable
+# couldn't be mapped. TODO: it would be better to resolve all these cases at
+# compile-time.
+sub _invalid_callback_def {
+ my ( $method, $callback_params ) = @_;
+ my $full_method_sym
+ = $method->full_method_sym( $method->get_class_cnick );
+ my $override_sym = $method->full_override_sym;
+ my $params = $method->get_param_list->to_c;
+ my $unused = '';
+ for my $var ( @{ $method->get_param_list->get_variables } ) {
+ my $var_name = $var->micro_sym;
+ $unused .= "CHY_UNUSED_VAR($var_name); ";
+ }
+ return <<END_CALLBACK_DEF;
+void
+$override_sym($params) {
+ $unused;
+ CFISH_THROW(CFISH_ERR, "Can't override $full_method_sym via binding");
+}
+END_CALLBACK_DEF
+}
+
+# Create a callback for a method which operates in a void context.
+sub _void_callback_def {
+ my ( $method, $callback_params ) = @_;
+ my $override_sym = $method->full_override_sym;
+ my $params = $method->get_param_list->to_c;
+ return <<END_CALLBACK_DEF;
+void
+$override_sym($params) {
+ cfish_Host_callback($callback_params);
+}
+END_CALLBACK_DEF
+}
+
+# Create a callback which returns a primitive type.
+sub _primitive_callback_def {
+ my ( $method, $callback_params ) = @_;
+ my $override_sym = $method->full_override_sym;
+ my $params = $method->get_param_list->to_c;
+ my $return_type = $method->get_return_type;
+ my $return_type_str = $return_type->to_c;
+ my $nat_func
+ = $return_type->is_floating ? "cfish_Host_callback_f64"
+ : $return_type->is_integer ? "cfish_Host_callback_i64"
+ : $return_type_str eq 'void*' ? "cfish_Host_callback_host"
+ : confess("unrecognized type: $return_type_str");
+ return <<END_CALLBACK_DEF;
+$return_type_str
+$override_sym($params) {
+ return ($return_type_str)$nat_func($callback_params);
+}
+END_CALLBACK_DEF
+}
+
+# Create a callback which returns an object type -- either a generic object or
+# a string.
+sub _obj_callback_def {
+ my ( $method, $callback_params ) = @_;
+ my $override_sym = $method->full_override_sym;
+ my $params = $method->get_param_list->to_c;
+ my $return_type = $method->get_return_type;
+ my $return_type_str = $return_type->to_c;
+ my $cb_func_name
+ = $return_type->is_string_type
+ ? "cfish_Host_callback_str"
+ : "cfish_Host_callback_obj";
+
+ my $nullable_check = "";
+ if ( !$return_type->nullable ) {
+ my $macro_sym = $method->get_macro_sym;
+ $nullable_check
+ = qq|if (!retval) { CFISH_THROW(CFISH_ERR, |
+ . qq|"$macro_sym() for class '%o' cannot return NULL", |
+ . qq|Cfish_Obj_Get_Class_Name((cfish_Obj*)self)); }\n |;
+ }
+
+ my $decrement = "";
+ if ( !$return_type->incremented ) {
+ $decrement = "LUCY_DECREF(retval);\n ";
+ }
+
+ return <<END_CALLBACK_DEF;
+$return_type_str
+$override_sym($params) {
+ $return_type_str retval = ($return_type_str)$cb_func_name($callback_params);
+ ${nullable_check}${decrement}return retval;
+}
+END_CALLBACK_DEF
+}
+
+# Create a function which throws a runtime error indicating that a method is
+# abstract. This serves as the implementation for methods which are
+# declared as "abstract" in a Clownfish header file.
+sub abstract_method_def {
+ my ( undef, $method ) = @_;
+ my $params = $method->get_param_list->to_c;
+ my $full_func_sym = $method->full_func_sym;
+ my $vtable = uc( $method->self_type->get_specifier );
+ my $return_type = $method->get_return_type;
+ my $return_type_str = $return_type->to_c;
+ my $macro_sym = $method->get_macro_sym;
+
+ # Build list of unused params and create an unreachable return statement
+ # if necessary, in order to thwart compiler warnings.
+ my $param_vars = $method->get_param_list->get_variables;
+ my $unused = "";
+ for ( my $i = 1; $i < @$param_vars; $i++ ) {
+ my $var_name = $param_vars->[$i]->micro_sym;
+ $unused .= "\n CHY_UNUSED_VAR($var_name);";
+ }
+ my $ret_statement = '';
+ if ( !$return_type->is_void ) {
+ $ret_statement = "\n CHY_UNREACHABLE_RETURN($return_type_str);";
+ }
+
+ return <<END_ABSTRACT_DEF;
+$return_type_str
+$full_func_sym($params) {
+ cfish_CharBuf *klass = self ? Cfish_Obj_Get_Class_Name((cfish_Obj*)self) : $vtable->name;$unused
+ CFISH_THROW(CFISH_ERR, "Abstract method '$macro_sym' not defined by %o", klass);$ret_statement
+}
+END_ABSTRACT_DEF
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Core::Method - Generate core C code for a method.
+
+=head1 DESCRIPTION
+
+Clownfish::Method is an abstract specification; this class generates C code
+which implements the specification.
+
+=head1 METHODS
+
+=head2 method_def
+
+ my $c_code = Clownfish::Binding::Core::Method->method_def(
+ method => $method,
+ $class => $class,
+ );
+
+Return C code for the static inline vtable method invocation function.
+
+=over
+
+=item * B<method> - A L<Clownfish::Method>.
+
+=item * B<class> - The L<Clownfish::Class> which will be invoking the method -
+LobsterClaw needs its own method invocation function even if the method was
+defined in Claw.
+
+=back
+
+=head2 typedef_dec
+
+ my $c_code = Clownfish::Binding::Core::Method->typedef_dec($method);
+
+Return C code expressing a typedef declaration for the method.
+
+=head2 callback_dec
+
+ my $c_code = Clownfish::Binding::Core::Method->callback_dec($method);
+
+Return C code declaring the Callback object for this method.
+
+=head2 callback_obj_def
+
+ my $c_code
+ = Clownfish::Binding::Core::Method->callback_obj_def($method);
+
+Return C code defining the Callback object for this method, which stores
+introspection data and a pointer to the callback function.
+
+=head2 callback_def
+
+ my $c_code = Clownfish::Binding::Core::Method->callback_def($method);
+
+Return C code implementing a callback to the Host for this method. This code
+is used when a Host method has overridden a method in a Clownfish class.
+
+=head2 abstract_method_def
+
+ my $c_code
+ = Clownfish::Binding::Core::Method->abstract_method_def($method);
+
+Return C code implementing a version of the method which throws an "abstract
+method" error at runtime.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Perl.pm b/clownfish/lib/Clownfish/Binding/Perl.pm
new file mode 100644
index 0000000..255716d
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Perl.pm
@@ -0,0 +1,531 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Perl;
+
+use Clownfish::Hierarchy;
+use Carp;
+use File::Spec::Functions qw( catfile );
+use Fcntl;
+
+use Clownfish::Parcel;
+use Clownfish::Class;
+use Clownfish::Function;
+use Clownfish::Method;
+use Clownfish::Variable;
+use Clownfish::Util qw( verify_args a_isa_b write_if_changed );
+use Clownfish::Binding::Perl::Class;
+use Clownfish::Binding::Perl::Method;
+use Clownfish::Binding::Perl::Constructor;
+
+our %new_PARAMS = (
+ parcel => undef,
+ hierarchy => undef,
+ lib_dir => undef,
+ boot_class => undef,
+ header => undef,
+ footer => undef,
+);
+
+sub new {
+ my $either = shift;
+ verify_args( \%new_PARAMS, @_ ) or confess $@;
+ my $self = bless { %new_PARAMS, @_, }, ref($either) || $either;
+ if ( !a_isa_b( $self->{parcel}, 'Clownfish::Parcel' ) ) {
+ $self->{parcel}
+ = Clownfish::Parcel->singleton( name => $self->{parcel} );
+ }
+ my $parcel = $self->{parcel};
+ for ( keys %new_PARAMS ) {
+ confess("$_ is mandatory") unless defined $self->{$_};
+ }
+
+ # Derive filenames.
+ my $lib = $self->{lib_dir};
+ my $dest_dir = $self->{hierarchy}->get_dest;
+ my @file_components = split( '::', $self->{boot_class} );
+ my @xs_file_components = @file_components;
+ $xs_file_components[-1] .= '.xs';
+ $self->{xs_path} = catfile( $lib, @xs_file_components );
+
+ $self->{pm_path} = catfile( $lib, @file_components, 'Autobinding.pm' );
+ $self->{boot_h_file} = $parcel->get_prefix . "boot.h";
+ $self->{boot_c_file} = $parcel->get_prefix . "boot.c";
+ $self->{boot_h_path} = catfile( $dest_dir, $self->{boot_h_file} );
+ $self->{boot_c_path} = catfile( $dest_dir, $self->{boot_c_file} );
+
+ # Derive the name of the bootstrap function.
+ $self->{boot_func}
+ = $parcel->get_prefix . $self->{boot_class} . '_bootstrap';
+ $self->{boot_func} =~ s/\W/_/g;
+
+ return $self;
+}
+
+sub write_bindings {
+ my $self = shift;
+ my $ordered = $self->{hierarchy}->ordered_classes;
+ my $registry = Clownfish::Binding::Perl::Class->registry;
+ my $hand_rolled_xs = "";
+ my $generated_xs = "";
+ my $xs = "";
+ my @xsubs;
+
+ # Build up a roster of all requested bindings.
+ my %has_constructors;
+ my %has_methods;
+ my %has_xs_code;
+ while ( my ( $class_name, $class_binding ) = each %$registry ) {
+ $has_constructors{$class_name} = 1
+ if $class_binding->get_bind_constructors;
+ $has_methods{$class_name} = 1
+ if $class_binding->get_bind_methods;
+ $has_xs_code{$class_name} = 1
+ if $class_binding->get_xs_code;
+ }
+
+ # Pound-includes for generated headers.
+ for my $class (@$ordered) {
+ my $include_h = $class->include_h;
+ $generated_xs .= qq|#include "$include_h"\n|;
+ }
+ $generated_xs .= "\n";
+
+ # Constructors.
+ for my $class (@$ordered) {
+ my $class_name = $class->get_class_name;
+ next unless delete $has_constructors{$class_name};
+ my $class_binding = $registry->{$class_name};
+ my @bound = $class_binding->constructor_bindings;
+ $generated_xs .= $_->xsub_def . "\n" for @bound;
+ push @xsubs, @bound;
+ }
+
+ # Methods.
+ for my $class (@$ordered) {
+ my $class_name = $class->get_class_name;
+ next unless delete $has_methods{$class_name};
+ my $class_binding = $registry->{$class_name};
+ my @bound = $class_binding->method_bindings;
+ $generated_xs .= $_->xsub_def . "\n" for @bound;
+ push @xsubs, @bound;
+ }
+
+ # Hand-rolled XS.
+ for my $class_name ( keys %has_xs_code ) {
+ my $class_binding = $registry->{$class_name};
+ $hand_rolled_xs .= $class_binding->get_xs_code . "\n";
+ }
+ %has_xs_code = ();
+
+ # Verify that all binding specs were processed.
+ my @leftover_ctor = keys %has_constructors;
+ if (@leftover_ctor) {
+ confess( "Constructor bindings spec'd for non-existant classes: "
+ . "'@leftover_ctor'" );
+ }
+ my @leftover_bound = keys %has_methods;
+ if (@leftover_bound) {
+ confess( "Method bindings spec'd for non-existant classes: "
+ . "'@leftover_bound'" );
+ }
+ my @leftover_xs = keys %has_xs_code;
+ if (@leftover_xs) {
+ confess( "Hand-rolled XS spec'd for non-existant classes: "
+ . "'@leftover_xs'" );
+ }
+
+ # Build up code for booting XSUBs at module load time.
+ my @xs_init_lines;
+ for my $xsub (@xsubs) {
+ my $c_name = $xsub->c_name;
+ my $perl_name = $xsub->perl_name;
+ push @xs_init_lines, qq|newXS("$perl_name", $c_name, file);|;
+ }
+ my $xs_init = join( "\n ", @xs_init_lines );
+
+ # Params hashes for arg checking of XSUBs that take labeled params.
+ my @params_hash_defs = grep {defined} map { $_->params_hash_def } @xsubs;
+ my $params_hash_defs = join( "\n", @params_hash_defs );
+
+ # Write out if there have been any changes.
+ my $xs_file_contents = $self->_xs_file_contents( $generated_xs, $xs_init,
+ $hand_rolled_xs );
+ my $pm_file_contents = $self->_pm_file_contents($params_hash_defs);
+ write_if_changed( $self->{xs_path}, $xs_file_contents );
+ write_if_changed( $self->{pm_path}, $pm_file_contents );
+}
+
+sub _xs_file_contents {
+ my ( $self, $generated_xs, $xs_init, $hand_rolled_xs ) = @_;
+ return <<END_STUFF;
+/* DO NOT EDIT!!!! This is an auto-generated file. */
+
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "XSBind.h"
+#include "boil.h"
+#include "$self->{boot_h_file}"
+
+#include "Lucy/Object/Host.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/StringHelper.h"
+
+#include "Charmonizer/Test.h"
+#include "Charmonizer/Test/AllTests.h"
+
+$generated_xs
+
+MODULE = Lucy PACKAGE = Lucy::Autobinding
+
+void
+init_autobindings()
+PPCODE:
+{
+ char* file = __FILE__;
+ CHY_UNUSED_VAR(cv);
+ CHY_UNUSED_VAR(items); $xs_init
+}
+
+$hand_rolled_xs
+
+END_STUFF
+}
+
+sub _pm_file_contents {
+ my ( $self, $params_hash_defs ) = @_;
+ return <<END_STUFF;
+# DO NOT EDIT!!!! This is an auto-generated file.
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Lucy::Autobinding;
+
+init_autobindings();
+
+$params_hash_defs
+
+1;
+
+END_STUFF
+}
+
+sub prepare_pod {
+ my $self = shift;
+ my $lib_dir = $self->{lib_dir};
+ my $ordered = $self->{hierarchy}->ordered_classes;
+ my @files_written;
+ my %has_pod;
+ my %modified;
+
+ my $registry = Clownfish::Binding::Perl::Class->registry;
+ $has_pod{ $_->get_class_name } = 1
+ for grep { $_->get_make_pod } values %$registry;
+
+ for my $class (@$ordered) {
+ my $class_name = $class->get_class_name;
+ my $class_binding = $registry->{$class_name} or next;
+ next unless delete $has_pod{$class_name};
+ my $pod = $class_binding->create_pod;
+ confess("Failed to generate POD for $class_name") unless $pod;
+
+ # Compare against existing file; rewrite if changed.
+ my $pod_file_path
+ = catfile( $lib_dir, split( '::', $class_name ) ) . ".pod";
+ my $existing = "";
+ if ( -e $pod_file_path ) {
+ open( my $pod_fh, "<", $pod_file_path )
+ or confess("Can't open '$pod_file_path': $!");
+ $existing = do { local $/; <$pod_fh> };
+ }
+ if ( $pod ne $existing ) {
+ $modified{$pod_file_path} = $pod;
+ }
+ }
+ my @leftover = keys %has_pod;
+ confess("Couldn't match pod to class for '@leftover'") if @leftover;
+
+ return \%modified;
+}
+
+sub write_boot {
+ my $self = shift;
+ $self->_write_boot_h;
+ $self->_write_boot_c;
+}
+
+sub _write_boot_h {
+ my $self = shift;
+ my $hierarchy = $self->{hierarchy};
+ my $filepath = catfile( $hierarchy->get_dest, $self->{boot_h_file} );
+ my $guard = uc("$self->{boot_class}_BOOT");
+ $guard =~ s/\W+/_/g;
+
+ unlink $filepath;
+ sysopen( my $fh, $filepath, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$filepath': $!");
+ print $fh <<END_STUFF;
+$self->{header}
+
+#ifndef $guard
+#define $guard 1
+
+void
+$self->{boot_func}();
+
+#endif /* $guard */
+
+$self->{footer}
+END_STUFF
+}
+
+my %ks_compat = (
+ 'Lucy::Plan::Schema' =>
+ [qw( KinoSearch::Plan::Schema KinoSearch::Schema )],
+ 'Lucy::Plan::FieldType' =>
+ [qw( KinoSearch::Plan::FieldType KinoSearch::FieldType )],
+ 'Lucy::Plan::FullTextType' => [
+ qw( KinoSearch::Plan::FullTextType KinoSearch::FieldType::FullTextType )
+ ],
+ 'Lucy::Plan::StringType' => [
+ qw( KinoSearch::Plan::StringType KinoSearch::FieldType::StringType )],
+ 'Lucy::Plan::BlobType' =>
+ [qw( KinoSearch::Plan::BlobType KinoSearch::FieldType::BlobType )],
+ 'Lucy::Analysis::PolyAnalyzer' =>
+ [qw( KinoSearch::Analysis::PolyAnalyzer )],
+ 'Lucy::Analysis::RegexTokenizer' =>
+ [qw( KinoSearch::Analysis::Tokenizer )],
+ 'Lucy::Analysis::CaseFolder' => [
+ qw( KinoSearch::Analysis::CaseFolder KinoSearch::Analysis::LCNormalizer )
+ ],
+ 'Lucy::Analysis::SnowballStopFilter' =>
+ [qw( KinoSearch::Analysis::Stopalizer )],
+ 'Lucy::Analysis::SnowballStemmer' =>
+ [qw( KinoSearch::Analysis::Stemmer )],
+);
+
+sub _write_boot_c {
+ my $self = shift;
+ my $hierarchy = $self->{hierarchy};
+ my $ordered = $hierarchy->ordered_classes;
+ my $num_classes = scalar @$ordered;
+ my $pound_includes = "";
+ my $registrations = "";
+ my $isa_pushes = "";
+
+ for my $class (@$ordered) {
+ my $include_h = $class->include_h;
+ $pound_includes .= qq|#include "$include_h"\n|;
+ next if $class->inert;
+
+ # Ignore return value from VTable_add_to_registry, since it's OK if
+ # multiple threads contend for adding these permanent VTables and some
+ # fail.
+ $registrations
+ .= qq| cfish_VTable_add_to_registry(|
+ . $class->full_vtable_var
+ . qq|);\n|;
+
+ # Add aliases for selected KinoSearch classes which allow old indexes
+ # to be read.
+ my $class_name = $class->get_class_name;
+ my $aliases = $ks_compat{$class_name};
+ if ($aliases) {
+ my $vtable_var = $class->full_vtable_var;
+ for my $alias (@$aliases) {
+ my $len = length($alias);
+ $registrations
+ .= qq| Cfish_ZCB_Assign_Str(alias, "$alias", $len);\n|
+ . qq| cfish_VTable_add_alias_to_registry($vtable_var,\n|
+ . qq| (cfish_CharBuf*)alias);\n|;
+ }
+ }
+
+ my $parent = $class->get_parent;
+ next unless $parent;
+ my $parent_class = $parent->get_class_name;
+ $isa_pushes .= qq| isa = get_av("$class_name\::ISA", 1);\n|;
+ $isa_pushes .= qq| av_push(isa, newSVpv("$parent_class", 0));\n|;
+ }
+ my $filepath = catfile( $hierarchy->get_dest, $self->{boot_c_file} );
+ unlink $filepath;
+ sysopen( my $fh, $filepath, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$filepath': $!");
+ print $fh <<END_STUFF;
+$self->{header}
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "$self->{boot_h_file}"
+#include "boil.h"
+$pound_includes
+
+void
+$self->{boot_func}() {
+ AV *isa;
+ cfish_ZombieCharBuf *alias = CFISH_ZCB_WRAP_STR("", 0);
+$registrations
+$isa_pushes
+}
+
+$self->{footer}
+
+END_STUFF
+}
+
+sub write_xs_typemap {
+ my $self = shift;
+ Clownfish::Binding::Perl::TypeMap->write_xs_typemap(
+ hierarchy => $self->{hierarchy}, );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Perl - Perl bindings for a Clownfish::Hierarchy.
+
+=head1 DESCRIPTION
+
+Clownfish::Binding::Perl presents an interface for auto-generating XS and
+Perl code to bind C code for a Clownfish class hierarchy to Perl.
+
+In theory this module could be much more flexible and its API could be more
+elegant. There are many ways which you could walk the parsed parcels,
+classes, methods, etc. in a Clownfish::Hierarchy and generate binding code.
+However, our needs are very limited, so we are content with a "one size fits
+one" solution.
+
+In particular, this module assumes that the XS bindings for all classes in the
+hierarchy should be assembled into a single shared object which belongs to the
+primary, "boot" class. There's no reason why it could not write one .xs file
+per class, or one per parcel, instead.
+
+The files written by this class are derived from the name of the boot class.
+If it is "Crustacean", the following files will be generated.
+
+ # Generated by write_bindings()
+ $lib_dir/Crustacean.xs
+ $lib_dir/Crustacean/Autobinding.pm
+
+ # Generated by write_boot()
+ $hierarchy_dest_dir/crust_boot.h
+ $hierarchy_dest_dir/crust_boot.c
+
+=head1 METHODS
+
+=head2 new
+
+ my $perl_binding = Clownfish::Binding::Perl->new(
+ boot_class => 'Crustacean', # required
+ parcel => 'Crustacean', # required
+ hierarchy => $hierarchy, # required
+ lib_dir => 'lib', # required
+ header => "/* Autogenerated file */\n", # required
+ footer => $copyfoot, # required
+ );
+
+=over
+
+=item * B<boot_class> - The name of the main class, which will own the shared
+object.
+
+=item * B<parcel> - The L<Clownfish::Parcel> to which the C<boot_class>
+belongs.
+
+=item * B<hierarchy> - A Clownfish::Hierarchy.
+
+=item * B<lib_dir> - location of the Perl lib directory to which files will be
+written.
+
+=item * B<header> - Text which will be prepended to generated C/XS files --
+typically, an "autogenerated file" warning.
+
+=item * B<footer> - Text to be appended to the end of generated C/XS files --
+typically copyright information.
+
+=back
+
+=head2 write_bindings
+
+ $perl_binding->write_bindings;
+
+Generate the XS bindings (including "Autobind.pm) for all classes in the
+hierarchy.
+
+=head2 prepare_pod
+
+ my $filepaths_and_pod = $perl_binding->prepare_pod;
+ while ( my ( $filepath, $pod ) = each %$filepaths_and_pod ) {
+ add_to_cleanup($filepath);
+ spew_file( $filepath, $pod );
+ }
+
+Auto-generate POD for all classes bindings which were spec'd with C<make_pod>
+directives. See whether a .pod file exists and is up to date.
+
+Return a hash representing POD files that need to be modified; the keys are
+filepaths, and the values are the POD file content.
+
+The caller must take responsibility for actually writing out the POD files,
+after adding the filepaths to cleanup records and so on.
+
+=head2 write_boot
+
+ $perl_binding->write_boot;
+
+Write out "boot" files to the Hierarchy's C<dest_dir> which contain code for
+bootstrapping Clownfish classes.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Binding/Perl/Class.pm b/clownfish/lib/Clownfish/Binding/Perl/Class.pm
new file mode 100644
index 0000000..b503358
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Perl/Class.pm
@@ -0,0 +1,475 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Perl::Class;
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+our %registry;
+sub registry { \%registry }
+
+our %register_PARAMS = (
+ parcel => undef,
+ class_name => undef,
+ bind_methods => undef,
+ bind_constructors => undef,
+ make_pod => undef,
+ xs_code => undef,
+ client => undef,
+);
+
+sub register {
+ my $either = shift;
+ verify_args( \%register_PARAMS, @_ ) or confess $@;
+ my $self = bless { %register_PARAMS, @_, }, ref($either) || $either;
+
+ # Validate.
+ confess("Missing required param 'class_name'")
+ unless $self->{class_name};
+ confess("$self->{class_name} already registered")
+ if exists $registry{ $self->{class_name} };
+
+ # Retrieve Clownfish::Class client, if it will be needed.
+ if ( $self->{bind_methods}
+ || $self->{bind_constructors}
+ || $self->{make_pod} )
+ {
+ $self->{client} = Clownfish::Class->fetch_singleton(
+ parcel => $self->{parcel},
+ class_name => $self->{class_name},
+ );
+ confess("Can't fetch singleton for $self->{class_name}")
+ unless $self->{client};
+ }
+
+ # Add to registry.
+ $registry{ $self->{class_name} } = $self;
+
+ return $self;
+}
+
+sub get_class_name { shift->{class_name} }
+sub get_bind_methods { shift->{bind_methods} }
+sub get_bind_constructors { shift->{bind_constructors} }
+sub get_make_pod { shift->{make_pod} }
+sub get_client { shift->{client} }
+sub get_xs_code { shift->{xs_code} }
+
+sub constructor_bindings {
+ my $self = shift;
+ my @bound = map {
+ my $xsub = Clownfish::Binding::Perl::Constructor->new(
+ class => $self->{client},
+ alias => $_,
+ );
+ } @{ $self->{bind_constructors} };
+ return @bound;
+}
+
+sub method_bindings {
+ my $self = shift;
+ my $client = $self->{client};
+ my $meth_list = $self->{bind_methods};
+ my @bound;
+
+ # Assemble a list of methods to be bound for this class.
+ my %meth_to_bind;
+ for my $meth_namespec (@$meth_list) {
+ my ( $alias, $name )
+ = $meth_namespec =~ /^(.*?)\|(.*)$/
+ ? ( $1, $2 )
+ : ( lc($meth_namespec), $meth_namespec );
+ $meth_to_bind{$name} = { alias => $alias };
+ }
+
+ # Iterate over all this class's methods, stopping to bind each one that
+ # was spec'd.
+ for my $method ( @{ $client->methods } ) {
+ my $meth_name = $method->get_macro_sym;
+ my $bind_args = delete $meth_to_bind{$meth_name};
+ next unless $bind_args;
+
+ # Safety checks against excess binding code or private methods.
+ if ( !$method->novel ) {
+ confess( "Binding spec'd for method '$meth_name' in class "
+ . "$self->{class_name}, but it's overridden and "
+ . "should be bound via the parent class" );
+ }
+ elsif ( $method->private ) {
+ confess( "Binding spec'd for method '$meth_name' in class "
+ . "$self->{class_name}, but it's private" );
+ }
+
+ # Create an XSub binding for each override. Each of these directly
+ # calls the implementing function, rather than invokes the method on
+ # the object using VTable method dispatch. Doing things this way
+ # allows SUPER:: invocations from Perl-space to work properly.
+ for my $descendant ( @{ $client->tree_to_ladder } ) { # includes self
+ my $real_method = $descendant->novel_method( lc($meth_name) );
+ next unless $real_method;
+
+ # Create the binding, add it to the array.
+ my $method_binding = Clownfish::Binding::Perl::Method->new(
+ method => $real_method,
+ %$bind_args,
+ );
+ push @bound, $method_binding;
+ }
+ }
+
+ # Verify that we processed all methods.
+ my @leftover_meths = keys %meth_to_bind;
+ confess("Leftover for $self->{class_name}: '@leftover_meths'")
+ if @leftover_meths;
+
+ return @bound;
+}
+
+sub _gen_subroutine_pod {
+ my ( $self, %args ) = @_;
+ my ( $func, $sub_name, $class, $code_sample, $class_name )
+ = @args{qw( func name class sample class_name )};
+ my $param_list = $func->get_param_list;
+ my $args = "";
+ my $num_vars = $param_list->num_vars;
+
+ # Only allow "public" subs to be exposed as part of the public API.
+ confess("$class_name->$sub_name is not public") unless $func->public;
+
+ # Get documentation, which may be inherited.
+ my $docucom = $func->get_docucomment;
+ if ( !$docucom ) {
+ my $micro_sym = $func->micro_sym;
+ my $parent = $class;
+ while ( $parent = $parent->get_parent ) {
+ my $parent_func = $parent->method($micro_sym);
+ last unless $parent_func;
+ $docucom = $parent_func->get_docucomment;
+ last if $docucom;
+ }
+ }
+ confess("No DocuComment for '$sub_name' in '$class_name'")
+ unless $docucom;
+
+ # Build string summarizing arguments to use in header.
+ if ( $num_vars > 2 or ( $args{is_constructor} && $num_vars > 1 ) ) {
+ $args = " I<[labeled params]> ";
+ }
+ elsif ( $param_list->num_vars ) {
+ $args = $func->get_param_list->name_list;
+ $args =~ s/self.*?(?:,\s*|$)//; # kill self param
+ }
+
+ # Add code sample.
+ my $pod = "=head2 $sub_name($args)\n\n";
+ if ( defined($code_sample) && length($code_sample) ) {
+ $pod .= "$code_sample\n";
+ }
+
+ # Incorporate "description" text from DocuComment.
+ if ( my $long_doc = $docucom->get_description ) {
+ $pod .= _perlify_doc_text($long_doc) . "\n\n";
+ }
+
+ # Add params in a list.
+ my $param_names = $docucom->get_param_names;
+ my $param_docs = $docucom->get_param_docs;
+ if (@$param_names) {
+ $pod .= "=over\n\n";
+ for ( my $i = 0; $i <= $#$param_names; $i++ ) {
+ $pod .= "=item *\n\n";
+ $pod .= "B<$param_names->[$i]> - $param_docs->[$i]\n\n";
+ }
+ $pod .= "=back\n\n";
+ }
+
+ # Add return value description, if any.
+ if ( defined( my $retval = $docucom->get_retval ) ) {
+ $pod .= "Returns: $retval\n\n";
+ }
+
+ return $pod;
+}
+
+sub create_pod {
+ my $self = shift;
+ my $pod_args = $self->{make_pod} or return;
+ my $class = $self->{client} or die "No client for $self->{class_name}";
+ my $class_name = $class->get_class_name;
+ my $docucom = $class->get_docucomment;
+ confess("No DocuComment for '$class_name'") unless $docucom;
+ my $brief = $docucom->get_brief;
+ my $description
+ = _perlify_doc_text( $pod_args->{description} || $docucom->get_long );
+
+ # Create SYNOPSIS.
+ my $synopsis_pod = '';
+ if ( defined $pod_args->{synopsis} ) {
+ $synopsis_pod = qq|=head1 SYNOPSIS\n\n$pod_args->{synopsis}\n|;
+ }
+
+ # Create CONSTRUCTORS.
+ my $constructor_pod = "";
+ my $constructors = $pod_args->{constructors} || [];
+ if ( defined $pod_args->{constructor} ) {
+ push @$constructors, $pod_args->{constructor};
+ }
+ if (@$constructors) {
+ $constructor_pod = "=head1 CONSTRUCTORS\n\n";
+ for my $spec (@$constructors) {
+ if ( !ref $spec ) {
+ $constructor_pod .= _perlify_doc_text($spec);
+ }
+ else {
+ my $func_name = $spec->{func} || 'init';
+ my $init_func = $class->function($func_name);
+ my $ctor_name = $spec->{name} || 'new';
+ my $code_sample = $spec->{sample};
+ $constructor_pod .= _perlify_doc_text(
+ $self->_gen_subroutine_pod(
+ func => $init_func,
+ name => $ctor_name,
+ sample => $code_sample,
+ class => $class,
+ class_name => $class_name,
+ is_constructor => 1,
+ )
+ );
+ }
+ }
+ }
+
+ # Create METHODS, possibly including an ABSTRACT METHODS section.
+ my @method_docs;
+ my $methods_pod = "";
+ my @abstract_method_docs;
+ my $abstract_methods_pod = "";
+ for my $spec ( @{ $pod_args->{methods} } ) {
+ my $meth_name = ref($spec) ? $spec->{name} : $spec;
+ my $method = $class->method($meth_name);
+ confess("Can't find method '$meth_name' in class '$class_name'")
+ unless $method;
+ my $method_pod;
+ if ( ref($spec) ) {
+ $method_pod = $spec->{pod};
+ }
+ else {
+ $method_pod = $self->_gen_subroutine_pod(
+ func => $method,
+ name => $meth_name,
+ sample => '',
+ class => $class,
+ class_name => $class_name
+ );
+ }
+ if ( $method->abstract ) {
+ push @abstract_method_docs, _perlify_doc_text($method_pod);
+ }
+ else {
+ push @method_docs, _perlify_doc_text($method_pod);
+ }
+ }
+ if (@method_docs) {
+ $methods_pod = join( "", "=head1 METHODS\n\n", @method_docs );
+ }
+ if (@abstract_method_docs) {
+ $abstract_methods_pod = join( "", "=head1 ABSTRACT METHODS\n\n",
+ @abstract_method_docs );
+ }
+
+ # Build an INHERITANCE section describing class ancestry.
+ my $child = $class;
+ my @ancestors;
+ while ( defined( my $parent = $child->get_parent ) ) {
+ push @ancestors, $parent;
+ $child = $parent;
+ }
+ my $inheritance_pod = "";
+ if (@ancestors) {
+ $inheritance_pod = "=head1 INHERITANCE\n\n";
+ $inheritance_pod .= $class->get_class_name;
+ for my $ancestor (@ancestors) {
+ $inheritance_pod .= " isa L<" . $ancestor->get_class_name . ">";
+ }
+ $inheritance_pod .= ".\n";
+ }
+
+ # Put it all together.
+ my $pod = <<END_POD;
+# Auto-generated file -- DO NOT EDIT!!!!!
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+==head1 NAME
+
+$class_name - $brief
+
+$synopsis_pod
+
+==head1 DESCRIPTION
+
+$description
+
+$constructor_pod
+
+$abstract_methods_pod
+
+$methods_pod
+
+$inheritance_pod
+
+==cut
+
+END_POD
+
+ # Kill off stupid hack which allows us to embed pod in this file without
+ # messing up what you see when you perldoc it.
+ $pod =~ s/^==/=/gm;
+
+ return $pod;
+}
+
+sub _perlify_doc_text {
+ my $documentation = shift;
+
+ # Remove double-equals hack needed to fool perldoc, PAUSE, etc. :P
+ $documentation =~ s/^==/=/mg;
+
+ # Change <code>foo</code> to C<< foo >>.
+ $documentation =~ s#<code>(.*?)</code>#C<< $1 >>#gsm;
+
+ # Lowercase all method names: Open_In() => open_in()
+ $documentation
+ =~ s/([A-Z][A-Za-z0-9]*(?:_[A-Z][A-Za-z0-9]*)*\(\))/\L$1\E/gsm;
+
+ # Change all instances of NULL to 'undef'
+ $documentation =~ s/NULL/undef/g;
+
+ # Change "Err_error" to "Lucy->error".
+ $documentation =~ s/Err_error/Lucy->error/g;
+
+ return $documentation;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Perl::Class - Generate Perl binding code for a
+Clownfish::Class.
+
+=head1 CLASS METHODS
+
+=head1 register
+
+ Clownfish::Binding::Perl::Class->register(
+ parcel => 'MyProject' , # required
+ class_name => 'Foo::FooJr', # required
+ bind_methods => [qw( Do_Stuff _get_foo|Get_Foo )], # default: undef
+ bind_constructors => [qw( new _new2|init2 )], # default: undef
+ make_pod => [qw( get_foo )], # default: undef
+ xs_code => undef, # default: undef
+ );
+
+Create a new class binding and lodge it in the registry. May only be called
+once for each unique class name, and must be called after all classes have
+been parsed (via Clownfish::Hierarchy's build()).
+
+=over
+
+=item * B<parcel> - A L<Clownfish::Parcel> or parcel name.
+
+=item * B<class_name> - The name of the class to be registered.
+
+=item * B<xs_code> - Raw XS code to be included in the final .xs file
+generated by Clownfish::Binding::Perl. The XS directives PACKAGE and
+MODULE should be specified.
+
+=item * B<bind_methods> - An array of names for novel methods for which XS
+bindings should be auto-generated, supplied using Clownfish's C<Macro_Name>
+method-naming convention. The Perl subroutine name will be derived by
+lowercasing C<Method_Name> to C<method_name>, but this can be overridden by
+prepending an alias and a pipe: e.g. C<_get_foo|Get_Foo>.
+
+=item * B<bind_constructors> - An array of constructor names. The default
+implementing function is the class's C<init> function, unless it is overridden
+using a pipe-separated string: C<_new2|init2> would create a Perl subroutine
+"_new2" which would invoke C<myproj_FooJr_init2>.
+
+=item * B<make_pod> - A specification for generating POD. TODO: document this
+spec, or break it up into multiple methods. (For now, just see examples from
+the source code.)
+
+=back
+
+=head1 registry
+
+ my $registry = Clownfish::Binding::Perl::Class->registry;
+ while ( my $class_name, $class_binding ) = each %$registry ) {
+ ...
+ }
+
+Return the hash registry used by register(). The keys are class names, and
+the values are Clownfish::Binding::Perl::Class objects.
+
+=head1 OBJECT METHODS
+
+=head2 get_class_name get_bind_methods get_bind_methods get_make_pod
+get_xs_code get_client
+
+Accessors. C<get_client> retrieves the Clownfish::Class module to be
+bound.
+
+=head2 constructor_bindings
+
+ my @ctor_bindings = $class_binding->constructor_bindings;
+
+Return a list of Clownfish::Binding::Perl::Constructor objects created as
+per the C<bind_constructors> spec.
+
+=head2 method_bindings
+
+ my @method_bindings = $class_binding->method_bindings;
+
+Return a list of Clownfish::Binding::Perl::Method objects created as per
+the C<bind_methods> spec.
+
+=head2 create_pod
+
+ my $pod = $class_binding->create_pod;
+
+Auto-generate POD according to the make_pod spec, if such a spec was supplied.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Perl/Constructor.pm b/clownfish/lib/Clownfish/Binding/Perl/Constructor.pm
new file mode 100644
index 0000000..43f7715
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Perl/Constructor.pm
@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Perl::Constructor;
+use base qw( Clownfish::Binding::Perl::Subroutine );
+use Carp;
+use Clownfish::ParamList;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $class = delete $args{class};
+ my $alias = delete $args{alias};
+ my $init_func_name = $alias =~ s/^(\w+)\|(\w+)$/$1/ ? $2 : 'init';
+ my $class_name = $class->get_class_name;
+
+ # Find the implementing function.
+ my $func;
+ for my $function ( @{ $class->functions } ) {
+ next unless $function->micro_sym eq $init_func_name;
+ $func = $function;
+ last;
+ }
+ confess("Missing or invalid init() function for $class_name")
+ unless $func;
+
+ my $self = $either->SUPER::new(
+ param_list => $func->get_param_list,
+ retval_type => $func->get_return_type,
+ class_name => $class_name,
+ use_labeled_params => 1,
+ alias => $alias,
+ %args
+ );
+ $self->{init_func} = $func;
+ return $self;
+}
+
+sub xsub_def {
+ my $self = shift;
+ my $c_name = $self->c_name;
+ my $param_list = $self->{param_list};
+ my $name_list = $param_list->name_list;
+ my $arg_inits = $param_list->get_initial_values;
+ my $arg_vars = $param_list->get_variables;
+ my $func_sym = $self->{init_func}->full_func_sym;
+ my $allot_params = $self->build_allot_params;
+
+ # Compensate for swallowed refcounts.
+ my $refcount_mods = "";
+ for ( my $i = 1; $i <= $#$arg_vars; $i++ ) {
+ my $var = $arg_vars->[$i];
+ my $type = $var->get_type;
+ if ( $type->is_object and $type->decremented ) {
+ my $name = $var->micro_sym;
+ $refcount_mods .= "\n LUCY_INCREF($name);";
+ }
+ }
+
+ # Last, so that earlier exceptions while fetching params don't trigger bad
+ # DESTROY.
+ my $self_var = $arg_vars->[0];
+ my $self_type = $self_var->get_type->to_c;
+ my $self_assign
+ = qq|$self_type self = ($self_type)XSBind_new_blank_obj(ST(0));|;
+
+ return <<END_STUFF;
+XS($c_name);
+XS($c_name) {
+ dXSARGS;
+ CHY_UNUSED_VAR(cv);
+ if (items < 1) { CFISH_THROW(CFISH_ERR, "Usage: %s(class_name, ...)", GvNAME(CvGV(cv))); }
+ SP -= items;
+
+ $allot_params
+ $self_assign$refcount_mods
+
+ $self_type retval = $func_sym($name_list);
+ if (retval) {
+ ST(0) = (SV*)Cfish_Obj_To_Host((cfish_Obj*)retval);
+ Cfish_Obj_Dec_RefCount((cfish_Obj*)retval);
+ }
+ else {
+ ST(0) = newSV(0);
+ }
+ sv_2mortal(ST(0));
+ XSRETURN(1);
+}
+
+END_STUFF
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Perl::Constructor - Binding for an object method.
+
+=head1 DESCRIPTION
+
+This class isa Clownfish::Binding::Perl::Subroutine -- see its
+documentation for various code-generating routines.
+
+Constructors are always bound to accept labeled params, even if there is only
+a single argument.
+
+=head1 METHODS
+
+=head2 new
+
+ my $constructor_binding = Clownfish::Binding::Perl::Constructor->new(
+ class => $class,
+ alias => "_new|init2",
+ );
+
+=over
+
+=item * B<class> - A L<Clownfish::Class>.
+
+=item * B<alias> - A specifier for the name of the constructor, and
+optionally, a specifier for the implementing function. If C<alias> has a pipe
+character in it, the text to the left of the pipe will be used as the Perl
+alias, and the text to the right will be used to determine which C function
+should be bound. The default function is "init".
+
+=back
+
+=head2 xsub_def
+
+Generate the XSUB code.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Perl/Method.pm b/clownfish/lib/Clownfish/Binding/Perl/Method.pm
new file mode 100644
index 0000000..b6ffdb2
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Perl/Method.pm
@@ -0,0 +1,275 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Perl::Method;
+use base qw( Clownfish::Binding::Perl::Subroutine );
+use Clownfish::Util qw( verify_args );
+use Clownfish::Binding::Perl::TypeMap qw( from_perl to_perl );
+use Carp;
+
+our %new_PARAMS = (
+ method => undef,
+ alias => undef,
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ confess $@ unless verify_args( \%new_PARAMS, %args );
+
+ # Derive arguments to SUPER constructor from supplied Method.
+ my $method = delete $args{method};
+ $args{retval_type} ||= $method->get_return_type;
+ $args{param_list} ||= $method->get_param_list;
+ $args{alias} ||= $method->micro_sym;
+ $args{class_name} ||= $method->get_class_name;
+ if ( !defined $args{use_labeled_params} ) {
+ $args{use_labeled_params}
+ = $method->get_param_list->num_vars > 2
+ ? 1
+ : 0;
+ }
+ my $self = $either->SUPER::new(%args);
+ $self->{method} = $method;
+
+ return $self;
+}
+
+sub xsub_def {
+ my $self = shift;
+ if ( $self->{use_labeled_params} ) {
+ return $self->_xsub_def_labeled_params;
+ }
+ else {
+ return $self->_xsub_def_positional_args;
+ }
+}
+
+# Build XSUB function body.
+sub _xsub_body {
+ my $self = shift;
+ my $method = $self->{method};
+ my $full_func_sym = $method->full_func_sym;
+ my $param_list = $method->get_param_list;
+ my $arg_vars = $param_list->get_variables;
+ my $name_list = $param_list->name_list;
+ my $body = "";
+
+ # Compensate for functions which eat refcounts.
+ for my $arg_var (@$arg_vars) {
+ my $arg_type = $arg_var->get_type;
+ next unless $arg_type->is_object;
+ next unless $arg_type->decremented;
+ my $var_name = $arg_var->micro_sym;
+ $body .= "LUCY_INCREF($var_name);\n ";
+ }
+
+ if ( $method->void ) {
+ # Invoke method in void context.
+ $body .= qq|$full_func_sym($name_list);\n| . qq| XSRETURN(0);|;
+ }
+ else {
+ # Return a value for method invoked in a scalar context.
+ my $return_type = $method->get_return_type;
+ my $type_str = $return_type->to_c;
+ my $retval_assignment
+ = "ST(0) = " . to_perl( $return_type, 'retval' ) . ';';
+ my $decrement = "";
+ if ( $return_type->is_object and $return_type->incremented ) {
+ $decrement = "\n LUCY_DECREF(retval);";
+ }
+ $body .= qq|$type_str retval = $full_func_sym($name_list);
+ $retval_assignment$decrement
+ sv_2mortal( ST(0) );
+ XSRETURN(1);|
+ }
+
+ return $body;
+}
+
+sub _xsub_def_positional_args {
+ my $self = shift;
+ my $method = $self->{method};
+ my $param_list = $method->get_param_list;
+ my $arg_vars = $param_list->get_variables;
+ my $arg_inits = $param_list->get_initial_values;
+ my $num_args = $param_list->num_vars;
+ my $c_name = $self->c_name;
+ my $body = $self->_xsub_body;
+
+ # Determine how many args are truly required and build an error check.
+ my $min_required = $num_args;
+ while ( defined $arg_inits->[ $min_required - 1 ] ) {
+ $min_required--;
+ }
+ my @xs_arg_names;
+ for ( my $i = 0; $i < $min_required; $i++ ) {
+ push @xs_arg_names, $arg_vars->[$i]->micro_sym;
+ }
+ my $xs_name_list = join( ", ", @xs_arg_names );
+ my $num_args_check;
+ if ( $min_required < $num_args ) {
+ $num_args_check
+ = qq|if (items < $min_required) { |
+ . qq|CFISH_THROW(CFISH_ERR, "Usage: %s(%s)", GvNAME(CvGV(cv)),|
+ . qq| "$xs_name_list"); }|;
+ }
+ else {
+ $num_args_check
+ = qq|if (items != $num_args) { |
+ . qq| CFISH_THROW(CFISH_ERR, "Usage: %s(%s)", GvNAME(CvGV(cv)), |
+ . qq|"$xs_name_list"); }|;
+ }
+
+ # Var assignments.
+ my @var_assignments;
+ for ( my $i = 0; $i < @$arg_vars; $i++ ) {
+ my $var = $arg_vars->[$i];
+ my $val = $arg_inits->[$i];
+ my $var_name = $var->micro_sym;
+ my $var_type = $var->get_type;
+ my $type_c = $var_type->to_c;
+ my $statement;
+ if ( $i == 0 ) { # $self
+ $statement
+ = _self_assign_statement( $var_type, $method->micro_sym );
+ }
+ else {
+ if ( defined $val ) {
+ $statement
+ = "$type_c $var_name = "
+ . "( items >= $i && XSBind_sv_defined(ST($i)) ) ? "
+ . from_perl( $var_type, "ST($i)" )
+ . " : $val;";
+ }
+ else {
+ $statement = "$type_c $var_name = "
+ . from_perl( $var_type, "ST($i)" ) . ';';
+ }
+ }
+ push @var_assignments, $statement;
+ }
+ my $var_assignments = join "\n ", @var_assignments;
+
+ return <<END_STUFF;
+XS($c_name);
+XS($c_name) {
+ dXSARGS;
+ CHY_UNUSED_VAR(cv);
+ SP -= items;
+ $num_args_check;
+
+ /* Extract vars from Perl stack. */
+ $var_assignments
+
+ /* Execute */
+ $body
+}
+END_STUFF
+}
+
+sub _xsub_def_labeled_params {
+ my $self = shift;
+ my $c_name = $self->c_name;
+ my $param_list = $self->{param_list};
+ my $arg_inits = $param_list->get_initial_values;
+ my $arg_vars = $param_list->get_variables;
+ my $self_var = $arg_vars->[0];
+ my $self_assign = _self_assign_statement( $self_var->get_type,
+ $self->{method}->micro_sym );
+ my $allot_params = $self->build_allot_params;
+ my $body = $self->_xsub_body;
+
+ # Prepare error message for incorrect args.
+ my $name_list = $self_var->micro_sym . ", ...";
+ my $num_args_check
+ = qq|if (items < 1) { |
+ . qq|CFISH_THROW(CFISH_ERR, "Usage: %s(%s)", GvNAME(CvGV(cv)), |
+ . qq|"$name_list"); }|;
+
+ return <<END_STUFF;
+XS($c_name);
+XS($c_name) {
+ dXSARGS;
+ CHY_UNUSED_VAR(cv);
+ $num_args_check;
+ SP -= items;
+
+ /* Extract vars from Perl stack. */
+ $allot_params
+ $self_assign
+
+ /* Execute */
+ $body
+}
+END_STUFF
+}
+
+# Create an assignment statement for extracting $self from the Perl stack.
+sub _self_assign_statement {
+ my ( $type, $method_name ) = @_;
+ my $type_c = $type->to_c;
+ $type_c =~ /(\w+)\*$/ or die "Not an object type: $type_c";
+ my $vtable = uc($1);
+
+ # Make an exception for deserialize -- allow self to be NULL if called as
+ # a class method.
+ my $binding_func
+ = $method_name eq 'deserialize'
+ ? 'XSBind_maybe_sv_to_cfish_obj'
+ : 'XSBind_sv_to_cfish_obj';
+ return "$type_c self = ($type_c)$binding_func(ST(0), $vtable, NULL);";
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Perl::Method - Binding for an object method.
+
+=head1 DESCRIPTION
+
+This class isa Clownfish::Binding::Perl::Subroutine -- see its
+documentation for various code-generating routines.
+
+Method bindings use labeled parameters if the C function takes more than one
+argument (other than C<self>). If there is only one argument, the binding
+will be set up to accept a single positional argument.
+
+=head1 METHODS
+
+=head2 new
+
+ my $binding = Clownfish::Binding::Perl::Method->new(
+ method => $method, # required
+ );
+
+=over
+
+=item * B<method> - A L<Clownfish::Method>.
+
+=back
+
+=head2 xsub_def
+
+Generate the XSUB code.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Perl/Subroutine.pm b/clownfish/lib/Clownfish/Binding/Perl/Subroutine.pm
new file mode 100644
index 0000000..55e32ab
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Perl/Subroutine.pm
@@ -0,0 +1,268 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Perl::Subroutine;
+use Carp;
+use Scalar::Util qw( blessed );
+use Clownfish::Class;
+use Clownfish::Function;
+use Clownfish::Method;
+use Clownfish::Variable;
+use Clownfish::ParamList;
+use Clownfish::Util qw( verify_args );
+
+our %new_PARAMS = (
+ param_list => undef,
+ alias => undef,
+ class_name => undef,
+ retval_type => undef,
+ use_labeled_params => undef,
+);
+
+sub new {
+ my $either = shift;
+ verify_args( \%new_PARAMS, @_ ) or confess $@;
+ my $self = bless { %new_PARAMS, @_, }, ref($either) || $either;
+ for (qw( param_list class_name alias retval_type )) {
+ confess("$_ is required") unless defined $self->{$_};
+ }
+ return $self;
+}
+
+sub get_class_name { shift->{class_name} }
+sub use_labeled_params { shift->{use_labeled_params} }
+
+sub perl_name {
+ my $self = shift;
+ return "$self->{class_name}::$self->{alias}";
+}
+
+sub c_name {
+ my $self = shift;
+ my $c_name = "XS_" . $self->perl_name;
+ $c_name =~ s/:+/_/g;
+ return $c_name;
+}
+
+sub c_name_list {
+ my $self = shift;
+ return $self->{param_list}->name_list;
+}
+
+my %params_hash_vals_map = (
+ NULL => 'undef',
+ true => 1,
+ false => 0,
+);
+
+sub params_hash_def {
+ my $self = shift;
+ return unless $self->{use_labeled_params};
+
+ my $params_hash_name = $self->perl_name . "_PARAMS";
+ my $arg_vars = $self->{param_list}->get_variables;
+ my $vals = $self->{param_list}->get_initial_values;
+ my @pairs;
+ for ( my $i = 1; $i < @$arg_vars; $i++ ) {
+ my $var = $arg_vars->[$i];
+ my $val = $vals->[$i];
+ if ( !defined $val ) {
+ $val = 'undef';
+ }
+ elsif ( exists $params_hash_vals_map{$val} ) {
+ $val = $params_hash_vals_map{$val};
+ }
+ push @pairs, $var->micro_sym . " => $val,";
+ }
+
+ if (@pairs) {
+ my $list = join( "\n ", @pairs );
+ return qq|\%$params_hash_name = (\n $list\n);\n|;
+ }
+ else {
+ return qq|\%$params_hash_name = ();\n|;
+ }
+}
+
+my %prim_type_to_allot_macro = (
+ double => 'ALLOT_F64',
+ float => 'ALLOT_F32',
+ int => 'ALLOT_INT',
+ short => 'ALLOT_SHORT',
+ long => 'ALLOT_LONG',
+ size_t => 'ALLOT_SIZE_T',
+ uint64_t => 'ALLOT_U64',
+ uint32_t => 'ALLOT_U32',
+ uint16_t => 'ALLOT_U16',
+ uint8_t => 'ALLOT_U8',
+ int64_t => 'ALLOT_I64',
+ int32_t => 'ALLOT_I32',
+ int16_t => 'ALLOT_I16',
+ int8_t => 'ALLOT_I8',
+ chy_bool_t => 'ALLOT_BOOL',
+);
+
+sub _allot_params_arg {
+ my ( $type, $label, $required ) = @_;
+ confess("Not a Clownfish::Type")
+ unless blessed($type) && $type->isa('Clownfish::Type');
+ my $len = length($label);
+ my $req_string = $required ? 'true' : 'false';
+
+ if ( $type->is_object ) {
+ my $struct_sym = $type->get_specifier;
+ my $vtable = uc($struct_sym);
+ if ( $struct_sym =~ /^[a-z_]*(Obj|CharBuf)$/ ) {
+ # Share buffers rather than copy between Perl scalars and
+ # Clownfish string types.
+ return qq|ALLOT_OBJ(\&$label, "$label", $len, $req_string, |
+ . qq|$vtable, alloca(cfish_ZCB_size()))|;
+ }
+ else {
+ return qq|ALLOT_OBJ(\&$label, "$label", $len, $req_string, |
+ . qq|$vtable, NULL)|;
+ }
+ }
+ elsif ( $type->is_primitive ) {
+ if ( my $allot = $prim_type_to_allot_macro{ $type->to_c } ) {
+ return qq|$allot(\&$label, "$label", $len, $req_string)|;
+ }
+ }
+
+ confess( "Missing typemap for " . $type->to_c );
+}
+
+sub build_allot_params {
+ my $self = shift;
+ my $param_list = $self->{param_list};
+ my $arg_inits = $param_list->get_initial_values;
+ my $arg_vars = $param_list->get_variables;
+ my $params_hash = $self->perl_name . "_PARAMS";
+ my $allot_params = "";
+
+ # Declare variables and assign default values.
+ for ( my $i = 1; $i <= $#$arg_vars; $i++ ) {
+ my $arg_var = $arg_vars->[$i];
+ my $val = $arg_inits->[$i];
+ if ( !defined($val) ) {
+ $val = $arg_var->get_type->is_object ? 'NULL' : '0';
+ }
+ $allot_params .= $arg_var->local_c . " = $val;\n ";
+ }
+
+ # Iterate over args in param list.
+ $allot_params .= qq|chy_bool_t args_ok = XSBind_allot_params(\n|
+ . qq| &(ST(0)), 1, items, "$params_hash",\n|;
+ for ( my $i = 1; $i <= $#$arg_vars; $i++ ) {
+ my $var = $arg_vars->[$i];
+ my $val = $arg_inits->[$i];
+ my $required = defined $val ? 0 : 1;
+ my $name = $var->micro_sym;
+ my $type = $var->get_type;
+ $allot_params .= " "
+ . _allot_params_arg( $type, $name, $required ) . ",\n";
+ }
+ $allot_params .= qq| NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }|;
+
+ return $allot_params;
+}
+
+sub xsub_def { confess "Abstract method" }
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Perl::Subroutine - Abstract base binding for a
+Clownfish::Function.
+
+=head1 SYNOPSIS
+
+ # Abstract base class.
+
+=head1 DESCRIPTION
+
+This class is used to generate binding code for invoking Clownfish's
+functions and methods across the Perl/C barrier.
+
+=head1 METHODS
+
+=head2 new
+
+ my $binding = $subclass->SUPER::new(
+ param_list => $param_list, # required
+ alias => 'pinch', # required
+ class_name => 'Crustacean::Claw', # required
+ retval_type => $type, # required
+ use_labeled_params => 1, # default: false
+ );
+
+Abstract constructor.
+
+=over
+
+=item * B<param_list> - A L<Clownfish::ParamList>.
+
+=item * B<alias> - The local, unqualified name for the Perl subroutine that
+will be used to invoke the function.
+
+=item * B<class_name> - The name of the Perl class that the subroutine belongs
+to.
+
+=item * B<retval_type> - The return value's L<Type|Clownfish::Type>.
+
+=item * B<use_labeled_params> - True if the binding should take hash-style
+labeled parameters, false if it should take positional arguments.
+
+=back
+
+=head2 xsub_def
+
+Abstract method which must return C code (not XS code) defining the Perl XSUB.
+
+=head2 get_class_name use_labeled_params
+
+Accessors.
+
+=head2 perl_name
+
+Returns the fully-qualified perl sub name.
+
+=head2 c_name
+
+Returns the fully-qualified name of the C function that implements the XSUB.
+
+=head2 c_name_list
+
+Returns a string containing the names of arguments to feed to bound C
+function, joined by commas.
+
+=head2 params_hash_def
+
+Return Perl code initializing a package-global hash where all the keys are the
+names of labeled params. The hash's name consists of the the binding's
+perl_name() plus "_PARAMS".
+
+=cut
diff --git a/clownfish/lib/Clownfish/Binding/Perl/TypeMap.pm b/clownfish/lib/Clownfish/Binding/Perl/TypeMap.pm
new file mode 100644
index 0000000..088eedb
--- /dev/null
+++ b/clownfish/lib/Clownfish/Binding/Perl/TypeMap.pm
@@ -0,0 +1,298 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Binding::Perl::TypeMap;
+use base qw( Exporter );
+use Scalar::Util qw( blessed );
+use Carp;
+use Fcntl;
+
+our @EXPORT_OK = qw( from_perl to_perl );
+
+# Convert from a Perl scalar to a primitive type.
+my %primitives_from_perl = (
+ double => sub {"SvNV($_[0])"},
+ float => sub {"(float)SvNV($_[0])"},
+ int => sub {"(int)SvIV($_[0])"},
+ short => sub {"(short)SvIV($_[0])"},
+ long => sub {
+ "((sizeof(long) <= sizeof(IV)) ? "
+ . "(long)SvIV($_[0]) : (long)SvNV($_[0]))";
+ },
+ size_t => sub {"(size_t)SvIV($_[0])"},
+ uint64_t => sub {"(uint64_t)SvNV($_[0])"},
+ uint32_t => sub {"(uint32_t)SvUV($_[0])"},
+ uint16_t => sub {"(uint16_t)SvUV($_[0])"},
+ uint8_t => sub {"(uint8_t)SvUV($_[0])"},
+ int64_t => sub {"(int64_t)SvNV($_[0])"},
+ int32_t => sub {"(int32_t)SvIV($_[0])"},
+ int16_t => sub {"(int16_t)SvIV($_[0])"},
+ int8_t => sub {"(int8_t)SvIV($_[0])"},
+ chy_bool_t => sub {"SvTRUE($_[0]) ? 1 : 0"},
+);
+
+# Convert from a primitive type to a Perl scalar.
+my %primitives_to_perl = (
+ double => sub {"newSVnv($_[0])"},
+ float => sub {"newSVnv($_[0])"},
+ int => sub {"newSViv($_[0])"},
+ short => sub {"newSViv($_[0])"},
+ long => sub {
+ "((sizeof(long) <= sizeof(IV)) ? "
+ . "newSViv((IV)$_[0]) : newSVnv((NV)$_[0]))";
+ },
+ size_t => sub {"newSViv($_[0])"},
+ uint64_t => sub {
+ "sizeof(UV) == 8 ? newSVuv((UV)$_[0]) : newSVnv((NV)$_[0])";
+ },
+ uint32_t => sub {"newSVuv($_[0])"},
+ uint16_t => sub {"newSVuv($_[0])"},
+ uint8_t => sub {"newSVuv($_[0])"},
+ int64_t => sub {
+ "sizeof(IV) == 8 ? newSViv((IV)$_[0]) : newSVnv((NV)$_[0])";
+ },
+ int32_t => sub {"newSViv($_[0])"},
+ int16_t => sub {"newSViv($_[0])"},
+ int8_t => sub {"newSViv($_[0])"},
+ chy_bool_t => sub {"newSViv($_[0])"},
+);
+
+sub from_perl {
+ my ( $type, $xs_var ) = @_;
+ confess("Not a Clownfish::Type")
+ unless blessed($type) && $type->isa('Clownfish::Type');
+
+ if ( $type->is_object ) {
+ my $struct_sym = $type->get_specifier;
+ my $vtable = uc($struct_sym);
+ if ( $struct_sym =~ /^[a-z_]*(Obj|CharBuf)$/ ) {
+ # Share buffers rather than copy between Perl scalars and
+ # Clownfish string types.
+ return "($struct_sym*)XSBind_sv_to_cfish_obj($xs_var, "
+ . "$vtable, alloca(cfish_ZCB_size()))";
+ }
+ else {
+ return "($struct_sym*)XSBind_sv_to_cfish_obj($xs_var, "
+ . "$vtable, NULL)";
+ }
+ }
+ elsif ( $type->is_primitive ) {
+ if ( my $sub = $primitives_from_perl{ $type->to_c } ) {
+ return $sub->($xs_var);
+ }
+ }
+
+ confess( "Missing typemap for " . $type->to_c );
+}
+
+sub to_perl {
+ my ( $type, $cf_var ) = @_;
+ confess("Not a Clownfish::Type")
+ unless ref($type) && $type->isa('Clownfish::Type');
+ my $type_str = $type->to_c;
+
+ if ( $type->is_object ) {
+ return "($cf_var == NULL ? newSV(0) : "
+ . "XSBind_cfish_to_perl((cfish_Obj*)$cf_var))";
+ }
+ elsif ( $type->is_primitive ) {
+ if ( my $sub = $primitives_to_perl{$type_str} ) {
+ return $sub->($cf_var);
+ }
+ }
+ elsif ( $type->is_composite ) {
+ if ( $type_str eq 'void*' ) {
+ # Assume that void* is a reference SV -- either a hashref or an
+ # arrayref.
+ return "newRV_inc((SV*)($cf_var))";
+ }
+ }
+
+ confess("Missing typemap for '$type_str'");
+}
+
+sub write_xs_typemap {
+ my ( undef, %args ) = @_;
+ my $hierarchy = $args{hierarchy};
+
+ my $class_typemap_start = "";
+ my $class_typemap_input = "";
+ my $class_typemap_output = "";
+
+ for my $class ( @{ $hierarchy->ordered_classes } ) {
+ my $full_struct_sym = $class->full_struct_sym;
+ my $vtable = $class->full_vtable_var;
+ my $label = $vtable . "_";
+ $class_typemap_start .= "$full_struct_sym*\t$label\n";
+ $class_typemap_input .= <<END_INPUT;
+$label
+ \$var = ($full_struct_sym*)XSBind_sv_to_cfish_obj(\$arg, $vtable, NULL);
+
+END_INPUT
+
+ $class_typemap_output .= <<END_OUTPUT;
+$label
+ \$arg = (SV*)Cfish_Obj_To_Host((cfish_Obj*)\$var);
+ LUCY_DECREF(\$var);
+
+END_OUTPUT
+ }
+
+ # Blast it out.
+ sysopen( my $typemap_fh, 'typemap', O_CREAT | O_WRONLY | O_EXCL )
+ or die "Couldn't open 'typemap' for writing: $!";
+ print $typemap_fh <<END_STUFF;
+# Auto-generated file.
+
+TYPEMAP
+chy_bool_t\tCHY_BOOL
+int8_t\tCHY_SIGNED_INT
+int16_t\tCHY_SIGNED_INT
+int32_t\tCHY_SIGNED_INT
+int64_t\tCHY_BIG_SIGNED_INT
+uint8_t\tCHY_UNSIGNED_INT
+uint16_t\tCHY_UNSIGNED_INT
+uint32_t\tCHY_UNSIGNED_INT
+uint64_t\tCHY_BIG_UNSIGNED_INT
+
+const lucy_CharBuf*\tCONST_CHARBUF
+$class_typemap_start
+
+INPUT
+
+CHY_BOOL
+ \$var = (\$type)SvTRUE(\$arg);
+
+CHY_SIGNED_INT
+ \$var = (\$type)SvIV(\$arg);
+
+CHY_UNSIGNED_INT
+ \$var = (\$type)SvUV(\$arg);
+
+CHY_BIG_SIGNED_INT
+ \$var = (sizeof(IV) == 8) ? (\$type)SvIV(\$arg) : (\$type)SvNV(\$arg);
+
+CHY_BIG_UNSIGNED_INT
+ \$var = (sizeof(UV) == 8) ? (\$type)SvUV(\$arg) : (\$type)SvNV(\$arg);
+
+CONST_CHARBUF
+ \$var = (const cfish_CharBuf*)CFISH_ZCB_WRAP_STR(SvPVutf8_nolen(\$arg), SvCUR(\$arg));
+
+$class_typemap_input
+
+OUTPUT
+
+CHY_BOOL
+ sv_setiv(\$arg, (IV)\$var);
+
+CHY_SIGNED_INT
+ sv_setiv(\$arg, (IV)\$var);
+
+CHY_UNSIGNED_INT
+ sv_setuv(\$arg, (UV)\$var);
+
+CHY_BIG_SIGNED_INT
+ if (sizeof(IV) == 8) { sv_setiv(\$arg, (IV)\$var); }
+ else { sv_setnv(\$arg, (NV)\$var); }
+
+CHY_BIG_UNSIGNED_INT
+ if (sizeof(UV) == 8) { sv_setuv(\$arg, (UV)\$var); }
+ else { sv_setnv(\$arg, (NV)\$var); }
+
+$class_typemap_output
+
+END_STUFF
+
+ close $typemap_fh or die $!;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Binding::Perl::TypeMap - Convert between Clownfish and Perl via XS.
+
+=head1 DESCRIPTION
+
+TypeMap serves up C code fragments for translating between Perl data
+structures and Clownfish data structures. The functions to_perl() and
+from_perl() achieve this for individual types; write_xs_typemap() exports all
+types using the XS "typemap" format documented in C<perlxs>.
+
+=head1 FUNCTIONS
+
+=head2 from_perl
+
+ my $expression = from_perl( $type, $xs_var );
+
+Return an expression which converts from a Perl scalar to a variable of type
+$type.
+
+=over
+
+=item * B<type> - A Clownfish::Type, which will be used to select the
+mapping code.
+
+=item * B<xs_var> - The C name of the Perl scalar from which we are extracting
+a value.
+
+=back
+
+=head2 to_perl
+
+ my $c_code = to_perl( $type, $cf_var );
+
+Return an expression converts from a variable of type $type to a Perl scalar.
+
+=over
+
+=item * B<type> - A Clownfish::Type, which will be used to select the
+mapping code.
+
+=item * B<cf_var> - The name of the variable from which we are extracting a
+value.
+
+=back
+
+=head1 CLASS METHODS
+
+=head2 write_xs_typemap
+
+ Clownfish::Binding::Perl::Typemap->write_xs_typemap(
+ hierarchy => $hierarchy,
+ );
+
+=over
+
+=item * B<hierarchy> - A L<Clownfish::Hierarchy>.
+
+=back
+
+Auto-generate a "typemap" file that adheres to the conventions documented in
+L<perlxs>.
+
+We generate this file on the fly rather than maintain a static copy because we
+want an entry for each Clownfish type so that we can differentiate between
+them when checking arguments. Keeping the entries up-to-date manually as
+classes come and go would be a pain.
+
+=cut
diff --git a/clownfish/lib/Clownfish/CBlock.pm b/clownfish/lib/Clownfish/CBlock.pm
new file mode 100644
index 0000000..ea91d7a
--- /dev/null
+++ b/clownfish/lib/Clownfish/CBlock.pm
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::CBlock;
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+our %new_PARAMS = ( contents => undef, );
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ confess("Missing required param 'contents'")
+ unless defined $args{contents};
+ return $either->_new( $args{contents} );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::CBlock - A block of embedded C code.
+
+=head1 DESCRIPTION
+
+CBlock exists to support embedding literal C code within Clownfish header
+files:
+
+ class Crustacean::Lobster {
+ /* ... */
+
+ /** Give a lobstery greeting.
+ */
+ inert inline void
+ say_hello(Lobster *self);
+ }
+
+ __C__
+ #include <stdio.h>
+ static CHY_INLINE void
+ crust_Lobster_say_hello(crust_Lobster *self)
+ {
+ printf("Prepare to die, human scum.\n");
+ }
+ __END_C__
+
+=head1 METHODS
+
+=head2 new
+
+ my $c_block = Clownfish::CBlock->new(
+ contents => $text,
+ );
+
+=over
+
+=item * B<contents> - Raw C code.
+
+=back
+
+=head2 get_contents
+
+Accessor.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Class.pm b/clownfish/lib/Clownfish/Class.pm
new file mode 100644
index 0000000..039c4dd
--- /dev/null
+++ b/clownfish/lib/Clownfish/Class.pm
@@ -0,0 +1,294 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Class;
+use base qw( Clownfish::Symbol );
+use Carp;
+use Config;
+use Clownfish::Function;
+use Clownfish::Method;
+use Clownfish::Util qw(
+ verify_args
+ a_isa_b
+);
+use Clownfish::Dumpable;
+
+END { __PACKAGE__->_clear_registry() }
+
+our %create_PARAMS = (
+ source_class => undef,
+ class_name => undef,
+ cnick => undef,
+ parent_class_name => undef,
+ docucomment => undef,
+ inert => undef,
+ final => undef,
+ parcel => undef,
+ exposure => 'parcel',
+);
+
+our %fetch_singleton_PARAMS = (
+ parcel => undef,
+ class_name => undef,
+);
+
+sub fetch_singleton {
+ my ( undef, %args ) = @_;
+ verify_args( \%fetch_singleton_PARAMS, %args ) or confess $@;
+ # Maybe prepend parcel prefix.
+ my $parcel = $args{parcel};
+ if ( defined $parcel ) {
+ if ( !a_isa_b( $parcel, "Clownfish::Parcel" ) ) {
+ $parcel = Clownfish::Parcel->singleton( name => $parcel );
+ }
+ }
+ return _fetch_singleton( $parcel, $args{class_name} );
+}
+
+sub new { confess("The constructor for Clownfish::Class is create()") }
+
+sub create {
+ my ( $either, %args ) = @_;
+ verify_args( \%create_PARAMS, %args ) or confess $@;
+ $args{parcel} = Clownfish::Parcel->acquire( $args{parcel} );
+ my $package = ref($either) || $either;
+ return _create(
+ $package,
+ @args{
+ qw( parcel exposure class_name cnick micro_sym
+ docucomment source_class parent_class_name final inert )
+ }
+ );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Class - An object representing a single class definition.
+
+=head1 CONSTRUCTORS
+
+Clownfish::Class objects are stored as quasi-singletons, one for each
+unique parcel/class_name combination.
+
+=head2 fetch_singleton
+
+ my $class = Clownfish::Class->fetch_singleton(
+ parcel => 'Crustacean',
+ class_name => 'Crustacean::Lobster::LobsterClaw',
+ );
+
+Retrieve a Class, if one has already been created.
+
+=head2 create
+
+ my $class = Clownfish::Class->create(
+ parcel => 'Crustacean', # default: special
+ class_name => 'Crustacean::Lobster::LobsterClaw', # required
+ cnick => 'LobClaw', # default: special
+ exposure => 'public', # default: 'parcel'
+ source_class => undef, # default: same as class_name
+ parent_class_name => 'Crustacean::Claw', # default: undef
+ inert => undef, # default: undef
+ docucomment => $documcom, # default: undef,
+ );
+
+Create and register a quasi-singleton. May only be called once for each
+unique parcel/class_name combination.
+
+=over
+
+=item * B<parcel>, B<class_name>, B<cnick>, B<exposure> - see
+L<Clownfish::Symbol>.
+
+=item * B<source_class> - The name of the class that owns the file in which
+this class was declared. Should be "Foo" if "Foo::FooJr" is defined in
+C<Foo.cfh>.
+
+=item * B<parent_class_name> - The name of this class's parent class. Needed
+in order to establish the class hierarchy.
+
+=item * B<inert> - Should be true if the class is inert, i.e. cannot be
+instantiated.
+
+=item * B<docucomment> - A Clownfish::DocuComment describing this Class.
+
+=back
+
+=head1 METHODS
+
+=head2 get_cnick get_struct_sym get_parent_class_name get_source_class
+get_docucomment get_parent get_autocode inert final
+
+Accessors.
+
+=head2 set_parent
+
+ $class->set_parent($ancestor);
+
+Set the parent class.
+
+=head2 add_child
+
+ $class->add_child($child_class);
+
+Add a child class.
+
+=head2 add_method
+
+ $class->add_method($method);
+
+Add a Method to the class. Valid only before grow_tree() is called.
+
+=head2 add_function
+
+ $class->add_function($function);
+
+Add a Function to the class. Valid only before grow_tree() is called.
+
+=head2 add_member_var
+
+ $class->add_member_var($var);
+
+Add a member variable to the class. Valid only before grow_tree() is called.
+
+=head2 add_inert_var
+
+ $class->add_inert_var($var);
+
+Add an inert (class) variable to the class. Valid only before grow_tree() is
+called.
+
+=head2 add_attribute
+
+ $class->add_attribute($var, $value);
+
+Add an arbitrary attribute to the class.
+
+=head2 function
+
+ my $do_stuff_function = $class->function("do_stuff");
+
+Return the inert Function object for the supplied C<micro_sym>, if any.
+
+=head2 method
+
+ my $pinch_method = $class->method("Pinch");
+
+Return the Method object for the supplied C<micro_sym> / C<macro_sym>, if any.
+
+=head2 novel_method
+
+ my $pinch_method = $class->novel_method("Pinch");
+
+Return a Method object if the Method corresponding to the supplied string is
+novel.
+
+=head2 children
+
+ my $child_classes = $class->children;
+
+Return an array of all child classes.
+
+=head2 functions
+
+ my $functions = $class->functions;
+
+Return an array of all (inert) functions.
+
+=head2 methods
+
+ my $methods = $class->methods;
+
+Return an array of all methods.
+
+=head2 inert_vars
+
+ my $inert_vars = $class->inert_vars;
+
+Return an array of all inert (shared, class) variables.
+
+=head2 member_vars
+
+ my $members = $class->member_vars;
+
+Return an array of all member variables.
+
+=head2 novel_methods
+
+ my $novel_methods = $class->novel_methods;
+
+Return an array of all novel methods.
+
+=head2 novel_member_vars
+
+ my $new_members = $class->novel_member_vars;
+
+Return an array of all novel member variables.
+
+=head2 grow_tree
+
+ $class->grow_tree;
+
+Bequeath all inherited methods and members to children.
+
+=head2 tree_to_ladder
+
+ my $ordered = $class->tree_to_ladder;
+
+Return this class and all its child classes as an array, where all children
+appear after their parent nodes.
+
+=head2 include_h
+
+ my $relative_path = $class->include_h;
+
+Return a relative path to a C header file, appropriately formatted for a
+pound-include directive.
+
+=head2 append_autocode
+
+ $class->append_autocode($code);
+
+Append auxiliary C code.
+
+=head2 short_vtable_var
+
+The short name of the global VTable object for this class.
+
+=head2 full_vtable_var
+
+Fully qualified vtable variable name, including the parcel prefix.
+
+=head2 full_vtable_type
+
+The fully qualified C type specifier for this class's vtable, including the
+parcel prefix. Each vtable needs to have its own type because each has a
+variable number of methods at the end of the struct, and it's not possible to
+initialize a static struct with a flexible array at the end under C89.
+
+=head2 full_struct_sym
+
+Fully qualified struct symbol, including the parcel prefix.
+
+=cut
diff --git a/clownfish/lib/Clownfish/DocuComment.pm b/clownfish/lib/Clownfish/DocuComment.pm
new file mode 100644
index 0000000..4acaab6
--- /dev/null
+++ b/clownfish/lib/Clownfish/DocuComment.pm
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::DocuComment;
+use Clownfish;
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::DocuComment - Formatted comment a la Doxygen.
+
+=head1 SYNOPSIS
+
+ my $text = <<'END_COMMENT';
+ /** Brief description.
+ *
+ * Start the long description. More long description.
+ *
+ * @param foo A Foo.
+ * @param bar A Bar.
+ * @return a return value.
+ */
+ END_COMMENT
+ my $docucomment = Clownfish::DocuComment->parse($text);
+
+=head1 CONSTRUCTORS
+
+=head2 parse
+
+ my $self = Clownfish::DocuComment->parse($text);
+
+Parse comment text.
+
+=head2 new
+
+ my $self = Clownfish::DocuComment->new(
+ description => "Brief. Start long. More long.",
+ brief => "Brief.",
+ long => "Long start. More long.",
+ param_names => \@param_names,
+ param_docs => \@param_docs,
+ retval => "a return value."
+ );
+
+=over
+
+=item * B<description> - The complete description.
+
+=item * B<brief> - The first sentence of the description (a "brief"
+description).
+
+=item * B<long> - The description minus the first sentence.
+
+=item * B<param_names> - An array of param names.
+
+=item * B<param_docs> - An array containing a blurb for each param name.
+
+=item * B<retval> - Return value.
+
+=back
+
+=head1 METHODS
+
+=head2 get_description get_brief get_long get_param_names get_param_docs get_retval
+
+Accessors.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Dumpable.pm b/clownfish/lib/Clownfish/Dumpable.pm
new file mode 100644
index 0000000..c00b91e
--- /dev/null
+++ b/clownfish/lib/Clownfish/Dumpable.pm
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Dumpable;
+use Carp;
+use Clownfish::Class;
+use Clownfish::Type;
+use Clownfish::Method;
+use Clownfish::Variable;
+
+sub new {
+ my $either = shift;
+ my $package = ref($either) || $either;
+ return $either->_new();
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Dumpable - Auto-generate code for "dumpable" classes.
+
+=head1 SYNOPSIS
+
+ my $dumpable = Clownfish::Dumpable->new;
+ for my $class ( grep { $_->has_attribute('dumpable') } @classes ) {
+ $dumpable->add_dumpables($class);
+ }
+
+=head1 DESCRIPTION
+
+If a class declares that it has the attribute "dumpable", but does not declare
+either Dump or Load(), Clownfish::Dumpable will attempt to auto-generate
+those methods if methods inherited from the parent class do not suffice.
+
+ class Foo::Bar inherits Foo : dumpable {
+ Thing *thing;
+
+ public inert incremented Bar*
+ new();
+
+ void
+ Destroy(Bar *self);
+ }
+
+=head1 METHODS
+
+=head2 new
+
+ my $dumpable = Clownfish::Dumpable->new;
+
+Constructor. Takes no arguments.
+
+=head2 add_dumpables
+
+ $dumpable->add_dumpables($dumpable_class);
+
+Analyze a class with the attribute "dumpable" and add Dump() or Load() methods
+as necessary.
+
+=cut
diff --git a/clownfish/lib/Clownfish/File.pm b/clownfish/lib/Clownfish/File.pm
new file mode 100644
index 0000000..995c952
--- /dev/null
+++ b/clownfish/lib/Clownfish/File.pm
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::File;
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+our %new_PARAMS = ( source_class => undef, );
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ my $package = ref($either) || $either;
+ return $either->_new( $args{source_class} );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::File - Structured representation of the contents of a
+Clownfish source file.
+
+=head1 DESCRIPTION
+
+An abstraction representing a file which contains Clownfish code.
+
+=head1 METHODS
+
+=head2 new
+
+ my $file_obj = Clownfish::File->new(
+ source_class => 'Crustacean::Lobster', # required
+ );
+
+=over
+
+=item * B<source_class> - The class name associated with the source file,
+regardless of how what classes are defined in the source file. Example: If
+source_class is "Foo::Bar", that implies that the source file could be found
+at 'Foo/Bar.cfh' within the source directory and that the output C header file
+should be 'Foo/Bar.h' within the target include directory.
+
+=back
+
+=head2 add_block
+
+ $file_obj->add_block($block);
+
+Add an element to the blocks array. The block must be either a
+Clownfish::Class, a Clownfish::Parcel, or a Clownfish::CBlock.
+
+=head2 blocks
+
+ my $blocks = $file->blocks;
+
+Return all blocks as an arrayref.
+
+=head2 classes
+
+ my $classes = $file->classes;
+
+Return all Clownfish::Class blocks from the file as an arrayref.
+
+=head2 get_modified set_modified
+
+Accessors for the file's "modified" property, which is initially false.
+
+=head2 get_source_class
+
+Accessor.
+
+=head2 c_path h_path cfh_path
+
+ # '/path/to/Source/Class.c', etc.
+ my $c_path = $file->c_path('/path/to');
+ my $h_path = $file->h_path('/path/to');
+ my $cfh_path = $file->cfh_path('/path/to');
+
+Given a base directory, return a path name derived from the File's
+source_class with the specified extension.
+
+=head2 guard_name
+
+ # e.g. "H_CRUSTACEAN_LOBSTER"
+ my $guard_name = $file->guard_name
+
+Return a string used for an include guard in a C header, unique per file.
+
+=head2 guard_start
+
+Return a string opening the include guard.
+
+=head2 guard_close
+
+Return a string closing the include guard. Other classes count on being able
+to match this string.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Function.pm b/clownfish/lib/Clownfish/Function.pm
new file mode 100644
index 0000000..f3a83f8
--- /dev/null
+++ b/clownfish/lib/Clownfish/Function.pm
@@ -0,0 +1,122 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Function;
+use base qw( Clownfish::Symbol );
+use Carp;
+use Clownfish::Util qw( verify_args a_isa_b );
+use Clownfish::Type;
+use Clownfish::ParamList;
+
+my %new_PARAMS = (
+ return_type => undef,
+ class_name => undef,
+ class_cnick => undef,
+ param_list => undef,
+ micro_sym => undef,
+ docucomment => undef,
+ parcel => undef,
+ inline => undef,
+ exposure => undef,
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ $args{inline} ||= 0;
+ $args{parcel} = Clownfish::Parcel->acquire( $args{parcel} );
+ my $package = ref($either) || $either;
+ return $package->_new(
+ @args{
+ qw( parcel exposure class_name class_cnick micro_sym
+ return_type param_list docucomment inline )
+ }
+ );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Function - Metadata describing a function.
+
+=head1 METHODS
+
+=head2 new
+
+ my $type = Clownfish::Function->new(
+ class_name => 'Crustacean::Lobster::LobsterClaw', # required
+ class_cnick => 'LobClaw', # default: special
+ return_type => $int_type, # required
+ param_list => $param_list, # required
+ micro_sym => 'compare', # required
+ docucomment => $docucomment, # default: undef
+ parcel => 'Crustacean', # default: special
+ exposure => 'public', # default: parcel
+ inline => 1, # default: false
+ );
+
+=over
+
+=item * B<class_name> - The full name of the class in whose namespace the
+function resides.
+
+=item * B<class_cnick> - The C nickname for the class.
+
+=item * B<return_type> - A L<Clownfish::Type> representing the function's
+return type.
+
+=item * B<param_list> - A L<Clownfish::ParamList> representing the
+function's argument list.
+
+=item * B<micro_sym> - The lower case name of the function, without any
+namespacing prefixes.
+
+=item * B<docucomment> - A L<Clownfish::DocuComment> describing the
+function.
+
+=item * B<parcel> - A L<Clownfish::Parcel> or a parcel name.
+
+=item * B<exposure> - The function's exposure (see L<Clownfish::Symbol>).
+
+=item * B<inline> - Should be true if the function should be inlined by the
+compiler.
+
+=back
+
+=head2 get_return_type get_param_list get_docucomment inline
+
+Accessors.
+
+=head2 void
+
+Returns true if the function has a void return type, false otherwise.
+
+=head2 full_func_sym
+
+A synonym for full_sym().
+
+=head2 short_func_sym
+
+A synonym for short_sym().
+
+=cut
diff --git a/clownfish/lib/Clownfish/Hierarchy.pm b/clownfish/lib/Clownfish/Hierarchy.pm
new file mode 100644
index 0000000..62349a9
--- /dev/null
+++ b/clownfish/lib/Clownfish/Hierarchy.pm
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Hierarchy;
+use Carp;
+
+use Clownfish::Util qw( verify_args );
+use Clownfish::Class;
+use Clownfish::Parser;
+
+our %new_PARAMS = (
+ source => undef,
+ dest => undef,
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ my $package = ref($either) || $either;
+ my $parser = Clownfish::Parser->new;
+ return $package->_new( @args{qw( source dest )}, $parser );
+}
+
+sub _do_parse_file {
+ my ( $parser, $content, $source_class ) = @_;
+ $content = $parser->strip_plain_comments($content);
+ return $parser->file( $content, 0, source_class => $source_class, );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Hierarchy - A class hierarchy.
+
+=head1 DESCRIPTION
+
+A Clownfish::Hierarchy consists of all the classes defined in files within
+a source directory and its subdirectories.
+
+There may be more than one tree within the Hierarchy, since all "inert"
+classes are root nodes, and since Clownfish does not officially define any
+core classes itself from which all instantiable classes must descend.
+
+=head1 METHODS
+
+=head2 new
+
+ my $hierarchy = Clownfish::Hierarchy->new(
+ source => undef, # required
+ dest => undef, # required
+ );
+
+=over
+
+=item * B<source> - The directory we begin reading files from.
+
+=item * B<dest> - The directory where the autogenerated files will be written.
+
+=back
+
+=head2 build
+
+ $hierarchy->build;
+
+Parse every Clownfish header file which can be found under C<source>, building
+up the object hierarchy.
+
+=head2 ordered_classes
+
+ my $classes = $hierarchy->ordered_classes;
+
+Return all Classes as an array with the property that every parent class will
+precede all of its children.
+
+=head2 propagate_modified
+
+ $hierarchy->propagate_modified($modified);
+
+Visit all File objects in the hierarchy. If a parent node is modified, mark
+all of its children as modified.
+
+If the supplied argument is true, mark all Files as modified.
+
+=head2 get_source get_dest
+
+Accessors.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Method.pm b/clownfish/lib/Clownfish/Method.pm
new file mode 100644
index 0000000..7825bee
--- /dev/null
+++ b/clownfish/lib/Clownfish/Method.pm
@@ -0,0 +1,206 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Method;
+use base qw( Clownfish::Function );
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+my %new_PARAMS = (
+ return_type => undef,
+ class_name => undef,
+ class_cnick => undef,
+ param_list => undef,
+ macro_sym => undef,
+ docucomment => undef,
+ parcel => undef,
+ abstract => undef,
+ final => undef,
+ exposure => 'parcel',
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ $args{abstract} ||= 0;
+ $args{parcel} = Clownfish::Parcel->acquire( $args{parcel} );
+ $args{final} ||= 0;
+ my $package = ref($either) || $either;
+ return $package->_new(
+ @args{
+ qw( parcel exposure class_name class_cnick macro_sym
+ return_type param_list docucomment final abstract )
+ }
+ );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Method - Metadata describing an instance method.
+
+=head1 DESCRIPTION
+
+Clownfish::Method is a specialized subclass of Clownfish::Function, with
+the first argument required to be an Obj.
+
+When compiling Clownfish code to C, Method objects generate all the code
+that Function objects do, but also create symbols for indirect invocation via
+VTable.
+
+=head1 METHODS
+
+=head2 new
+
+ my $type = Clownfish::Method->new(
+ parcel => 'Crustacean', # default: special
+ class_name => 'Crustacean::Lobster::LobsterClaw', # required
+ class_cnick => 'LobClaw', # default: special
+ macro_sym => 'Pinch', # required
+ return_type => $void_type, # required
+ param_list => $param_list, # required
+ exposure => undef, # default: 'parcel'
+ docucomment => $docucomment, # default: undef
+ abstract => undef, # default: undef
+ final => 1, # default: undef
+ );
+
+=over
+
+=item * B<param_list> - A Clownfish::ParamList. The first element must be an
+object of the class identified by C<class_name>.
+
+=item * B<macro_sym> - The mixed case name which will be used when invoking the
+method.
+
+=item * B<abstract> - Indicate whether the method is abstract.
+
+=item * B<final> - Indicate whether the method is final.
+
+=item * B<parcel>, B<class_name>, B<class_cnick>, B<return_type>,
+B<docucomment>, - see L<Clownfish::Function>.
+
+=back
+
+=head2 abstract final get_macro_sym
+
+Getters.
+
+=head2 novel
+
+Returns true if the method's class is the first in the inheritance hierarchy
+in which the method was declared -- i.e. the method is neither inherited nor
+overridden.
+
+=head2 self_type
+
+Return the L<Clownfish::Type> for C<self>.
+
+=head2 short_method_sym
+
+ # e.g. "LobClaw_Pinch"
+ my $short_sym = $method->short_method_sym("LobClaw");
+
+Returns the symbol used to invoke the method (minus the parcel Prefix).
+
+=head2 full_method_sym
+
+ # e.g. "Crust_LobClaw_Pinch"
+ my $full_sym = $method->full_method_sym("LobClaw");
+
+Returns the fully-qualified symbol used to invoke the method.
+
+=head2 full_offset_sym
+
+ # e.g. "Crust_LobClaw_Pinch_OFFSET"
+ my $offset_sym = $method->full_offset_sym("LobClaw");
+
+Returns the fully qualified name of the variable which stores the method's
+vtable offset.
+
+=head2 full_callback_sym
+
+ # e.g. "crust_LobClaw_pinch_CALLBACK"
+ my $callback_sym = $method->full_calback_sym("LobClaw");
+
+Returns the fully qualified name of the variable which stores the method's
+Callback object.
+
+=head2 full_override_sym
+
+ # e.g. "crust_LobClaw_pinch_OVERRIDE"
+ my $override_func_sym = $method->full_override_sym("LobClaw");
+
+Returns the fully qualified name of the function which implements the callback
+to the host in the event that a host method has been defined which overrides
+this method.
+
+=head2 short_typedef
+
+ # e.g. "Claw_pinch_t"
+ my $short_typedef = $method->short_typedef;
+
+Returns the typedef symbol for this method, which is derived from the class
+nick of the first class in which the method was declared.
+
+=head2 full_typedef
+
+ # e.g. "crust_Claw_pinch_t"
+ my $full_typedef = $method->full_typedef;
+
+Returns the fully-qualified typedef symbol including parcel prefix.
+
+=head2 override
+
+ $method->override($method_being_overridden);
+
+Let the Method know that it is overriding a method which was defined in a
+parent class, and verify that the override is valid.
+
+All methods start out believing that they are "novel", because we don't know
+about inheritance until we build the hierarchy after all files have been
+parsed. override() is a way of going back and relabeling a method as
+overridden when new information has become available: in this case, that a
+parent class has defined a method with the same name.
+
+=head2 finalize
+
+ my $final_method = $method->finalize;
+
+As with override, above, this is for going back and changing the nature of a
+Method after new information has become available -- typically, when we
+discover that the method has been inherited by a "final" class.
+
+However, we don't modify the original Method as with override(). Inherited
+Method objects are shared between parent and child classes; if a shared Method
+object were to become final, it would interfere with its own inheritance. So,
+we make a copy, slightly modified to indicate that it is "final".
+
+=head2 compatible
+
+ confess("Can't override") unless $method->compatible($other);
+
+Returns true if the methods have signatures and attributes which allow
+one to override the other.
+
+=cut
diff --git a/clownfish/lib/Clownfish/ParamList.pm b/clownfish/lib/Clownfish/ParamList.pm
new file mode 100644
index 0000000..12e53ae
--- /dev/null
+++ b/clownfish/lib/Clownfish/ParamList.pm
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::ParamList;
+use Clownfish::Variable;
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+our %new_PARAMS = ( variadic => undef, );
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ my $class_name = ref($either) || $either;
+ my $variadic = delete $args{variadic} || 0;
+ return $class_name->_new($variadic);
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::ParamList - parameter list.
+
+=head1 DESCRIPTION
+
+=head1 METHODS
+
+=head2 new
+
+ my $param_list = Clownfish::ParamList->new(
+ variadic => 1, # default: false
+ );
+
+=over
+
+=item * B<variadic> - Should be true if the function is variadic.
+
+=back
+
+=head2 add_param
+
+ $param_list->add_param( $variable, $value );
+
+Add a parameter to the ParamList.
+
+=over
+
+=item * B<variable> - A L<Clownfish::Variable>.
+
+=item * B<value> - The default value for the parameter, which should be undef
+if there is no such value and the parameter is required.
+
+=back
+
+=head2 get_variables get_initial_values variadic
+
+Accessors.
+
+=head2 num_vars
+
+Return the number of variables in the ParamList, including "self" for methods.
+
+=head2 to_c
+
+ # Prints "Obj* self, Foo* foo, Bar* bar".
+ print $param_list->to_c;
+
+Return a list of the variable's types and names, joined by commas.
+
+=head2 name_list
+
+ # Prints "self, foo, bar".
+ print $param_list->name_list;
+
+Return the variable's names, joined by commas.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Parcel.pm b/clownfish/lib/Clownfish/Parcel.pm
new file mode 100644
index 0000000..bef7b8e
--- /dev/null
+++ b/clownfish/lib/Clownfish/Parcel.pm
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Parcel;
+use base qw( Exporter );
+use Clownfish;
+use Clownfish::Util qw( verify_args );
+use Scalar::Util qw( blessed );
+use Carp;
+
+END {
+ __PACKAGE__->reap_singletons();
+}
+
+our %singleton_PARAMS = (
+ name => undef,
+ cnick => undef,
+);
+
+sub singleton {
+ my ( $either, %args ) = @_;
+ verify_args( \%singleton_PARAMS, %args ) or confess $@;
+ my $package = ref($either) || $either;
+ return $package->_singleton( @args{qw( name cnick )} );
+}
+
+sub acquire {
+ my ( undef, $thing ) = @_;
+ if ( !defined $thing ) {
+ return Clownfish::Parcel->default_parcel;
+ }
+ elsif ( blessed($thing) ) {
+ confess("Not a Clownfish::Parcel")
+ unless $thing->isa('Clownfish::Parcel');
+ return $thing;
+ }
+ else {
+ return Clownfish::Parcel->singleton( name => $thing );
+ }
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Parcel - Collection of code.
+
+=head1 DESCRIPTION
+
+A Parcel is a cohesive collection of code, which could, in theory, be
+published as as a single entity.
+
+Clownfish supports two-tier manual namespacing, using a prefix, an optional
+class nickname, and the local symbol:
+
+ prefix_ClassNick_local_symbol
+
+Clownfish::Parcel supports the first tier, specifying initial prefixes.
+These prefixes come in three capitalization variants: prefix_, Prefix_, and
+PREFIX_.
+
+=head1 CLASS METHODS
+
+=head2 singleton
+
+ Clownfish::Parcel->singleton(
+ name => 'Crustacean',
+ cnick => 'Crust',
+ );
+
+Add a Parcel singleton to a global registry. May be called multiple times,
+but only with compatible arguments.
+
+=over
+
+=item *
+
+B<name> - The name of the parcel.
+
+=item *
+
+B<cnick> - The C nickname for the parcel, which will be used as a prefix for
+generated global symbols. Must be mixed case and start with a capital letter.
+Defaults to C<name>.
+
+=back
+
+=head2 default_parcel
+
+ $parcel ||= Clownfish::Parcel->default_parcel;
+
+Return the singleton for default parcel, which has no prefix.
+
+=head1 OBJECT METHODS
+
+=head2 get_prefix get_Prefix get_PREFIX
+
+Return one of the three capitalization variants for the parcel's prefix.
+
+=head2 acquire
+
+ $parcel = Clownfish::Parcel->aquire($parcel_name_or_parcel_object);
+
+Aquire a parcel one way or another. If the supplied argument is a Parcel,
+return it. If it's not defined, return the default Parcel. If it's a name,
+invoke singleton().
+
+=head2 get_name get_cnick
+
+Accessors.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Parser.pm b/clownfish/lib/Clownfish/Parser.pm
new file mode 100644
index 0000000..6c11a5d
--- /dev/null
+++ b/clownfish/lib/Clownfish/Parser.pm
@@ -0,0 +1,565 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Parser;
+use base qw( Parse::RecDescent );
+
+use Clownfish::Parcel;
+use Clownfish::Type;
+use Clownfish::Variable;
+use Clownfish::DocuComment;
+use Clownfish::Function;
+use Clownfish::Method;
+use Clownfish::Class;
+use Clownfish::CBlock;
+use Clownfish::File;
+use Carp;
+
+our $grammar = <<'END_GRAMMAR';
+
+file:
+ { Clownfish::Parser->set_parcel(undef); 0; }
+ major_block[%arg](s) eofile
+ { Clownfish::Parser->new_file( \%item, \%arg ) }
+
+major_block:
+ class_declaration[%arg]
+ | embed_c
+ | parcel_definition
+
+parcel_definition:
+ 'parcel' class_name cnick(?) ';'
+ {
+ my $parcel = Clownfish::Parser->new_parcel( \%item );
+ Clownfish::Parser->set_parcel($parcel);
+ $parcel;
+ }
+
+embed_c:
+ '__C__'
+ /.*?(?=__END_C__)/s
+ '__END_C__'
+ { Clownfish::CBlock->new( contents => $item[2] ) }
+
+class_declaration:
+ docucomment(?)
+ exposure_specifier(?) class_modifier(s?) 'class' class_name
+ cnick(?)
+ class_inheritance(?)
+ class_attribute(s?)
+ '{'
+ declaration_statement[
+ class => $item{class_name},
+ cnick => $item{'cnick(?)'}[0],
+ parent => $item{'class_inheritance(?)'}[0],
+ ](s?)
+ '}'
+ { Clownfish::Parser->new_class( \%item, \%arg ) }
+
+class_modifier:
+ 'inert'
+ | 'abstract'
+ | 'final'
+ { $item[1] }
+
+class_inheritance:
+ 'inherits' class_name
+ { $item[2] }
+
+class_attribute:
+ ':' /[a-z]+(?!\w)/
+ { $item[2] }
+
+class_name:
+ object_type_specifier ( "::" object_type_specifier )(s?)
+ { join('::', $item[1], @{ $item[2] } ) }
+
+cnick:
+ 'cnick'
+ /([A-Z][A-Za-z0-9]+)(?!\w)/
+ { $1 }
+
+declaration_statement:
+ var_declaration_statement[%arg]
+ | subroutine_declaration_statement[%arg]
+ | <error>
+
+var_declaration_statement:
+ exposure_specifier(?) variable_modifier(s?) type declarator ';'
+ {
+ $return = {
+ exposure => $item[1][0] || 'parcel',
+ modifiers => $item[2],
+ declared => Clownfish::Parser->new_var( \%item, \%arg ),
+ };
+ }
+
+subroutine_declaration_statement:
+ docucomment(?)
+ exposure_specifier(?)
+ subroutine_modifier(s?)
+ type
+ declarator
+ param_list
+ ';'
+ {
+ $return = {
+ exposure => $item[2],
+ modifiers => $item[3],
+ declared => Clownfish::Parser->new_sub( \%item, \%arg ),
+ };
+ }
+
+param_list:
+ '('
+ param_list_elem(s? /,/)
+ (/,\s*.../)(?)
+ ')'
+ {
+ Clownfish::Parser->new_param_list( $item[2], $item[3][0] ? 1 : 0 );
+ }
+
+param_list_elem:
+ param_variable assignment(?)
+ { [ $item[1], $item[2][0] ] }
+
+param_variable:
+ type declarator
+ { Clownfish::Parser->new_var(\%item); }
+
+assignment:
+ '=' scalar_constant
+ { $item[2] }
+
+type:
+ nullable(?) simple_type type_postfix(s?)
+ { Clownfish::Parser->simple_or_composite_type(\%item) }
+
+nullable:
+ 'nullable'
+
+simple_type:
+ object_type
+ | primitive_type
+ | void_type
+ | va_list_type
+ | arbitrary_type
+ { $item[1] }
+
+object_type:
+ type_qualifier(s?) object_type_specifier '*'
+ { Clownfish::Parser->new_object_type(\%item); }
+
+primitive_type:
+ c_integer_type
+ | chy_integer_type
+ | float_type
+ { $item[1] }
+
+c_integer_type:
+ type_qualifier(s?) c_integer_specifier
+ { Clownfish::Parser->new_integer_type(\%item) }
+
+chy_integer_type:
+ type_qualifier(s?) chy_integer_specifier
+ { Clownfish::Parser->new_integer_type(\%item) }
+
+float_type:
+ type_qualifier(s?) c_float_specifier
+ { Clownfish::Parser->new_float_type(\%item) }
+
+void_type:
+ type_qualifier(s?) void_type_specifier
+ { Clownfish::Parser->new_void_type(\%item) }
+
+va_list_type:
+ va_list_type_specifier
+ { Clownfish::Type->new_va_list }
+
+arbitrary_type:
+ arbitrary_type_specifier
+ { Clownfish::Parser->new_arbitrary_type(\%item); }
+
+type_qualifier:
+ 'const'
+ | 'incremented'
+ | 'decremented'
+ | 'nullable'
+
+subroutine_modifier:
+ 'inert'
+ | 'inline'
+ | 'abstract'
+ | 'final'
+ { $item[1] }
+
+exposure_specifier:
+ 'public'
+ | 'private'
+ | 'parcel'
+ | 'local'
+
+variable_modifier:
+ 'inert'
+ { $item[1] }
+
+type_specifier:
+ ( object_type_specifier
+ | primitive_type_specifier
+ | void_type_specifier
+ | va_list_type_specifier
+ | arbitrary_type_specifier
+ )
+ { $item[1] }
+
+primitive_type_specifier:
+ chy_integer_specifier
+ | c_integer_specifier
+ | c_float_specifier
+ { $item[1] }
+
+chy_integer_specifier:
+ /(?:chy_)?(bool)_t(?!\w)/
+
+c_integer_specifier:
+ /(?:(?:u?int(?:8|16|32|64)_t)|(?:char|int|short|long|size_t))(?!\w)/
+
+c_float_specifier:
+ /(?:float|double)(?!\w)/
+
+void_type_specifier:
+ /void(?!\w)/
+
+va_list_type_specifier:
+ /va_list(?!\w)/
+
+arbitrary_type_specifier:
+ /\w+_t(?!\w)/
+
+object_type_specifier:
+ /([a-z]+[a-z0-9]*_)?[A-Z]+[A-Z0-9]*[a-z]+[A-Za-z0-9]*(?!\w)/
+
+declarator:
+ identifier
+ { $item[1] }
+
+type_postfix:
+ '*'
+ { '*' }
+ | '[' ']'
+ { '[]' }
+ | '[' constant_expression ']'
+ { "[$item[2]]" }
+
+identifier:
+ ...!reserved_word /[a-zA-Z_]\w*/x
+ { $item[2] }
+
+docucomment:
+ /\/\*\*.*?\*\//s
+ { Clownfish::DocuComment->parse($item[1]) }
+
+constant_expression:
+ /\d+/
+ | /[A-Z_]+/
+
+scalar_constant:
+ hex_constant
+ | float_constant
+ | integer_constant
+ | string_literal
+ | 'NULL'
+ | 'true'
+ | 'false'
+
+integer_constant:
+ /(?:-\s*)?\d+/
+ { $item[1] }
+
+hex_constant:
+ /0x[a-fA-F0-9]+/
+ { $item[1] }
+
+float_constant:
+ /(?:-\s*)?\d+\.\d+/
+ { $item[1] }
+
+string_literal:
+ /"(?:[^"\\]|\\.)*"/
+ { $item[1] }
+
+reserved_word:
+ /(const|double|enum|extern|float|register|signed|sizeof
+ |inert|struct|typedef|union|unsigned|void)(?!\w)/x
+ | chy_integer_specifier
+ | c_integer_specifier
+
+eofile:
+ /^\Z/
+
+END_GRAMMAR
+
+sub new { return shift->SUPER::new($grammar) }
+
+sub strip_plain_comments {
+ my ( $self, $text ) = @_;
+ while ( $text =~ m#(/\*[^*].*?\*/)#ms ) {
+ my $blanked = $1;
+ $blanked =~ s/\S/ /g;
+ $text =~ s#/\*[^*].*?\*/#$blanked#ms;
+ }
+ return $text;
+}
+
+our $parcel = undef;
+sub set_parcel { $parcel = $_[1] }
+
+sub new_integer_type {
+ my ( undef, $item ) = @_;
+ my $specifier = $item->{c_integer_specifier}
+ || $item->{chy_integer_specifier};
+ my %args = ( specifier => $specifier );
+ $args{$_} = 1 for @{ $item->{'type_qualifier(s?)'} };
+ return Clownfish::Type->new_integer(%args);
+}
+
+sub new_float_type {
+ my ( undef, $item ) = @_;
+ my %args = ( specifier => $item->{c_float_specifier} );
+ $args{$_} = 1 for @{ $item->{'type_qualifier(s?)'} };
+ return Clownfish::Type->new_float(%args);
+}
+
+sub new_void_type {
+ my ( undef, $item ) = @_;
+ my %args;
+ $args{$_} = 1 for @{ $item->{'type_qualifier(s?)'} };
+ return Clownfish::Type->new_void(%args);
+}
+
+sub new_arbitrary_type {
+ my ( undef, $item ) = @_;
+ return Clownfish::Type->new_arbitrary(
+ specifier => $item->{arbitrary_type_specifier},
+ parcel => $parcel,
+ );
+}
+
+sub new_object_type {
+ my ( undef, $item ) = @_;
+ my %args = (
+ specifier => $item->{object_type_specifier},
+ parcel => $parcel,
+ );
+ $args{$_} = 1 for @{ $item->{'type_qualifier(s?)'} };
+ return Clownfish::Type->new_object(%args);
+}
+
+sub simple_or_composite_type {
+ my ( undef, $item ) = @_;
+ my $simple_type = $item->{simple_type};
+ my $postfixes = $item->{'type_postfix(s?)'};
+ my $nullable = scalar @{ $item->{'nullable(?)'} } ? 1 : undef;
+ my $type;
+
+ if ( !@$postfixes ) {
+ if ($nullable) {
+ my $type_class = ref($simple_type);
+ confess "$type_class can't be 'nullable'"
+ unless $simple_type->is_object;
+ $simple_type->set_nullable($nullable);
+ }
+ return $simple_type;
+ }
+ else {
+ my %args = (
+ child => $simple_type,
+ indirection => 0,
+ nullable => $nullable,
+ );
+ for my $postfix (@$postfixes) {
+ if ( $postfix =~ /\[/ ) {
+ $args{array} ||= '';
+ $args{array} .= $postfix;
+ }
+ elsif ( $postfix eq '*' ) {
+ $args{indirection}++;
+ }
+ }
+ return Clownfish::Type->new_composite(%args);
+ }
+}
+
+sub new_var {
+ my ( undef, $item, $arg ) = @_;
+ my $exposure = $item->{'exposure_specifier(?)'}[0];
+ my %args = $exposure ? ( exposure => $exposure ) : ();
+ if ($arg) {
+ $args{class_name} = $arg->{class} if $arg->{class};
+ $args{class_cnick} = $arg->{cnick} if $arg->{cnick};
+ }
+ return Clownfish::Variable->new(
+ parcel => $parcel,
+ type => $item->{type},
+ micro_sym => $item->{declarator},
+ %args,
+ );
+}
+
+sub new_param_list {
+ my ( undef, $param_list_elems, $variadic ) = @_;
+ my $param_list = Clownfish::ParamList->new( variadic => $variadic, );
+ for my $param (@$param_list_elems) {
+ $param_list->add_param( $param->[0], $param->[1] );
+ }
+ return $param_list;
+}
+
+sub new_sub {
+ my ( undef, $item, $arg ) = @_;
+ my $class;
+ my $modifiers = $item->{'subroutine_modifier(s?)'};
+ my $docucom = $item->{'docucomment(?)'}[0];
+ my $exposure = $item->{'exposure_specifier(?)'}[0];
+ my $inert = scalar grep { $_ eq 'inert' } @$modifiers;
+ my %extra_args = $exposure ? ( exposure => $exposure ) : ();
+
+ if ($inert) {
+ $class = 'Clownfish::Function';
+ $extra_args{micro_sym} = $item->{declarator};
+ $extra_args{inline} = scalar grep { $_ eq 'inline' } @$modifiers;
+ }
+ else {
+ $class = 'Clownfish::Method';
+ $extra_args{macro_sym} = $item->{declarator};
+ $extra_args{abstract} = scalar grep { $_ eq 'abstract' } @$modifiers;
+ $extra_args{final} = scalar grep { $_ eq 'final' } @$modifiers;
+ }
+
+ return $class->new(
+ parcel => $parcel,
+ docucomment => $docucom,
+ class_name => $arg->{class},
+ class_cnick => $arg->{cnick},
+ return_type => $item->{type},
+ param_list => $item->{param_list},
+ %extra_args,
+ );
+}
+
+sub new_class {
+ my ( undef, $item, $arg ) = @_;
+ my ( @member_vars, @inert_vars, @functions, @methods );
+ my $source_class = $arg->{source_class} || $item->{class_name};
+ my %class_modifiers
+ = map { ( $_ => 1 ) } @{ $item->{'class_modifier(s?)'} };
+ my %class_attributes
+ = map { ( $_ => 1 ) } @{ $item->{'class_attribute(s?)'} };
+
+ for my $declaration ( @{ $item->{'declaration_statement(s?)'} } ) {
+ my $declared = $declaration->{declared};
+ my $exposure = $declaration->{exposure};
+ my $modifiers = $declaration->{modifiers};
+ my $inert = ( scalar grep {/inert/} @$modifiers ) ? 1 : 0;
+ my $subs = $inert ? \@functions : \@methods;
+ my $vars = $inert ? \@inert_vars : \@member_vars;
+
+ if ( $declared->isa('Clownfish::Variable') ) {
+ push @$vars, $declared;
+ }
+ else {
+ push @$subs, $declared;
+ }
+ }
+
+ my $class = Clownfish::Class->create(
+ parcel => $parcel,
+ class_name => $item->{class_name},
+ cnick => $item->{'cnick(?)'}[0],
+ parent_class_name => $item->{'class_inheritance(?)'}[0],
+ docucomment => $item->{'docucomment(?)'}[0],
+ source_class => $source_class,
+ inert => $class_modifiers{inert},
+ final => $class_modifiers{final},
+ );
+ $class->add_method($_) for @methods;
+ $class->add_function($_) for @functions;
+ $class->add_member_var($_) for @member_vars;
+ $class->add_inert_var($_) for @inert_vars;
+ while ( my ( $var, $val ) = each %class_attributes ) {
+ $class->add_attribute( $var, $val );
+ }
+ return $class;
+}
+
+sub new_file {
+ my ( undef, $item, $arg ) = @_;
+ my $file = Clownfish::File->new( source_class => $arg->{source_class}, );
+ for my $block ( @{ $item->{'major_block(s)'} } ) {
+ $file->add_block($block);
+ }
+ return $file;
+}
+
+sub new_parcel {
+ my ( undef, $item ) = @_;
+ Clownfish::Parcel->singleton(
+ name => $item->{class_name},
+ cnick => $item->{'cnick(?)'}[0],
+ );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Parser - Parse Clownfish header files.
+
+=head1 SYNOPSIS
+
+ my $class_def = $parser->class($class_text);
+
+=head1 DESCRIPTION
+
+Clownfish::Parser is a combined lexer/parser which parses Clownfish header
+files. It is not at all strict, as it relies heavily on the C parser to pick
+up errors such as misspelled type names.
+
+=head1 METHODS
+
+=head2 new
+
+Constructor, takes no arguments.
+
+=head2 strip_plain_comments
+
+ my $stripped = $parser->strip_plain_comments($code_with_comments);
+
+Remove plain C comments from supplied code. All non-whitespace characters are
+turned to spaces; all whitespace characters are preserved, so that the number
+of lines is consistent between before and after.
+
+JavaDoc-syntax "DocuComments", which begin with "/**" are left alone.
+
+This is a sloppy implementation which will mangle quoted comments and such.
+
+=cut
diff --git a/clownfish/lib/Clownfish/Symbol.pm b/clownfish/lib/Clownfish/Symbol.pm
new file mode 100644
index 0000000..3b37b93
--- /dev/null
+++ b/clownfish/lib/Clownfish/Symbol.pm
@@ -0,0 +1,126 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Symbol;
+use Clownfish;
+use Clownfish::Parcel;
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+my %new_PARAMS = (
+ parcel => undef,
+ exposure => undef,
+ class_name => undef,
+ class_cnick => undef,
+ micro_sym => undef,
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ $args{parcel} = Clownfish::Parcel->acquire( $args{parcel} );
+ my $class_class = ref($either) || $either;
+ return $class_class->_new(
+ @args{qw( parcel exposure class_name class_cnick micro_sym )} );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Symbol - Base class for Clownfish symbols.
+
+=head1 DESCRIPTION
+
+Clownfish::Symbol serves as a parent class for entities which may live in the
+global namespace, such as classes, functions, methods, and variables.
+
+=head1 CONSTRUCTOR
+
+ my $symbol = Clownfish::Symbol->new(
+ parcel => 'Crustacean', # default: special
+ exposure => 'parcel', # required
+ class_name => 'Crustacean::Lobster', # default: undef
+ class_cnick => undef, # default: special
+ micro_sym => 'average_lifespan', # required
+ );
+
+=over
+
+=item * B<parcel> - A Clownfish::Parcel, or a string that can be used to
+create/retrieve one. If not supplied, will be assigned to the default Parcel.
+
+=item * B<exposure> - The scope in which the symbol is exposed. Must be
+'public', 'parcel', 'private', or 'local'.
+
+=item * B<class_name> - A optional class name, consisting of one or more
+components separated by "::". Each component must start with a capital
+letter, contain at least one lower-case letter, and consist entirely of the
+characters [A-Za-z0-9].
+
+=item * B<class_cnick> - The C nickname associated with the supplied class
+name. If not supplied, will be derived if possible from C<class_name> by
+extracting the last class name component.
+
+=item * B<micro_sym> - The local identifier for the symbol.
+
+=back
+
+=head1 OBJECT METHODS
+
+=head2 get_parcel get_class_name get_class_cnick get_exposure micro_sym
+
+Getters.
+
+=head2 get_prefix get_Prefix get_PREFIX
+
+Get a string prefix, delegating to C<parcel> member var.
+
+=head2 public parcel private local
+
+ if ( $sym->public ) { do_x() }
+ elsif ( $sym->parcel ) { do_y() }
+
+Indicate whether the symbol matches a given access level.
+
+=head2 equals
+
+ do_stuff() if $sym->equals($other_sym);
+
+Returns true if the symbols are "equal", false otherwise.
+
+=head2 short_sym
+
+ # e.g. "Lobster_average_lifespan"
+ print $symbol->short_sym;
+
+Returns the C representation for the symbol minus the parcel's prefix.
+
+=head2 full_sym
+
+ # e.g. "crust_Lobster_average_lifespan"
+ print $symbol->full_sym;
+
+Returns the fully qualified C representation for the symbol.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Type.pm b/clownfish/lib/Clownfish/Type.pm
new file mode 100644
index 0000000..11f8913
--- /dev/null
+++ b/clownfish/lib/Clownfish/Type.pm
@@ -0,0 +1,502 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Type;
+use Clownfish;
+use Clownfish::Parcel;
+use Clownfish::Util qw( verify_args a_isa_b );
+use Scalar::Util qw( blessed );
+use Carp;
+
+our %new_PARAMS = (
+ const => undef,
+ specifier => undef,
+ indirection => undef,
+ parcel => undef,
+ c_string => undef,
+ void => undef,
+ object => undef,
+ primitive => undef,
+ integer => undef,
+ floating => undef,
+ string_type => undef,
+ va_list => undef,
+ arbitrary => undef,
+ composite => undef,
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $package = ref($either) || $either;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+
+ my $flags = 0;
+ $flags |= CONST if $args{const};
+ $flags |= NULLABLE if $args{nullable};
+ $flags |= VOID if $args{void};
+ $flags |= OBJECT if $args{object};
+ $flags |= PRIMITIVE if $args{primitive};
+ $flags |= INTEGER if $args{integer};
+ $flags |= FLOATING if $args{floating};
+ $flags |= STRING_TYPE if $args{string_type};
+ $flags |= VA_LIST if $args{va_list};
+ $flags |= ARBITRARY if $args{arbitrary};
+ $flags |= COMPOSITE if $args{composite};
+
+ my $parcel
+ = $args{parcel}
+ ? Clownfish::Parcel->acquire( $args{parcel} )
+ : $args{parcel};
+
+ my $indirection = $args{indirection} || 0;
+ my $specifier = $args{specifier} || '';
+ my $c_string = $args{c_string} || '';
+
+ return $package->_new( $flags, $parcel, $specifier, $indirection,
+ $c_string );
+}
+
+our %new_integer_PARAMS = (
+ const => undef,
+ specifier => undef,
+);
+
+sub new_integer {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_integer_PARAMS, %args ) or confess $@;
+ my $flags = 0;
+ $flags |= CONST if $args{const};
+ my $package = ref($either) || $either;
+ return $package->_new_integer( $flags, $args{specifier} );
+}
+
+our %new_float_PARAMS = (
+ const => undef,
+ specifier => undef,
+);
+
+sub new_float {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_float_PARAMS, %args ) or confess $@;
+ my $flags = 0;
+ $flags |= CONST if $args{const};
+ my $package = ref($either) || $either;
+ return $package->_new_float( $flags, $args{specifier} );
+}
+
+our %new_object_PARAMS = (
+ const => undef,
+ specifier => undef,
+ indirection => 1,
+ parcel => undef,
+ incremented => 0,
+ decremented => 0,
+ nullable => 0,
+);
+
+sub new_object {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_object_PARAMS, %args ) or confess $@;
+ my $flags = 0;
+ $flags |= INCREMENTED if $args{incremented};
+ $flags |= DECREMENTED if $args{decremented};
+ $flags |= NULLABLE if $args{nullable};
+ $flags |= CONST if $args{const};
+ $args{indirection} = 1 unless defined $args{indirection};
+ my $parcel = Clownfish::Parcel->acquire( $args{parcel} );
+ my $package = ref($either) || $either;
+ confess("Missing required param 'specifier'")
+ unless defined $args{specifier};
+ return $package->_new_object( $flags, $parcel, $args{specifier},
+ $args{indirection} );
+}
+
+our %new_composite_PARAMS = (
+ child => undef,
+ indirection => undef,
+ array => undef,
+ nullable => undef,
+);
+
+sub new_composite {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_composite_PARAMS, %args ) or confess $@;
+ my $flags = 0;
+ $flags |= NULLABLE if $args{nullable};
+ my $indirection = $args{indirection} || 0;
+ my $array = defined $args{array} ? $args{array} : "";
+ my $package = ref($either) || $either;
+ return $package->_new_composite( $flags, $args{child}, $indirection,
+ $array );
+}
+
+our %new_void_PARAMS = ( const => undef, );
+
+sub new_void {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_void_PARAMS, %args ) or confess $@;
+ my $package = ref($either) || $either;
+ return $package->_new_void( !!$args{const} );
+}
+
+sub new_va_list {
+ my $either = shift;
+ verify_args( {}, @_ ) or confess $@;
+ my $package = ref($either) || $either;
+ return $either->_new_va_list();
+}
+
+our %new_arbitrary_PARAMS = (
+ parcel => undef,
+ specifier => undef,
+);
+
+sub new_arbitrary {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_arbitrary_PARAMS, %args ) or confess $@;
+ my $package = ref($either) || $either;
+ my $parcel = Clownfish::Parcel->acquire( $args{parcel} );
+ return $package->_new_arbitrary( $parcel, $args{specifier} );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Type - A variable's type.
+
+=head1 METHODS
+
+=head2 new
+
+ my $type = MyType->new(
+ specifier => 'char', # default undef
+ indirection => undef, # default 0
+ const => 1, # default undef
+ parcel => undef, # default undef
+ c_string => undef, # default undef
+ );
+
+Generic constructor.
+
+=over
+
+=item *
+
+B<specifier> - The C name for the type, not including any indirection or array
+subscripts.
+
+=item *
+
+B<indirection> - integer indicating level of indirection. Example: the C type
+"float**" has a specifier of "float" and indirection 2.
+
+=item *
+
+B<const> - should be true if the type is const.
+
+=item *
+
+B<parcel> - A Clownfish::Parcel or a parcel name.
+
+=item *
+
+B<c_string> - The C representation of the type.
+
+=back
+
+=head2 new_integer
+
+ my $type = Clownfish::Type->new_integer(
+ const => 1, # default: undef
+ specifier => 'char', # required
+ );
+
+Return a Type representing an integer primitive.
+
+Support is limited to a subset of the standard C integer types:
+
+ int8_t
+ int16_t
+ int32_t
+ int64_t
+ uint8_t
+ uint16_t
+ uint32_t
+ uint64_t
+ char
+ short
+ int
+ long
+ size_t
+
+Many others are not supported: "signed" or "unsigned" anything, "long long",
+"ptrdiff_t", "off_t", etc.
+
+The following Charmonizer typedefs are supported:
+
+ bool_t
+
+=over
+
+=item * B<const> - Should be true if the type is const.
+
+=item * B<specifier> - Must match one of the supported types.
+
+=back
+
+=head2 new_float
+
+ my $type = Clownfish::Type->new_float(
+ const => 1, # default: undef
+ specifier => 'double', # required
+ );
+
+Return a Type representing a floating point primitive.
+
+Two specifiers are supported:
+
+ float
+ double
+
+=over
+
+=item * B<const> - Should be true if the type is const.
+
+=item * B<specifier> - Must match one of the supported types.
+
+=back
+
+=cut
+
+=head2 new_composite
+
+ my $type = Clownfish::Type->new_composite(
+ child => $char_type, # required
+ indirection => undef, # default 0
+ array => '[]', # default undef,
+ const => 1, # default undef
+ );
+
+Constructor for a composite type which is made up of repetitions of a single,
+uniform subtype.
+
+=over
+
+=item *
+
+B<child> - The Type which the composite is comprised of.
+
+=item *
+
+B<indirection> - integer indicating level of indirection. Example: the C type
+"float**" has indirection 2.
+
+=item *
+
+B<array> - A string describing an array postfix.
+
+=item *
+
+B<const> - should be 1 if the type is const.
+
+=back
+
+=head2 new_object
+
+ my $type = Clownfish::Type->new_object(
+ specifier => "Lobster", # required
+ parcel => "Crustacean", # default: the default Parcel.
+ const => undef, # default undef
+ indirection => 1, # default 1
+ incremented => 1, # default 0
+ decremented => 0, # default 0
+ nullable => 1, # default 0
+ );
+
+Create a Type representing an object. The Type's C<specifier> must match the
+last component of the class name -- i.e. for the class "Crustacean::Lobster"
+it must be "Lobster".
+
+=over
+
+=item * B<specifier> - Required. Must follow the rules for
+L<Clownfish::Class> class name components.
+
+=item * B<parcel> - A L<Clownfish::Parcel> or a parcel name.
+
+=item * B<const> - Should be true if the Type is const. Note that this refers
+to the object itself and not the pointer.
+
+=item * B<indirection> - Level of indirection. Must be 1 if supplied.
+
+=item * B<incremented> - Indicate whether the caller must take responsibility
+for an added refcount.
+
+=item * B<decremented> - Indicate whether the caller must account for
+for a refcount decrement.
+
+=item * B<nullable> - Indicate whether the object specified by this type may
+be NULL.
+
+=back
+
+The Parcel's prefix will be prepended to the specifier by new_object().
+
+=head2 new_void
+
+ my $type = Clownfish::Type->new_void(
+ specifier => 'void', # default: void
+ const => 1, # default: undef
+ );
+
+Return a Type representing a the 'void' keyword in C. It can be used either
+for a void return type, or in conjuction with with new_composite() to support
+the C<void*> opaque pointer type.
+
+=over
+
+=item * B<specifier> - Must be "void" if supplied.
+
+=item * B<const> - Should be true if the type is const. (Useful in the
+context of C<const void*>).
+
+=back
+
+=head2 new_va_list
+
+ my $type = Clownfish::Type->new_va_list(
+ specifier => 'va_list', # default: va_list
+ );
+
+Create a Type representing C's va_list, from stdarg.h.
+
+=over
+
+=item * B<specifier>. Must be "va_list" if supplied.
+
+=back
+
+=head2 new_arbitrary
+
+ my $type = Clownfish::Type->new_arbitrary(
+ specifier => 'floatint_t', # required
+ parcel => 'Crustacean', # default: undef
+ );
+
+"Arbitrary" types are a hack that spares us from having to support C types
+with complex declaration syntaxes -- such as unions, structs, enums, or
+function pointers -- from within Clownfish itself.
+
+The only constraint is that the C<specifier> must end in "_t". This allows us
+to create complex types in a C header file...
+
+ typedef union { float f; int i; } floatint_t;
+
+... pound-include the C header, then use the resulting typedef in a Clownfish
+header file and have it parse as an "arbitrary" type.
+
+ floatint_t floatint;
+
+=over
+
+=item * B<specifier> - The name of the type, which must end in "_t".
+
+=item * B<parcel> - A L<Clownfish::Parcel> or a parcel name.
+
+=back
+
+If C<parcel> is supplied and C<specifier> begins with a capital letter, the
+Parcel's prefix will be prepended to the specifier:
+
+ foo_t -> foo_t # no prefix prepending
+ Lobster_foo_t -> crust_Lobster_foo_t # prefix prepended
+
+=cut
+
+=head2 equals
+
+ do_stuff() if $type->equals($other);
+
+Returns true if two Clownfish::Type objects are equivalent.
+
+=head2 similar
+
+ do_stuff() if $type->similar($other_type);
+
+Weak checking of type which allows for covariant return types. Calling this
+method on anything other than an object type is an error.
+
+=head2 to_c
+
+ # Declare variable "foo".
+ print $type->to_c . " foo;\n";
+
+Return the C representation of the type.
+
+=head2 set_c_string
+
+Set the C representation of the type.
+
+=head2 get_specifier get_parcel get_indirection get_array const nullable set_specifier set_nullable
+
+Accessors.
+
+=head2 is_object is_primitive is_integer is_floating is_composite is_void
+
+ do_stuff() if $type->is_object;
+
+Identify the flavor of Type, which is determined by the constructor which was
+used to create it.
+
+=over
+
+=item * is_object: Clownfish::Type->new_object
+
+=item * is_primitive: Either Clownfish::Type->new_integer or
+Clownfish::Type->new_float
+
+=item * is_integer: Clownfish::Type->new_integer
+
+=item * is_floating: Clownfish::Type->new_float
+
+=item * is_void: Clownfish::Type->new_void
+
+=item * is_composite: Clownfish::Type->new_composite
+
+=back
+
+=head2 is_string_type
+
+Returns true if $type represents a Clownfish type which holds unicode
+strings.
+
+=head2 incremented
+
+Returns true if the Type is incremented. Only applicable to object Types.
+
+=head2 decremented
+
+Returns true if the Type is decremented. Only applicable to object Types.
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Util.pm b/clownfish/lib/Clownfish/Util.pm
new file mode 100644
index 0000000..9ccb51f
--- /dev/null
+++ b/clownfish/lib/Clownfish/Util.pm
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Util;
+use Clownfish;
+use base qw( Exporter );
+use Scalar::Util qw( blessed );
+use Carp;
+use Fcntl;
+
+our @EXPORT_OK = qw(
+ slurp_text
+ current
+ strip_c_comments
+ verify_args
+ a_isa_b
+ write_if_changed
+ trim_whitespace
+);
+
+sub verify_args {
+ my $defaults = shift; # leave the rest of @_ intact
+
+ # Verify that args came in pairs.
+ if ( @_ % 2 ) {
+ my ( $package, $filename, $line ) = caller(1);
+ $@ = "Parameter error: odd number of args at $filename line $line\n";
+ return 0;
+ }
+
+ # Verify keys, ignore values.
+ while (@_) {
+ my ( $var, undef ) = ( shift, shift );
+ next if exists $defaults->{$var};
+ my ( $package, $filename, $line ) = caller(1);
+ $@ = "Invalid parameter: '$var' at $filename line $line\n";
+ return 0;
+ }
+
+ return 1;
+}
+
+sub a_isa_b {
+ my ( $thing, $class ) = @_;
+ return 0 unless blessed($thing);
+ return $thing->isa($class);
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Util - Miscellaneous helper functions.
+
+=head1 DESCRIPTION
+
+Clownfish::Util provides a few convenience functions used internally by
+other Clownfish modules.
+
+=head1 FUNCTIONS
+
+=head2 slurp_text
+
+ my $foo_contents = slurp_text('foo.txt');
+
+Open a file, read it in (as text), return its contents.
+
+=head2 current
+
+ compile('foo.c') unless current( 'foo.c', 'foo.o' );
+
+Given two elements, which may be either scalars or arrays, verify that
+everything in the second group exists and was created later than anything in
+the first group.
+
+=head2 verify_args
+
+ verify_args( \%defaults, @_ ) or confess $@;
+
+Verify that named parameters exist in a defaults hash. Returns false and sets
+$@ if a problem is detected.
+
+=head2 strip_c_comments
+
+ my $c_minus_comments = strip_c_comments($c_source_code);
+
+Quick 'n' dirty stripping of C comments. Will massacre stuff like comments
+embedded in string literals, so watch out.
+
+=head2 write_if_changed
+
+ write_if_changed( $path, $content );
+
+Test whether there's a file at C<$path> which already matches C<$content>
+exactly. If something has changed, write the file. Otherwise do nothing (and
+avoid bumping the file's modification time).
+
+=cut
+
diff --git a/clownfish/lib/Clownfish/Variable.pm b/clownfish/lib/Clownfish/Variable.pm
new file mode 100644
index 0000000..ad25e7b
--- /dev/null
+++ b/clownfish/lib/Clownfish/Variable.pm
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Clownfish::Variable;
+use base qw( Clownfish::Symbol );
+use Clownfish;
+use Clownfish::Util qw( verify_args );
+use Carp;
+
+our %new_PARAMS = (
+ type => undef,
+ micro_sym => undef,
+ parcel => undef,
+ exposure => 'local',
+ class_name => undef,
+ class_cnick => undef,
+);
+
+sub new {
+ my ( $either, %args ) = @_;
+ verify_args( \%new_PARAMS, %args ) or confess $@;
+ $args{exposure} ||= 'local';
+ $args{parcel} = Clownfish::Parcel->acquire( $args{parcel} );
+ my $package = ref($either) || $either;
+ return $package->_new(
+ @args{qw( parcel exposure class_name class_cnick micro_sym type )} );
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Clownfish::Variable - A Clownfish variable.
+
+=head1 DESCRIPTION
+
+A variable, having a L<Type|Clownfish::Type>, a micro_sym (i.e. name), an
+exposure, and optionally, a location in the global namespace hierarchy.
+
+Variable objects which exist only within a local scope, e.g. those within
+parameter lists, do not need to know about class. In contrast, inert class
+vars, for example, need to know class information so that they can declare
+themselves properly.
+
+=head1 METHODS
+
+=head2 new
+
+ my $var = Clownfish::Variable->new(
+ parcel => 'Crustacean',
+ type => $int32_t_type, # required
+ micro_sym => 'average_lifespan', # required
+ exposure => 'parcel', # default: 'local'
+ class_name => "Crustacean::Lobster", # default: undef
+ class_cnick => "Lobster", # default: undef
+ );
+
+=over
+
+=item * B<type> - A L<Clownfish::Type>.
+
+=item * B<micro_sym> - The variable's name, without any namespacing prefixes.
+
+=item * B<exposure> - See L<Clownfish::Symbol>.
+
+=item * B<class_name> - See L<Clownfish::Symbol>.
+
+=item * B<class_cnick> - See L<Clownfish::Symbol>.
+
+=back
+
+=head2 local_c
+
+ # e.g. "int32_t average_lifespan"
+ print $variable->local_c;
+
+Returns a string with the Variable's C type and its C<micro_sym>.
+
+=head2 global_c
+
+ # e.g. "int32_t crust_Lobster_average_lifespan"
+ print $variable->global_c;
+
+Returns a string with the Variable's C type and its fully qualified name
+within the global namespace.
+
+=head2 local_declaration
+
+ # e.g. "int32_t average_lifespan;"
+ print $variable->local_declaration;
+
+Returns C code appropriate for declaring the variable in a local scope, such
+as within a struct definition, or as an automatic variable within a C
+function.
+
+=cut
diff --git a/clownfish/src/CFCBase.c b/clownfish/src/CFCBase.c
new file mode 100644
index 0000000..3c55da2
--- /dev/null
+++ b/clownfish/src/CFCBase.c
@@ -0,0 +1,69 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCUtil.h"
+
+CFCBase*
+CFCBase_allocate(size_t size, const char *klass) {
+ CFCBase *self = (CFCBase*)CALLOCATE(size, 1);
+ self->perl_obj = CFCUtil_make_perl_obj(self, klass);
+ return self;
+}
+
+void
+CFCBase_destroy(CFCBase *self) {
+ FREEMEM(self);
+}
+
+CFCBase*
+CFCBase_incref(CFCBase *self) {
+ if (self) {
+ SvREFCNT_inc((SV*)self->perl_obj);
+ }
+ return self;
+}
+
+unsigned
+CFCBase_decref(CFCBase *self) {
+ if (!self) { return 0; }
+ unsigned modified_refcount = SvREFCNT((SV*)self->perl_obj) - 1;
+ /* When the SvREFCNT for this Perl object falls to zero, DESTROY will be
+ * invoked from Perl space for the class that the Perl object was blessed
+ * into. Thus even though the very simple CFC object model does not
+ * generally support polymorphism, we get it for object destruction. */
+ SvREFCNT_dec((SV*)self->perl_obj);
+ return modified_refcount;
+}
+
+void*
+CFCBase_get_perl_obj(CFCBase *self) {
+ return self->perl_obj;
+}
+
+const char*
+CFCBase_get_cfc_class(CFCBase *self) {
+ return HvNAME(SvSTASH((SV*)self->perl_obj));
+}
+
+
diff --git a/clownfish/src/CFCBase.h b/clownfish/src/CFCBase.h
new file mode 100644
index 0000000..4585e7c
--- /dev/null
+++ b/clownfish/src/CFCBase.h
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCBASE
+#define H_CFCBASE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCBase CFCBase;
+
+#ifdef CFC_NEED_BASE_STRUCT_DEF
+struct CFCBase {
+ void *perl_obj;
+};
+#endif
+
+/** Allocate a new CFC object.
+ *
+ * @param size Size of the desired allocation in bytes.
+ * @param klass Class name.
+ */
+CFCBase*
+CFCBase_allocate(size_t size, const char *klass);
+
+/** Clean up CFCBase member variables as necessary and free the object blob
+ * itself.
+ */
+void
+CFCBase_destroy(CFCBase *self);
+
+/** Increment the refcount of the object.
+ *
+ * @return the object itself, allowing an assignment idiom.
+ */
+CFCBase*
+CFCBase_incref(CFCBase *self);
+
+/** Decrement the refcount of the object.
+ *
+ * @return the modified refcount.
+ */
+unsigned
+CFCBase_decref(CFCBase *self);
+
+/** Return the CFC object's cached Perl object.
+ */
+void*
+CFCBase_get_perl_obj(CFCBase *self);
+
+/** Return the class name of the CFC object. (Not the class name of any
+ * parsed object the CFC object might represent.)
+ */
+const char*
+CFCBase_get_cfc_class(CFCBase *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCBASE */
+
diff --git a/clownfish/src/CFCCBlock.c b/clownfish/src/CFCCBlock.c
new file mode 100644
index 0000000..dc66901
--- /dev/null
+++ b/clownfish/src/CFCCBlock.c
@@ -0,0 +1,57 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCCBlock.h"
+#include "CFCUtil.h"
+
+struct CFCCBlock {
+ CFCBase base;
+ char *contents;
+};
+
+CFCCBlock*
+CFCCBlock_new(const char *contents) {
+ CFCCBlock *self = (CFCCBlock*)CFCBase_allocate(sizeof(CFCCBlock),
+ "Clownfish::CBlock");
+ return CFCCBlock_init(self, contents);
+}
+
+CFCCBlock*
+CFCCBlock_init(CFCCBlock *self, const char *contents) {
+ CFCUTIL_NULL_CHECK(contents);
+ self->contents = CFCUtil_strdup(contents);
+ return self;
+}
+
+void
+CFCCBlock_destroy(CFCCBlock *self) {
+ FREEMEM(self->contents);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+const char*
+CFCCBlock_get_contents(CFCCBlock *self) {
+ return self->contents;
+}
+
diff --git a/clownfish/src/CFCCBlock.h b/clownfish/src/CFCCBlock.h
new file mode 100644
index 0000000..2ddaa7c
--- /dev/null
+++ b/clownfish/src/CFCCBlock.h
@@ -0,0 +1,43 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCCBLOCK
+#define H_CFCCBLOCK
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCCBlock CFCCBlock;
+
+CFCCBlock*
+CFCCBlock_new(const char *contents);
+
+CFCCBlock*
+CFCCBlock_init(CFCCBlock *self, const char *contents);
+
+void
+CFCCBlock_destroy(CFCCBlock *self);
+
+const char*
+CFCCBlock_get_contents(CFCCBlock *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCCBLOCK */
+
diff --git a/clownfish/src/CFCClass.c b/clownfish/src/CFCClass.c
new file mode 100644
index 0000000..60c9166
--- /dev/null
+++ b/clownfish/src/CFCClass.c
@@ -0,0 +1,761 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_SYMBOL_STRUCT_DEF
+#include "CFCSymbol.h"
+#include "CFCClass.h"
+#include "CFCDumpable.h"
+#include "CFCFunction.h"
+#include "CFCMethod.h"
+#include "CFCParcel.h"
+#include "CFCDocuComment.h"
+#include "CFCUtil.h"
+#include "CFCVariable.h"
+
+typedef struct CFCClassAttribute {
+ char *name;
+ char *value;
+} CFCClassAttribute;
+
+typedef struct CFCClassRegEntry {
+ char *key;
+ struct CFCClass *klass;
+} CFCClassRegEntry;
+
+static CFCClassRegEntry *registry = NULL;
+static size_t registry_size = 0;
+static size_t registry_cap = 0;
+
+
+struct CFCClass {
+ CFCSymbol symbol;
+ int tree_grown;
+ CFCDocuComment *docucomment;
+ struct CFCClass *parent;
+ struct CFCClass **children;
+ size_t num_kids;
+ CFCFunction **functions;
+ size_t num_functions;
+ CFCMethod **methods;
+ size_t num_methods;
+ CFCVariable **member_vars;
+ size_t num_member_vars;
+ CFCVariable **inert_vars;
+ size_t num_inert_vars;
+ CFCClassAttribute **attributes;
+ size_t num_attributes;
+ char *autocode;
+ char *source_class;
+ char *parent_class_name;
+ int is_final;
+ int is_inert;
+ char *struct_sym;
+ char *full_struct_sym;
+ char *short_vtable_var;
+ char *full_vtable_var;
+ char *full_vtable_type;
+ char *include_h;
+};
+
+// Link up parents and kids.
+static void
+S_establish_ancestry(CFCClass *self);
+
+// Pass down member vars to from parent to children.
+static void
+S_bequeath_member_vars(CFCClass *self);
+
+// Create auto-generated methods. This must be called after member vars are
+// passed down but before methods are passed down.
+static void
+S_generate_automethods(CFCClass *self);
+
+// Create dumpable functions unless hand coded versions were supplied.
+static void
+S_create_dumpables(CFCClass *self);
+
+// Pass down methods to from parent to children.
+static void
+S_bequeath_methods(CFCClass *self);
+
+CFCClass*
+CFCClass_create(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *cnick,
+ const char *micro_sym, CFCDocuComment *docucomment,
+ const char *source_class, const char *parent_class_name,
+ int is_final, int is_inert) {
+ CFCClass *self = (CFCClass*)CFCBase_allocate(sizeof(CFCClass),
+ "Clownfish::Class");
+ return CFCClass_do_create(self, parcel, exposure, class_name, cnick,
+ micro_sym, docucomment, source_class,
+ parent_class_name, is_final, is_inert);
+}
+
+CFCClass*
+CFCClass_do_create(CFCClass *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *cnick, const char *micro_sym,
+ CFCDocuComment *docucomment, const char *source_class,
+ const char *parent_class_name, int is_final, int is_inert) {
+ CFCUTIL_NULL_CHECK(class_name);
+ exposure = exposure ? exposure : "parcel";
+ micro_sym = micro_sym ? micro_sym : "class";
+ CFCSymbol_init((CFCSymbol*)self, parcel, exposure, class_name, cnick,
+ micro_sym);
+ self->parent = NULL;
+ self->tree_grown = false;
+ self->autocode = (char*)CALLOCATE(1, sizeof(char));
+ self->children = (CFCClass**)CALLOCATE(1, sizeof(CFCClass*));
+ self->num_kids = 0;
+ self->functions = (CFCFunction**)CALLOCATE(1, sizeof(CFCFunction*));
+ self->num_functions = 0;
+ self->methods = (CFCMethod**)CALLOCATE(1, sizeof(CFCMethod*));
+ self->num_methods = 0;
+ self->member_vars = (CFCVariable**)CALLOCATE(1, sizeof(CFCVariable*));
+ self->num_member_vars = 0;
+ self->inert_vars = (CFCVariable**)CALLOCATE(1, sizeof(CFCVariable*));
+ self->num_inert_vars = 0;
+ self->attributes = (CFCClassAttribute**)CALLOCATE(1, sizeof(CFCClassAttribute*));
+ self->num_attributes = 0;
+ self->parent_class_name = CFCUtil_strdup(parent_class_name);
+ self->docucomment
+ = (CFCDocuComment*)CFCBase_incref((CFCBase*)docucomment);
+
+ // Assume that Foo::Bar should be found in Foo/Bar.h.
+ self->source_class = source_class
+ ? CFCUtil_strdup(source_class)
+ : CFCUtil_strdup(class_name);
+
+ // Cache several derived symbols.
+ const char *last_colon = strrchr(class_name, ':');
+ self->struct_sym = last_colon
+ ? CFCUtil_strdup(last_colon + 1)
+ : CFCUtil_strdup(class_name);
+ const char *prefix = CFCSymbol_get_prefix((CFCSymbol*)self);
+ size_t prefix_len = strlen(prefix);
+ size_t struct_sym_len = strlen(self->struct_sym);
+ self->short_vtable_var = (char*)MALLOCATE(struct_sym_len + 1);
+ self->full_struct_sym = (char*)MALLOCATE(prefix_len + struct_sym_len + 1);
+ self->full_vtable_var = (char*)MALLOCATE(prefix_len + struct_sym_len + 1);
+ self->full_vtable_type = (char*)MALLOCATE(prefix_len + struct_sym_len + 3 + 1);
+ size_t i;
+ for (i = 0; i < struct_sym_len; i++) {
+ self->short_vtable_var[i] = toupper(self->struct_sym[i]);
+ }
+ self->short_vtable_var[struct_sym_len] = '\0';
+ int check = sprintf(self->full_struct_sym, "%s%s", prefix,
+ self->struct_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ for (i = 0; self->full_struct_sym[i] != '\0'; i++) {
+ self->full_vtable_var[i] = toupper(self->full_struct_sym[i]);
+ }
+ self->full_vtable_var[i] = '\0';
+ check = sprintf(self->full_vtable_type, "%s_VT", self->full_vtable_var);
+ if (check < 0) { croak("sprintf failed"); }
+
+ // Cache the relative path to the autogenerated C header file.
+ size_t source_class_len = strlen(self->source_class);
+ self->include_h = (char*)MALLOCATE(source_class_len + 3);
+ int j;
+ for (i = 0, j = 0; i < source_class_len; i++) {
+ if (self->source_class[i] == ':') {
+ self->include_h[j++] = '/';
+ i++;
+ }
+ else {
+ self->include_h[j++] = self->source_class[i];
+ }
+ }
+ self->include_h[j] = '\0';
+ strcat(self->include_h, ".h");
+
+ self->is_final = !!is_final;
+ self->is_inert = !!is_inert;
+
+ // Store in registry.
+ CFCClass_register(self);
+
+ return self;
+}
+
+void
+CFCClass_destroy(CFCClass *self) {
+ CFCBase_decref((CFCBase*)self->docucomment);
+ CFCBase_decref((CFCBase*)self->parent);
+ size_t i;
+ for (i = 0; self->children[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->children[i]);
+ }
+ for (i = 0; self->functions[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->functions[i]);
+ }
+ for (i = 0; self->methods[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->methods[i]);
+ }
+ for (i = 0; self->member_vars[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->member_vars[i]);
+ }
+ for (i = 0; self->inert_vars[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->inert_vars[i]);
+ }
+ for (i = 0; self->attributes[i] != NULL; i++) {
+ CFCClassAttribute *attribute = self->attributes[i];
+ FREEMEM(attribute->name);
+ FREEMEM(attribute->value);
+ FREEMEM(attribute);
+ }
+ FREEMEM(self->children);
+ FREEMEM(self->functions);
+ FREEMEM(self->methods);
+ FREEMEM(self->member_vars);
+ FREEMEM(self->inert_vars);
+ FREEMEM(self->attributes);
+ FREEMEM(self->autocode);
+ FREEMEM(self->source_class);
+ FREEMEM(self->parent_class_name);
+ FREEMEM(self->struct_sym);
+ FREEMEM(self->short_vtable_var);
+ FREEMEM(self->full_struct_sym);
+ FREEMEM(self->full_vtable_var);
+ FREEMEM(self->full_vtable_type);
+ CFCSymbol_destroy((CFCSymbol*)self);
+}
+
+void
+CFCClass_register(CFCClass *self) {
+ if (registry_size == registry_cap) {
+ size_t new_cap = registry_cap + 10;
+ registry = (CFCClassRegEntry*)REALLOCATE(
+ registry,
+ (new_cap + 1) * sizeof(CFCClassRegEntry));
+ size_t i;
+ for (i = registry_cap; i <= new_cap; i++) {
+ registry[i].key = NULL;
+ registry[i].klass = NULL;
+ }
+ registry_cap = new_cap;
+ }
+ CFCParcel *parcel = CFCSymbol_get_parcel((CFCSymbol*)self);
+ const char *class_name = CFCSymbol_get_class_name((CFCSymbol*)self);
+ CFCClass *existing = CFCClass_fetch_singleton(parcel, class_name);
+ const char *key = self->full_struct_sym;
+ if (existing) {
+ croak("New class %s conflicts with existing class %s",
+ CFCSymbol_get_class_name((CFCSymbol*)self),
+ CFCSymbol_get_class_name((CFCSymbol*)existing));
+ }
+ registry[registry_size].key = CFCUtil_strdup(key);
+ registry[registry_size].klass = (CFCClass*)CFCBase_incref((CFCBase*)self);
+ registry_size++;
+}
+
+CFCClass*
+CFCClass_fetch_singleton(CFCParcel *parcel, const char *class_name) {
+ CFCUTIL_NULL_CHECK(class_name);
+
+ // Build up the key.
+ const char *last_colon = strrchr(class_name, ':');
+ const char *struct_sym = last_colon
+ ? last_colon + 1
+ : class_name;
+ const char *prefix = parcel ? CFCParcel_get_prefix(parcel) : "";
+ size_t prefix_len = strlen(prefix);
+ size_t struct_sym_len = strlen(struct_sym);
+ const size_t MAX_LEN = 256;
+ if (prefix_len + struct_sym_len > MAX_LEN) {
+ croak("names too long: '%s', '%s'", prefix, struct_sym);
+ }
+ char key[MAX_LEN + 1];
+ int check = sprintf(key, "%s%s", prefix, struct_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ size_t i;
+ for (i = 0; i < registry_size; i++) {
+ if (strcmp(registry[i].key, key) == 0) {
+ return registry[i].klass;
+ }
+ }
+ return NULL;
+}
+
+void
+CFCClass_clear_registry(void) {
+ size_t i;
+ for (i = 0; i < registry_size; i++) {
+ CFCBase_decref((CFCBase*)registry[i].klass);
+ }
+ FREEMEM(registry);
+ registry_size = 0;
+ registry_cap = 0;
+ registry = NULL;
+}
+
+void
+CFCClass_add_child(CFCClass *self, CFCClass *child) {
+ CFCUTIL_NULL_CHECK(child);
+ if (self->tree_grown) { croak("Can't call add_child after grow_tree"); }
+ self->num_kids++;
+ size_t size = (self->num_kids + 1) * sizeof(CFCClass*);
+ self->children = (CFCClass**)REALLOCATE(self->children, size);
+ self->children[self->num_kids - 1]
+ = (CFCClass*)CFCBase_incref((CFCBase*)child);
+ self->children[self->num_kids] = NULL;
+}
+
+void
+CFCClass_add_function(CFCClass *self, CFCFunction *func) {
+ CFCUTIL_NULL_CHECK(func);
+ if (self->tree_grown) {
+ croak("Can't call add_function after grow_tree");
+ }
+ self->num_functions++;
+ size_t size = (self->num_functions + 1) * sizeof(CFCFunction*);
+ self->functions = (CFCFunction**)REALLOCATE(self->functions, size);
+ self->functions[self->num_functions - 1]
+ = (CFCFunction*)CFCBase_incref((CFCBase*)func);
+ self->functions[self->num_functions] = NULL;
+}
+
+void
+CFCClass_add_method(CFCClass *self, CFCMethod *method) {
+ CFCUTIL_NULL_CHECK(method);
+ if (self->tree_grown) {
+ croak("Can't call add_method after grow_tree");
+ }
+ if (self->is_inert) {
+ croak("Can't add_method to an inert class");
+ }
+ self->num_methods++;
+ size_t size = (self->num_methods + 1) * sizeof(CFCMethod*);
+ self->methods = (CFCMethod**)REALLOCATE(self->methods, size);
+ self->methods[self->num_methods - 1]
+ = (CFCMethod*)CFCBase_incref((CFCBase*)method);
+ self->methods[self->num_methods] = NULL;
+}
+
+void
+CFCClass_add_member_var(CFCClass *self, CFCVariable *var) {
+ CFCUTIL_NULL_CHECK(var);
+ if (self->tree_grown) {
+ croak("Can't call add_member_var after grow_tree");
+ }
+ self->num_member_vars++;
+ size_t size = (self->num_member_vars + 1) * sizeof(CFCVariable*);
+ self->member_vars = (CFCVariable**)REALLOCATE(self->member_vars, size);
+ self->member_vars[self->num_member_vars - 1]
+ = (CFCVariable*)CFCBase_incref((CFCBase*)var);
+ self->member_vars[self->num_member_vars] = NULL;
+}
+
+void
+CFCClass_add_inert_var(CFCClass *self, CFCVariable *var) {
+ CFCUTIL_NULL_CHECK(var);
+ if (self->tree_grown) {
+ croak("Can't call add_inert_var after grow_tree");
+ }
+ self->num_inert_vars++;
+ size_t size = (self->num_inert_vars + 1) * sizeof(CFCVariable*);
+ self->inert_vars = (CFCVariable**)REALLOCATE(self->inert_vars, size);
+ self->inert_vars[self->num_inert_vars - 1]
+ = (CFCVariable*)CFCBase_incref((CFCBase*)var);
+ self->inert_vars[self->num_inert_vars] = NULL;
+}
+
+void
+CFCClass_add_attribute(CFCClass *self, const char *name, const char *value) {
+ if (!name || !strlen(name)) { croak("'name' is required"); }
+ if (CFCClass_has_attribute(self, name)) {
+ croak("Attribute '%s' already registered");
+ }
+ CFCClassAttribute *attribute
+ = (CFCClassAttribute*)MALLOCATE(sizeof(CFCClassAttribute));
+ attribute->name = CFCUtil_strdup(name);
+ attribute->value = CFCUtil_strdup(value);
+ self->num_attributes++;
+ size_t size = (self->num_attributes + 1) * sizeof(CFCClassAttribute*);
+ self->attributes = (CFCClassAttribute**)REALLOCATE(self->attributes, size);
+ self->attributes[self->num_attributes - 1] = attribute;
+ self->attributes[self->num_attributes] = NULL;
+}
+
+int
+CFCClass_has_attribute(CFCClass *self, const char *name) {
+ CFCUTIL_NULL_CHECK(name);
+ size_t i;
+ for (i = 0; i < self->num_attributes; i++) {
+ if (strcmp(name, self->attributes[i]->name) == 0) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static CFCFunction*
+S_find_func(CFCFunction **funcs, const char *sym) {
+ const size_t MAX_LEN = 128;
+ char lcsym[MAX_LEN + 1];
+ size_t sym_len = strlen(sym);
+ if (sym_len > MAX_LEN) { croak("sym too long: '%s'", sym); }
+ size_t i;
+ for (i = 0; i <= sym_len; i++) {
+ lcsym[i] = tolower(sym[i]);
+ }
+ for (i = 0; funcs[i] != NULL; i++) {
+ CFCFunction *func = funcs[i];
+ const char *func_micro_sym = CFCSymbol_micro_sym((CFCSymbol*)func);
+ if (strcmp(lcsym, func_micro_sym) == 0) {
+ return func;
+ }
+ }
+ return NULL;
+}
+
+CFCFunction*
+CFCClass_function(CFCClass *self, const char *sym) {
+ return S_find_func(self->functions, sym);
+}
+
+CFCMethod*
+CFCClass_method(CFCClass *self, const char *sym) {
+ return (CFCMethod*)S_find_func((CFCFunction**)self->methods, sym);
+}
+
+CFCMethod*
+CFCClass_novel_method(CFCClass *self, const char *sym) {
+ CFCMethod *method = CFCClass_method(self, sym);
+ if (method) {
+ const char *cnick = CFCClass_get_cnick(self);
+ const char *meth_cnick
+ = CFCSymbol_get_class_cnick((CFCSymbol*)method);
+ if (strcmp(cnick, meth_cnick) == 0) {
+ return method;
+ }
+ }
+ return NULL;
+}
+
+// Pass down member vars to from parent to children.
+static void
+S_bequeath_member_vars(CFCClass *self) {
+ size_t i;
+ for (i = 0; self->children[i] != NULL; i++) {
+ CFCClass *child = self->children[i];
+ size_t num_vars = self->num_member_vars + child->num_member_vars;
+ size_t size = (num_vars + 1) * sizeof(CFCVariable*);
+ child->member_vars
+ = (CFCVariable**)REALLOCATE(child->member_vars, size);
+ memmove(child->member_vars + self->num_member_vars,
+ child->member_vars,
+ child->num_member_vars * sizeof(CFCVariable*));
+ memcpy(child->member_vars, self->member_vars,
+ self->num_member_vars * sizeof(CFCVariable*));
+ size_t j;
+ for (j = 0; self->member_vars[j] != NULL; j++) {
+ CFCBase_incref((CFCBase*)child->member_vars[j]);
+ }
+ child->num_member_vars = num_vars;
+ child->member_vars[num_vars] = NULL;
+ S_bequeath_member_vars(child);
+ }
+}
+
+static void
+S_bequeath_methods(CFCClass *self) {
+ size_t child_num;
+ for (child_num = 0; self->children[child_num] != NULL; child_num++) {
+ CFCClass *child = self->children[child_num];
+
+ // Create array of methods, preserving exact order so vtables match up.
+ size_t num_methods = 0;
+ size_t max_methods = self->num_methods + child->num_methods;
+ CFCMethod **methods = (CFCMethod**)MALLOCATE(
+ (max_methods + 1) * sizeof(CFCMethod*));
+
+ // Gather methods which child inherits or overrides.
+ size_t i;
+ for (i = 0; i < self->num_methods; i++) {
+ CFCMethod *method = self->methods[i];
+ const char *micro_sym = CFCSymbol_micro_sym((CFCSymbol*)method);
+ CFCMethod *child_method = CFCClass_method(child, micro_sym);
+ if (child_method) {
+ CFCMethod_override(child_method, method);
+ methods[num_methods++] = child_method;
+ }
+ else {
+ methods[num_methods++] = method;
+ }
+ }
+
+ // Append novel child methods to array. Child methods which were just
+ // marked via CFCMethod_override() a moment ago are skipped.
+ for (i = 0; i < child->num_methods; i++) {
+ CFCMethod *method = child->methods[i];
+ if (CFCMethod_novel(method)) {
+ methods[num_methods++] = method;
+ }
+ }
+ methods[num_methods] = NULL;
+
+ // Manage refcounts and assign new array. Transform to final methods
+ // if child class is a final class.
+ if (child->is_final) {
+ for (i = 0; i < num_methods; i++) {
+ methods[i] = CFCMethod_finalize(methods[i]);
+ }
+ }
+ else {
+ for (i = 0; i < num_methods; i++) {
+ CFCBase_incref((CFCBase*)methods[i]);
+ }
+ }
+ for (i = 0; i < child->num_methods; i++) {
+ CFCBase_decref((CFCBase*)child->methods[i]);
+ }
+ FREEMEM(child->methods);
+ child->methods = methods;
+ child->num_methods = num_methods;
+
+ // Pass it all down to the next generation.
+ S_bequeath_methods(child);
+ child->tree_grown = true;
+ }
+}
+
+// Let the children know who their parent class is.
+static void
+S_establish_ancestry(CFCClass *self) {
+ size_t i;
+ for (i = 0; i < self->num_kids; i++) {
+ CFCClass *child = self->children[i];
+ // This is a circular reference and thus a memory leak, but we don't
+ // care, because we have to have everything in memory at once anyway.
+ CFCClass_set_parent(child, self);
+ S_establish_ancestry(child);
+ }
+}
+
+static size_t
+S_family_tree_size(CFCClass *self) {
+ size_t count = 1; // self
+ size_t i;
+ for (i = 0; i < self->num_kids; i++) {
+ count += S_family_tree_size(self->children[i]);
+ }
+ return count;
+}
+
+static void
+S_create_dumpables(CFCClass *self) {
+ if (CFCClass_has_attribute(self, "dumpable")) {
+ CFCDumpable *dumpable = CFCDumpable_new();
+ CFCDumpable_add_dumpables(dumpable, self);
+ CFCBase_decref((CFCBase*)dumpable);
+ }
+}
+
+void
+CFCClass_grow_tree(CFCClass *self) {
+ if (self->tree_grown) {
+ croak("Can't call grow_tree more than once");
+ }
+ S_establish_ancestry(self);
+ S_bequeath_member_vars(self);
+ S_generate_automethods(self);
+ S_bequeath_methods(self);
+ self->tree_grown = 1;
+}
+
+static void
+S_generate_automethods(CFCClass *self) {
+ S_create_dumpables(self);
+ size_t i;
+ for (i = 0; i < self->num_kids; i++) {
+ S_generate_automethods(self->children[i]);
+ }
+}
+
+// Return value is valid only so long as object persists (elements are not
+// refcounted).
+CFCClass**
+CFCClass_tree_to_ladder(CFCClass *self) {
+ size_t ladder_len = S_family_tree_size(self);
+ CFCClass **ladder = (CFCClass**)MALLOCATE((ladder_len + 1) * sizeof(CFCClass*));
+ ladder[ladder_len] = NULL;
+ size_t step = 0;
+ ladder[step++] = self;
+ size_t i;
+ for (i = 0; i < self->num_kids; i++) {
+ CFCClass *child = self->children[i];
+ CFCClass **child_ladder = CFCClass_tree_to_ladder(child);
+ size_t j;
+ for (j = 0; child_ladder[j] != NULL; j++) {
+ ladder[step++] = child_ladder[j];
+ }
+ FREEMEM(child_ladder);
+ }
+ return ladder;
+}
+
+static CFCSymbol**
+S_novel_syms(CFCClass *self, CFCSymbol **syms) {
+ const char *cnick = CFCSymbol_get_class_cnick((CFCSymbol*)self);
+ size_t count = 0;
+ while (syms[count] != NULL) { count++; }
+ size_t amount = (count + 1) * sizeof(CFCSymbol*);
+ CFCSymbol **novel = (CFCSymbol**)MALLOCATE(amount);
+ size_t num_novel = 0;
+ size_t i;
+ for (i = 0; i < count; i++) {
+ CFCSymbol *sym = syms[i];
+ const char *sym_cnick = CFCSymbol_get_class_cnick((CFCSymbol*)sym);
+ if (strcmp(sym_cnick, cnick) == 0) {
+ novel[num_novel++] = sym;
+ }
+ }
+ novel[num_novel] = NULL;
+ return novel;
+}
+
+CFCMethod**
+CFCClass_novel_methods(CFCClass *self) {
+ return (CFCMethod**)S_novel_syms(self, (CFCSymbol**)self->methods);
+}
+
+CFCVariable**
+CFCClass_novel_member_vars(CFCClass *self) {
+ return (CFCVariable**)S_novel_syms(self, (CFCSymbol**)self->member_vars);
+}
+
+CFCClass**
+CFCClass_children(CFCClass *self) {
+ return self->children;
+}
+
+CFCFunction**
+CFCClass_functions(CFCClass *self) {
+ return self->functions;
+}
+
+CFCMethod**
+CFCClass_methods(CFCClass *self) {
+ return self->methods;
+}
+
+CFCVariable**
+CFCClass_member_vars(CFCClass *self) {
+ return self->member_vars;
+}
+
+CFCVariable**
+CFCClass_inert_vars(CFCClass *self) {
+ return self->inert_vars;
+}
+
+const char*
+CFCClass_get_cnick(CFCClass *self) {
+ return CFCSymbol_get_class_cnick((CFCSymbol*)self);
+}
+
+void
+CFCClass_set_parent(CFCClass *self, CFCClass *parent) {
+ CFCBase_decref((CFCBase*)self->parent);
+ self->parent = (CFCClass*)CFCBase_incref((CFCBase*)parent);
+}
+
+CFCClass*
+CFCClass_get_parent(CFCClass *self) {
+ return self->parent;
+}
+
+void
+CFCClass_append_autocode(CFCClass *self, const char *autocode) {
+ size_t size = strlen(self->autocode) + strlen(autocode) + 1;
+ self->autocode = (char*)REALLOCATE(self->autocode, size);
+ strcat(self->autocode, autocode);
+}
+
+const char*
+CFCClass_get_autocode(CFCClass *self) {
+ return self->autocode;
+}
+
+const char*
+CFCClass_get_source_class(CFCClass *self) {
+ return self->source_class;
+}
+
+const char*
+CFCClass_get_parent_class_name(CFCClass *self) {
+ return self->parent_class_name;
+}
+
+int
+CFCClass_final(CFCClass *self) {
+ return self->is_final;
+}
+
+int
+CFCClass_inert(CFCClass *self) {
+ return self->is_inert;
+}
+
+const char*
+CFCClass_get_struct_sym(CFCClass *self) {
+ return self->struct_sym;
+}
+
+const char*
+CFCClass_full_struct_sym(CFCClass *self) {
+ return self->full_struct_sym;
+}
+
+const char*
+CFCClass_short_vtable_var(CFCClass *self) {
+ return self->short_vtable_var;
+}
+
+const char*
+CFCClass_full_vtable_var(CFCClass *self) {
+ return self->full_vtable_var;
+}
+
+const char*
+CFCClass_full_vtable_type(CFCClass *self) {
+ return self->full_vtable_type;
+}
+
+const char*
+CFCClass_include_h(CFCClass *self) {
+ return self->include_h;
+}
+
+struct CFCDocuComment*
+CFCClass_get_docucomment(CFCClass *self) {
+ return self->docucomment;
+}
+
diff --git a/clownfish/src/CFCClass.h b/clownfish/src/CFCClass.h
new file mode 100644
index 0000000..dd080a2
--- /dev/null
+++ b/clownfish/src/CFCClass.h
@@ -0,0 +1,168 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCCLASS
+#define H_CFCCLASS
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCClass CFCClass;
+struct CFCParcel;
+struct CFCDocuComment;
+struct CFCFunction;
+struct CFCMethod;
+struct CFCVariable;
+
+CFCClass*
+CFCClass_create(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *cnick,
+ const char *micro_sym, struct CFCDocuComment *docucomment,
+ const char *source_class, const char *parent_class_name,
+ int is_final, int is_inert);
+
+CFCClass*
+CFCClass_do_create(CFCClass *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *cnick, const char *micro_sym,
+ struct CFCDocuComment *docucomment,
+ const char *source_class, const char *parent_class_name,
+ int is_final, int is_inert);
+
+void
+CFCClass_destroy(CFCClass *self);
+
+CFCClass*
+CFCClass_fetch_singleton(struct CFCParcel *parcel, const char *class_name);
+
+void
+CFCClass_register(CFCClass *self);
+
+void
+CFCClass_clear_registry(void);
+
+void
+CFCClass_add_child(CFCClass *self, CFCClass *child);
+
+void
+CFCClass_add_function(CFCClass *self, struct CFCFunction *func);
+
+void
+CFCClass_add_method(CFCClass *self, struct CFCMethod *method);
+
+void
+CFCClass_add_member_var(CFCClass *self, struct CFCVariable *var);
+
+void
+CFCClass_add_inert_var(CFCClass *self, struct CFCVariable *var);
+
+void
+CFCClass_add_attribute(CFCClass *self, const char *name, const char *value);
+
+int
+CFCClass_has_attribute(CFCClass *self, const char *name);
+
+struct CFCFunction*
+CFCClass_function(CFCClass *self, const char *sym);
+
+struct CFCMethod*
+CFCClass_method(CFCClass *self, const char *sym);
+
+struct CFCMethod*
+CFCClass_novel_method(CFCClass *self, const char *sym);
+
+void
+CFCClass_grow_tree(CFCClass *self);
+
+CFCClass**
+CFCClass_tree_to_ladder(CFCClass *self);
+
+struct CFCMethod**
+CFCClass_novel_methods(CFCClass *self);
+
+struct CFCVariable**
+CFCClass_novel_member_vars(CFCClass *self);
+
+CFCClass**
+CFCClass_children(CFCClass *self);
+
+struct CFCFunction**
+CFCClass_functions(CFCClass *self);
+
+struct CFCMethod**
+CFCClass_methods(CFCClass *self);
+
+struct CFCVariable**
+CFCClass_member_vars(CFCClass *self);
+
+struct CFCVariable**
+CFCClass_inert_vars(CFCClass *self);
+
+const char*
+CFCClass_get_cnick(CFCClass *self);
+
+void
+CFCClass_set_parent(CFCClass *self, CFCClass *parent);
+
+CFCClass*
+CFCClass_get_parent(CFCClass *self);
+
+void
+CFCClass_append_autocode(CFCClass *self, const char *autocode);
+
+const char*
+CFCClass_get_autocode(CFCClass *self);
+
+const char*
+CFCClass_get_source_class(CFCClass *self);
+
+const char*
+CFCClass_get_parent_class_name(CFCClass *self);
+
+int
+CFCClass_final(CFCClass *self);
+
+int
+CFCClass_inert(CFCClass *self);
+
+const char*
+CFCClass_get_struct_sym(CFCClass *self);
+
+const char*
+CFCClass_full_struct_sym(CFCClass *self);
+
+const char*
+CFCClass_short_vtable_var(CFCClass *self);
+
+const char*
+CFCClass_full_vtable_var(CFCClass *self);
+
+const char*
+CFCClass_full_vtable_type(CFCClass *self);
+
+const char*
+CFCClass_include_h(CFCClass *self);
+
+struct CFCDocuComment*
+CFCClass_get_docucomment(CFCClass *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCCLASS */
+
diff --git a/clownfish/src/CFCDocuComment.c b/clownfish/src/CFCDocuComment.c
new file mode 100644
index 0000000..fb7118c
--- /dev/null
+++ b/clownfish/src/CFCDocuComment.c
@@ -0,0 +1,267 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCDocuComment.h"
+#include "CFCUtil.h"
+
+struct CFCDocuComment {
+ CFCBase base;
+ char *description;
+ char *brief;
+ char *long_des;
+ char **param_names;
+ char **param_docs;
+ char *retval;
+};
+
+/** Remove comment open, close, and left border from raw comment text.
+ */
+static void
+S_strip(char *comment) {
+ size_t len = strlen(comment);
+ char *scratch = (char*)MALLOCATE(len + 1);
+
+ // Establish that comment text begins with "/**" and ends with "*/".
+ if (strstr(comment, "/**") != comment
+ || strstr(comment, "*/") != (comment + len - 2)
+ ) {
+ croak("Malformed comment");
+ }
+
+ // Capture text minus beginning "/**", ending "*/", and left border.
+ size_t i = 3;
+ size_t max = len - 2;
+ while ((isspace(comment[i]) || comment[i] == '*') && i < max) {
+ i++;
+ }
+ size_t j = 0;
+ for (; i < max; i++) {
+ while (comment[i] == '\n' && i < max) {
+ scratch[j++] = comment[i];
+ i++;
+ while (isspace(comment[i]) && comment[i] != '\n' && i < max) {
+ i++;
+ }
+ if (comment[i] == '*') { i++; }
+ if (comment[i] == ' ') { i++; }
+ }
+ if (i < max) {
+ scratch[j++] = comment[i];
+ }
+ }
+
+ // Modify original string in place.
+ for (i = 0; i < j; i++) {
+ comment[i] = scratch[i];
+ }
+ comment[j] = '\0';
+
+ // Clean up.
+ FREEMEM(scratch);
+}
+
+CFCDocuComment*
+CFCDocuComment_parse(const char *raw_text) {
+ char *text = CFCUtil_strdup(raw_text);
+ CFCDocuComment *self
+ = (CFCDocuComment*)CFCBase_allocate(sizeof(CFCDocuComment),
+ "Clownfish::DocuComment");
+
+ // Strip whitespace, comment open, close, and left border.
+ CFCUtil_trim_whitespace(text);
+ S_strip(text);
+
+ // Extract the brief description.
+ {
+ char *ptr = text;
+ size_t len = strlen(text);
+ char *limit = strchr(ptr, '@');
+ if (!limit) {
+ limit = text + len;
+ }
+ while (ptr < limit) {
+ if (*ptr == '.'
+ && ((ptr == limit - 1) || isspace(*(ptr + 1)))
+ ) {
+ ptr++;
+ size_t brief_len = ptr - text;
+ self->brief = CFCUtil_strdup(text);
+ self->brief[brief_len] = '\0';
+ break;
+ }
+ ptr++;
+ }
+ }
+ if (!self->brief) {
+ self->brief = CFCUtil_strdup("");
+ }
+
+ // Extract @param directives.
+ size_t num_params = 0;
+ self->param_names = (char**)CALLOCATE(num_params + 1, sizeof(char*));
+ self->param_docs = (char**)CALLOCATE(num_params + 1, sizeof(char*));
+ {
+ char *candidate = strstr(text, "@param");
+ size_t len = strlen(text);
+ char *limit = text + len;
+ while (candidate) {
+ // Extract param name.
+ char *ptr = candidate + sizeof("@param") - 1;
+ if (!isspace(*ptr) || ptr > limit) {
+ croak("Malformed @param directive in '%s'", raw_text);
+ }
+ while (isspace(*ptr) && ptr < limit) { ptr++; }
+ char *param_name = ptr;
+ while ((isalnum(*ptr) || *ptr == '_') && ptr < limit) { ptr++; }
+ size_t param_name_len = ptr - param_name;
+ if (!param_name_len) {
+ croak("Malformed @param directive in '%s'", raw_text);
+ }
+
+ // Extract param description.
+ while (isspace(*ptr) && ptr < limit) { ptr++; }
+ char *param_doc = ptr;
+ while (ptr < limit) {
+ if (*ptr == '@') { break; }
+ else if (*ptr == '\n' && ptr < limit) {
+ ptr++;
+ while (ptr < limit && *ptr != '\n' && isspace(*ptr)) {
+ ptr++;
+ }
+ if (*ptr == '\n' || *ptr == '@') { break; }
+ }
+ else {
+ ptr++;
+ }
+ }
+ size_t param_doc_len = ptr - param_doc;
+
+ num_params++;
+ size_t size = (num_params + 1) * sizeof(char*);
+ self->param_names = (char**)REALLOCATE(self->param_names, size);
+ self->param_docs = (char**)REALLOCATE(self->param_docs, size);
+ self->param_names[num_params - 1]
+ = CFCUtil_strndup(param_name, param_name_len);
+ self->param_docs[num_params - 1]
+ = CFCUtil_strndup(param_doc, param_doc_len);
+ CFCUtil_trim_whitespace(self->param_names[num_params - 1]);
+ CFCUtil_trim_whitespace(self->param_docs[num_params - 1]);
+ self->param_names[num_params] = NULL;
+ self->param_docs[num_params] = NULL;
+
+ if (ptr == limit) {
+ break;
+ }
+ else if (ptr > limit) {
+ croak("Overran end of string while parsing '%s'", raw_text);
+ }
+ candidate = strstr(ptr, "@param");
+ }
+ }
+
+ // Extract full description.
+ self->description = CFCUtil_strdup(text);
+ {
+ char *terminus = strstr(self->description, "@param");
+ if (!terminus) {
+ terminus = strstr(self->description, "@return");
+ }
+ if (terminus) {
+ *terminus = '\0';
+ }
+ }
+ CFCUtil_trim_whitespace(self->description);
+
+ // Extract long description.
+ self->long_des = CFCUtil_strdup(self->description + strlen(self->brief));
+ CFCUtil_trim_whitespace(self->long_des);
+
+ // Extract @return directive.
+ char *maybe_retval = strstr(text, "@return ");
+ if (maybe_retval) {
+ self->retval = CFCUtil_strdup(maybe_retval + sizeof("@return ") - 1);
+ char *terminus = strchr(self->retval, '@');
+ if (terminus) {
+ *terminus = '\0';
+ }
+ CFCUtil_trim_whitespace(self->retval);
+ }
+
+ FREEMEM(text);
+ return self;
+}
+
+void
+CFCDocuComment_destroy(CFCDocuComment *self) {
+ size_t i;
+ if (self->param_names) {
+ for (i = 0; self->param_names[i] != NULL; i++) {
+ FREEMEM(self->param_names[i]);
+ }
+ FREEMEM(self->param_names);
+ }
+ if (self->param_docs) {
+ for (i = 0; self->param_docs[i] != NULL; i++) {
+ FREEMEM(self->param_docs[i]);
+ }
+ FREEMEM(self->param_docs);
+ }
+ FREEMEM(self->description);
+ FREEMEM(self->brief);
+ FREEMEM(self->long_des);
+ FREEMEM(self->retval);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+const char*
+CFCDocuComment_get_description(CFCDocuComment *self) {
+ return self->description;
+}
+
+const char*
+CFCDocuComment_get_brief(CFCDocuComment *self) {
+ return self->brief;
+}
+
+const char*
+CFCDocuComment_get_long(CFCDocuComment *self) {
+ return self->long_des;
+}
+
+const char**
+CFCDocuComment_get_param_names(CFCDocuComment *self) {
+ return (const char**)self->param_names;
+}
+
+const char**
+CFCDocuComment_get_param_docs(CFCDocuComment *self) {
+ return (const char**)self->param_docs;
+}
+
+const char*
+CFCDocuComment_get_retval(CFCDocuComment *self) {
+ return self->retval;
+}
+
diff --git a/clownfish/src/CFCDocuComment.h b/clownfish/src/CFCDocuComment.h
new file mode 100644
index 0000000..e487bf2
--- /dev/null
+++ b/clownfish/src/CFCDocuComment.h
@@ -0,0 +1,43 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+typedef struct CFCDocuComment CFCDocuComment;
+
+CFCDocuComment*
+CFCDocuComment_parse(const char *raw_text);
+
+void
+CFCDocuComment_destroy(CFCDocuComment *self);
+
+const char*
+CFCDocuComment_get_description(CFCDocuComment *self);
+
+const char*
+CFCDocuComment_get_brief(CFCDocuComment *self);
+
+const char*
+CFCDocuComment_get_long(CFCDocuComment *self);
+
+const char**
+CFCDocuComment_get_param_names(CFCDocuComment *self);
+
+const char**
+CFCDocuComment_get_param_docs(CFCDocuComment *self);
+
+// May be NULL.
+const char*
+CFCDocuComment_get_retval(CFCDocuComment *self);
+
diff --git a/clownfish/src/CFCDumpable.c b/clownfish/src/CFCDumpable.c
new file mode 100644
index 0000000..14bbe31
--- /dev/null
+++ b/clownfish/src/CFCDumpable.c
@@ -0,0 +1,416 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCDumpable.h"
+#include "CFCClass.h"
+#include "CFCFunction.h"
+#include "CFCMethod.h"
+#include "CFCParamList.h"
+#include "CFCParcel.h"
+#include "CFCSymbol.h"
+#include "CFCType.h"
+#include "CFCVariable.h"
+#include "CFCUtil.h"
+
+// Add an autogenerated Dump method to the CFCClass.
+static void
+S_add_dump_method(CFCClass *klass);
+
+// Add an autogenerated Load method to the CFCClass.
+static void
+S_add_load_method(CFCClass *klass);
+
+// Create a Clownfish::Method object for either Dump() or Load().
+static CFCMethod*
+S_make_method_obj(CFCClass *klass, const char *method_name);
+
+// Generate code for dumping a single member var.
+static void
+S_process_dump_member(CFCClass *klass, CFCVariable *member, char *buf,
+ size_t buf_size);
+
+// Generate code for loading a single member var.
+static void
+S_process_load_member(CFCClass *klass, CFCVariable *member, char *buf,
+ size_t buf_size);
+
+struct CFCDumpable {
+ CFCBase base;
+};
+
+CFCDumpable*
+CFCDumpable_new(void) {
+ CFCDumpable *self = (CFCDumpable*)CFCBase_allocate(sizeof(CFCDumpable),
+ "Clownfish::Dumpable");
+ return CFCDumpable_init(self);
+}
+
+CFCDumpable*
+CFCDumpable_init(CFCDumpable *self) {
+ return self;
+}
+
+void
+CFCDumpable_destroy(CFCDumpable *self) {
+ CFCBase_destroy((CFCBase*)self);
+}
+
+void
+CFCDumpable_add_dumpables(CFCDumpable *self, CFCClass *klass) {
+ (void)self;
+
+ if (!CFCClass_has_attribute(klass, "dumpable")) {
+ croak("Class %s isn't dumpable",
+ CFCSymbol_get_class_name((CFCSymbol*)klass));
+ }
+
+ // Inherit Dump/Load from parent if no novel member vars.
+ CFCClass *parent = CFCClass_get_parent(klass);
+ if (parent && CFCClass_has_attribute(parent, "dumpable")) {
+ CFCVariable **novel = CFCClass_novel_member_vars(klass);
+ int needs_autogenerated_dumpables = novel[0] != NULL ? true : false;
+ FREEMEM(novel);
+ if (!needs_autogenerated_dumpables) { return; }
+ }
+
+ if (!CFCClass_novel_method(klass, "Dump")) {
+ S_add_dump_method(klass);
+ }
+ if (!CFCClass_novel_method(klass, "Load")) {
+ S_add_load_method(klass);
+ }
+}
+
+static CFCMethod*
+S_make_method_obj(CFCClass *klass, const char *method_name) {
+ const char *klass_struct_sym = CFCClass_get_struct_sym(klass);
+ const char *klass_name = CFCSymbol_get_class_name((CFCSymbol*)klass);
+ const char *klass_cnick = CFCSymbol_get_class_cnick((CFCSymbol*)klass);
+ CFCParcel *klass_parcel = CFCSymbol_get_parcel((CFCSymbol*)klass);
+ CFCParcel *cf_parcel = CFCParcel_clownfish_parcel();
+
+ CFCType *return_type
+ = CFCType_new_object(CFCTYPE_INCREMENTED, cf_parcel, "Obj", 1);
+ CFCType *self_type = CFCType_new_object(0, klass_parcel, klass_struct_sym, 1);
+ CFCVariable *self_var = CFCVariable_new(klass_parcel, NULL, klass_name,
+ klass_cnick, "self", self_type);
+ CFCParamList *param_list = NULL;
+
+ if (strcmp(method_name, "Dump") == 0) {
+ param_list = CFCParamList_new(false);
+ CFCParamList_add_param(param_list, self_var, NULL);
+ }
+ else if (strcmp(method_name, "Load") == 0) {
+ CFCType *dump_type = CFCType_new_object(0, cf_parcel, "Obj", 1);
+ CFCVariable *dump_var = CFCVariable_new(cf_parcel, NULL, NULL,
+ NULL, "dump", dump_type);
+ param_list = CFCParamList_new(false);
+ CFCParamList_add_param(param_list, self_var, NULL);
+ CFCParamList_add_param(param_list, dump_var, NULL);
+ CFCBase_decref((CFCBase*)dump_var);
+ CFCBase_decref((CFCBase*)dump_type);
+ }
+ else {
+ croak("Unexpected method_name: '%s'", method_name);
+ }
+
+ CFCMethod *method = CFCMethod_new(klass_parcel, "public", klass_name,
+ klass_cnick, method_name, return_type,
+ param_list, NULL, false, false);
+
+ CFCBase_decref((CFCBase*)param_list);
+ CFCBase_decref((CFCBase*)self_type);
+ CFCBase_decref((CFCBase*)self_var);
+ CFCBase_decref((CFCBase*)return_type);
+
+ return method;
+}
+
+static void
+S_add_dump_method(CFCClass *klass) {
+ CFCMethod *method = S_make_method_obj(klass, "Dump");
+ CFCClass_add_method(klass, method);
+ CFCBase_decref((CFCBase*)method);
+ const char *full_func_sym = CFCFunction_full_func_sym((CFCFunction*)method);
+ const char *full_typedef = CFCMethod_full_typedef(method);
+ const char *full_struct = CFCClass_full_struct_sym(klass);
+ const char *vtable_var = CFCClass_full_vtable_var(klass);
+ const char *cnick = CFCClass_get_cnick(klass);
+ CFCClass *parent = CFCClass_get_parent(klass);
+ const size_t BUF_SIZE = 400;
+ char buf[BUF_SIZE];
+
+ if (parent && CFCClass_has_attribute(parent, "dumpable")) {
+ const char pattern[] =
+ "cfish_Obj*\n"
+ "%s(%s *self)\n"
+ "{\n"
+ " %s super_dump = (%s)SUPER_METHOD(%s, %s, Dump);\n"
+ " cfish_Hash *dump = (cfish_Hash*)super_dump(self);\n";
+ size_t amount = sizeof(pattern)
+ + strlen(full_func_sym)
+ + strlen(full_struct)
+ + strlen(full_typedef) * 2
+ + strlen(vtable_var)
+ + strlen(cnick)
+ + 50;
+ char *autocode = (char*)MALLOCATE(amount);
+ int check = sprintf(autocode, pattern, full_func_sym, full_struct,
+ full_typedef, full_typedef, vtable_var, cnick);
+ if (check < 0) { CFCUtil_die("sprintf failed"); }
+ CFCClass_append_autocode(klass, autocode);
+ FREEMEM(autocode);
+ CFCVariable **novel = CFCClass_novel_member_vars(klass);
+ size_t i;
+ for (i = 0; novel[i] != NULL; i++) {
+ S_process_dump_member(klass, novel[i], buf, BUF_SIZE);
+ }
+ FREEMEM(novel);
+ }
+ else {
+ const char pattern[] =
+ "cfish_Obj*\n"
+ "%s(%s *self)\n"
+ "{\n"
+ " cfish_Hash *dump = cfish_Hash_new(0);\n"
+ " Cfish_Hash_Store_Str(dump, \"_class\", 6,\n"
+ " (cfish_Obj*)Cfish_CB_Clone(Cfish_Obj_Get_Class_Name((cfish_Obj*)self)));\n";
+ size_t amount = sizeof(pattern)
+ + strlen(full_func_sym)
+ + strlen(full_struct)
+ + 50;
+ char *autocode = (char*)MALLOCATE(amount);
+ int check = sprintf(autocode, pattern, full_func_sym, full_struct);
+ if (check < 0) { croak("sprintf failed"); }
+ CFCClass_append_autocode(klass, autocode);
+ FREEMEM(autocode);
+ CFCVariable **members = CFCClass_member_vars(klass);
+ size_t i;
+ for (i = 0; members[i] != NULL; i++) {
+ S_process_dump_member(klass, members[i], buf, BUF_SIZE);
+ }
+ }
+
+ CFCClass_append_autocode(klass, " return (cfish_Obj*)dump;\n}\n\n");
+}
+
+static void
+S_add_load_method(CFCClass *klass) {
+ CFCMethod *method = S_make_method_obj(klass, "Load");
+ CFCClass_add_method(klass, method);
+ CFCBase_decref((CFCBase*)method);
+ const char *full_func_sym = CFCFunction_full_func_sym((CFCFunction*)method);
+ const char *full_typedef = CFCMethod_full_typedef(method);
+ const char *full_struct = CFCClass_full_struct_sym(klass);
+ const char *vtable_var = CFCClass_full_vtable_var(klass);
+ const char *cnick = CFCClass_get_cnick(klass);
+ CFCClass *parent = CFCClass_get_parent(klass);
+ const size_t BUF_SIZE = 400;
+ char buf[BUF_SIZE];
+
+ if (parent && CFCClass_has_attribute(parent, "dumpable")) {
+ const char pattern[] =
+ "cfish_Obj*\n"
+ "%s(%s *self, cfish_Obj *dump)\n"
+ "{\n"
+ " cfish_Hash *source = (cfish_Hash*)CFISH_CERTIFY(dump, CFISH_HASH);\n"
+ " %s super_load = (%s)SUPER_METHOD(%s, %s, Load);\n"
+ " %s *loaded = (%s*)super_load(self, dump);\n";
+ size_t amount = sizeof(pattern)
+ + strlen(full_func_sym)
+ + strlen(full_struct) * 3
+ + strlen(full_typedef) * 2
+ + strlen(vtable_var)
+ + strlen(cnick)
+ + 50;
+ char *autocode = (char*)MALLOCATE(amount);
+ int check = sprintf(autocode, pattern, full_func_sym, full_struct,
+ full_typedef, full_typedef, vtable_var, cnick,
+ full_struct, full_struct);
+ if (check < 0) { croak("sprintf failed"); }
+ CFCClass_append_autocode(klass, autocode);
+ FREEMEM(autocode);
+ CFCVariable **novel = CFCClass_novel_member_vars(klass);
+ size_t i;
+ for (i = 0; novel[i] != NULL; i++) {
+ S_process_load_member(klass, novel[i], buf, BUF_SIZE);
+ }
+ FREEMEM(novel);
+ }
+ else {
+ const char pattern[] =
+ "cfish_Obj*\n"
+ "%s(%s *self, cfish_Obj *dump)\n"
+ "{\n"
+ " cfish_Hash *source = (cfish_Hash*)CFISH_CERTIFY(dump, CFISH_HASH);\n"
+ " cfish_CharBuf *class_name = (cfish_CharBuf*)CFISH_CERTIFY(\n"
+ " Cfish_Hash_Fetch_Str(source, \"_class\", 6), CFISH_CHARBUF);\n"
+ " cfish_VTable *vtable = cfish_VTable_singleton(class_name, NULL);\n"
+ " %s *loaded = (%s*)Cfish_VTable_Make_Obj(vtable);\n"
+ " CHY_UNUSED_VAR(self);\n";
+ size_t amount = sizeof(pattern)
+ + strlen(full_func_sym)
+ + strlen(full_struct) * 3
+ + 50;
+ char *autocode = (char*)MALLOCATE(amount);
+ int check = sprintf(autocode, pattern, full_func_sym, full_struct,
+ full_struct, full_struct);
+ if (check < 0) { croak("sprintf failed"); }
+ CFCClass_append_autocode(klass, autocode);
+ FREEMEM(autocode);
+ CFCVariable **members = CFCClass_member_vars(klass);
+ size_t i;
+ for (i = 0; members[i] != NULL; i++) {
+ S_process_load_member(klass, members[i], buf, BUF_SIZE);
+ }
+ }
+
+ CFCClass_append_autocode(klass, " return (cfish_Obj*)loaded;\n}\n\n");
+}
+
+static void
+S_process_dump_member(CFCClass *klass, CFCVariable *member, char *buf,
+ size_t buf_size) {
+ CFCUTIL_NULL_CHECK(member);
+ CFCType *type = CFCVariable_get_type(member);
+ const char *name = CFCSymbol_micro_sym((CFCSymbol*)member);
+ unsigned name_len = (unsigned)strlen(name);
+ const char *specifier = CFCType_get_specifier(type);
+
+ // Skip the VTable and the refcount/host-object.
+ if (strcmp(specifier, "lucy_VTable") == 0
+ || strcmp(specifier, "lucy_ref_t") == 0
+ ) {
+ return;
+ }
+
+ if (CFCType_is_integer(type) || CFCType_is_floating(type)) {
+ char int_pattern[] =
+ " Cfish_Hash_Store_Str(dump, \"%s\", %u, (cfish_Obj*)cfish_CB_newf(\"%%i64\", (int64_t)self->%s));\n";
+ char float_pattern[] =
+ " Cfish_Hash_Store_Str(dump, \"%s\", %u, (cfish_Obj*)cfish_CB_newf(\"%%f64\", (double)self->%s));\n";
+ const char *pattern = CFCType_is_integer(type)
+ ? int_pattern : float_pattern;
+ size_t needed = strlen(pattern) + name_len * 2 + 20;
+ if (buf_size < needed) {
+ croak("Buffer not big enough (%lu < %lu)",
+ (unsigned long)buf_size, (unsigned long)needed);
+ }
+ int check = sprintf(buf, pattern, name, name_len, name);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else if (CFCType_is_object(type)) {
+ char pattern[] =
+ " if (self->%s) {\n"
+ " Cfish_Hash_Store_Str(dump, \"%s\", %u, Cfish_Obj_Dump((cfish_Obj*)self->%s));\n"
+ " }\n";
+
+ size_t needed = strlen(pattern) + name_len * 3 + 20;
+ if (buf_size < needed) {
+ croak("Buffer not big enough (%lu < %lu)",
+ (unsigned long)buf_size, (unsigned long)needed);
+ }
+ int check = sprintf(buf, pattern, name, name, name_len, name);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else {
+ croak("Don't know how to dump a %s", CFCType_get_specifier(type));
+ }
+
+ CFCClass_append_autocode(klass, buf);
+}
+
+static void
+S_process_load_member(CFCClass *klass, CFCVariable *member, char *buf,
+ size_t buf_size) {
+ CFCUTIL_NULL_CHECK(member);
+ CFCType *type = CFCVariable_get_type(member);
+ const char *type_str = CFCType_to_c(type);
+ const char *name = CFCSymbol_micro_sym((CFCSymbol*)member);
+ unsigned name_len = (unsigned)strlen(name);
+ char extraction[200];
+ const char *specifier = CFCType_get_specifier(type);
+ size_t specifier_len = strlen(specifier);
+
+ // Skip the VTable and the refcount/host-object.
+ if (strcmp(specifier, "lucy_VTable") == 0
+ || strcmp(specifier, "lucy_ref_t") == 0
+ ) {
+ return;
+ }
+
+ if (strlen(type_str) + 100 > sizeof(extraction)) { // play it safe
+ croak("type_str too long: '%s'", type_str);
+ }
+ if (CFCType_is_integer(type)) {
+ int check = sprintf(extraction, "(%s)Cfish_Obj_To_I64(var)", type_str);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else if (CFCType_is_floating(type)) {
+ int check = sprintf(extraction, "(%s)Cfish_Obj_To_F64(var)", type_str);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else if (CFCType_is_object(type)) {
+ char vtable_var[50];
+ if (specifier_len > sizeof(vtable_var) - 2) {
+ croak("specifier too long: '%s'", specifier);
+ }
+ size_t i;
+ for (i = 0; i <= specifier_len; i++) {
+ vtable_var[i] = toupper(specifier[i]);
+ }
+ int check = sprintf(extraction,
+ "(%s*)CFISH_CERTIFY(Cfish_Obj_Load(var, var), %s)",
+ specifier, vtable_var);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else {
+ croak("Don't know how to load %s", specifier);
+ }
+
+ const char *pattern =
+ " {\n"
+ " cfish_Obj *var = Cfish_Hash_Fetch_Str(source, \"%s\", %u);\n"
+ " if (var) { loaded->%s = %s; }\n"
+ " }\n";
+ size_t needed = sizeof(pattern)
+ + (name_len * 2)
+ + strlen(extraction)
+ + 20;
+ if (buf_size < needed) {
+ croak("Buffer not big enough (%lu < %lu)", (unsigned long)buf_size,
+ (unsigned long)needed);
+ }
+ int check = sprintf(buf, pattern, name, name_len, name, extraction);
+ if (check < 0) { croak("sprintf failed"); }
+
+ CFCClass_append_autocode(klass, buf);
+}
+
+
diff --git a/clownfish/src/CFCDumpable.h b/clownfish/src/CFCDumpable.h
new file mode 100644
index 0000000..884afca
--- /dev/null
+++ b/clownfish/src/CFCDumpable.h
@@ -0,0 +1,44 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCDUMPABLE
+#define H_CFCDUMPABLE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCDumpable CFCDumpable;
+struct CFCClass;
+
+CFCDumpable*
+CFCDumpable_new(void);
+
+CFCDumpable*
+CFCDumpable_init(CFCDumpable *self);
+
+void
+CFCDumpable_destroy(CFCDumpable *self);
+
+void
+CFCDumpable_add_dumpables(CFCDumpable *self, struct CFCClass *klass);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCDUMPABLE */
+
diff --git a/clownfish/src/CFCFile.c b/clownfish/src/CFCFile.c
new file mode 100644
index 0000000..9de0318
--- /dev/null
+++ b/clownfish/src/CFCFile.c
@@ -0,0 +1,269 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#ifdef _WIN32
+#define PATH_SEP "\\"
+#define PATH_SEP_CHAR '\\'
+#else
+#define PATH_SEP "/"
+#define PATH_SEP_CHAR '/'
+#endif
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCFile.h"
+#include "CFCUtil.h"
+#include "CFCClass.h"
+
+struct CFCFile {
+ CFCBase base;
+ CFCBase **blocks;
+ CFCClass **classes;
+ int modified;
+ char *source_class;
+ char *guard_name;
+ char *guard_start;
+ char *guard_close;
+ char *path_part;
+};
+
+CFCFile*
+CFCFile_new(const char *source_class) {
+
+ CFCFile *self = (CFCFile*)CFCBase_allocate(sizeof(CFCFile),
+ "Clownfish::File");
+ return CFCFile_init(self, source_class);
+}
+
+CFCFile*
+CFCFile_init(CFCFile *self, const char *source_class) {
+ CFCUTIL_NULL_CHECK(source_class);
+ self->modified = false;
+ self->source_class = CFCUtil_strdup(source_class);
+ self->blocks = (CFCBase**)CALLOCATE(1, sizeof(CFCBase*));
+ self->classes = (CFCClass**)CALLOCATE(1, sizeof(CFCBase*));
+
+ // Derive include guard name, plus C code for opening and closing the
+ // guard.
+ size_t len = strlen(source_class);
+ self->guard_name = (char*)MALLOCATE(len + sizeof("H_") + 1);
+ self->guard_start = (char*)MALLOCATE(len * 2 + 40);
+ self->guard_close = (char*)MALLOCATE(len + 20);
+ memcpy(self->guard_name, "H_", 2);
+ size_t i, j;
+ for (i = 0, j = 2; i < len; i++, j++) {
+ char c = source_class[i];
+ if (c == ':') {
+ self->guard_name[j] = '_';
+ i++;
+ }
+ else {
+ self->guard_name[j] = toupper(c);
+ }
+ }
+ self->guard_name[j] = '\0';
+ int check = sprintf(self->guard_start, "#ifndef %s\n#define %s 1\n",
+ self->guard_name, self->guard_name);
+ if (check < 0) { croak("sprintf failed"); }
+ check = sprintf(self->guard_close, "#endif /* %s */\n",
+ self->guard_name);
+ if (check < 0) { croak("sprintf failed"); }
+
+ // Cache partial path derived from source_class.
+ self->path_part = (char*)MALLOCATE(len + 1);
+ for (i = 0, j = 0; i < len; i++, j++) {
+ char c = source_class[i];
+ if (c == ':') {
+ self->path_part[j] = PATH_SEP_CHAR;
+ i++;
+ }
+ else {
+ self->path_part[j] = c;
+ }
+ }
+ self->path_part[j] = '\0';
+
+ return self;
+}
+
+void
+CFCFile_destroy(CFCFile *self) {
+ size_t i;
+ for (i = 0; self->blocks[i] != NULL; i++) {
+ CFCBase_decref(self->blocks[i]);
+ }
+ FREEMEM(self->blocks);
+ for (i = 0; self->classes[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->classes[i]);
+ }
+ FREEMEM(self->classes);
+ FREEMEM(self->guard_name);
+ FREEMEM(self->guard_start);
+ FREEMEM(self->guard_close);
+ FREEMEM(self->source_class);
+ FREEMEM(self->path_part);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+void
+CFCFile_add_block(CFCFile *self, CFCBase *block) {
+ CFCUTIL_NULL_CHECK(block);
+ const char *cfc_class = CFCBase_get_cfc_class(block);
+
+ // Add to classes array if the block is a CFCClass.
+ if (strcmp(cfc_class, "Clownfish::Class") == 0) {
+ size_t num_class_blocks = 0;
+ while (self->classes[num_class_blocks] != NULL) {
+ num_class_blocks++;
+ }
+ num_class_blocks++;
+ size_t size = (num_class_blocks + 1) * sizeof(CFCClass*);
+ self->classes = (CFCClass**)REALLOCATE(self->classes, size);
+ self->classes[num_class_blocks - 1]
+ = (CFCClass*)CFCBase_incref(block);
+ self->classes[num_class_blocks] = NULL;
+ }
+
+ // Add to blocks array.
+ if (strcmp(cfc_class, "Clownfish::Class") == 0
+ || strcmp(cfc_class, "Clownfish::Parcel") == 0
+ || strcmp(cfc_class, "Clownfish::CBlock") == 0
+ ) {
+ size_t num_blocks = 0;
+ while (self->blocks[num_blocks] != NULL) {
+ num_blocks++;
+ }
+ num_blocks++;
+ size_t size = (num_blocks + 1) * sizeof(CFCBase*);
+ self->blocks = (CFCBase**)REALLOCATE(self->blocks, size);
+ self->blocks[num_blocks - 1] = CFCBase_incref(block);
+ self->blocks[num_blocks] = NULL;
+ }
+ else {
+ croak("Wrong kind of object: '%s'", cfc_class);
+ }
+}
+
+static void
+S_some_path(CFCFile *self, char *buf, size_t buf_size, const char *base_dir,
+ const char *ext) {
+ size_t needed = CFCFile_path_buf_size(self, base_dir);
+ if (strlen(ext) > 4) {
+ croak("ext cannot be more than 4 characters.");
+ }
+ if (needed > buf_size) {
+ croak("Need buf_size of %lu, but got %lu", needed, buf_size);
+ }
+ if (base_dir) {
+ int check = sprintf(buf, "%s" PATH_SEP "%s%s", base_dir,
+ self->path_part, ext);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else {
+ int check = sprintf(buf, "%s%s", self->path_part, ext);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ size_t i;
+ for (i = 0; buf[i] != '\0'; i++) {
+ #ifdef _WIN32
+ if (buf[i] == '/') { buf[i] = '\\'; }
+ #else
+ if (buf[i] == '\\') { buf[i] = '/'; }
+ #endif
+ }
+}
+
+size_t
+CFCFile_path_buf_size(CFCFile *self, const char *base_dir) {
+ size_t size = strlen(self->path_part);
+ size += 4; // Max extension length.
+ size += 1; // NULL-termination.
+ if (base_dir) {
+ size += strlen(base_dir);
+ size += strlen(PATH_SEP);
+ }
+ return size;
+}
+
+void
+CFCFile_c_path(CFCFile *self, char *buf, size_t buf_size,
+ const char *base_dir) {
+ S_some_path(self, buf, buf_size, base_dir, ".c");
+}
+
+void
+CFCFile_h_path(CFCFile *self, char *buf, size_t buf_size,
+ const char *base_dir) {
+ S_some_path(self, buf, buf_size, base_dir, ".h");
+}
+
+void
+CFCFile_cfh_path(CFCFile *self, char *buf, size_t buf_size,
+ const char *base_dir) {
+ S_some_path(self, buf, buf_size, base_dir, ".cfh");
+}
+
+CFCBase**
+CFCFile_blocks(CFCFile *self) {
+ return self->blocks;
+}
+
+CFCClass**
+CFCFile_classes(CFCFile *self) {
+ return self->classes;
+}
+
+void
+CFCFile_set_modified(CFCFile *self, int modified) {
+ self->modified = !!modified;
+}
+
+int
+CFCFile_get_modified(CFCFile *self) {
+ return self->modified;
+}
+
+const char*
+CFCFile_get_source_class(CFCFile *self) {
+ return self->source_class;
+}
+
+const char*
+CFCFile_guard_name(CFCFile *self) {
+ return self->guard_name;
+}
+
+const char*
+CFCFile_guard_start(CFCFile *self) {
+ return self->guard_start;
+}
+
+const char*
+CFCFile_guard_close(CFCFile *self) {
+ return self->guard_close;
+}
+
diff --git a/clownfish/src/CFCFile.h b/clownfish/src/CFCFile.h
new file mode 100644
index 0000000..1f11a28
--- /dev/null
+++ b/clownfish/src/CFCFile.h
@@ -0,0 +1,87 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCFILE
+#define H_CFCFILE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCFile CFCFile;
+struct CFCBase;
+struct CFCClass;
+
+CFCFile*
+CFCFile_new(const char *source_class);
+
+CFCFile*
+CFCFile_init(CFCFile *self, const char *source_class);
+
+void
+CFCFile_destroy(CFCFile *self);
+
+void
+CFCFile_add_block(CFCFile *self, CFCBase *block);
+
+/** Calculate the size of the buffer needed for a call to c_path(), h_path(),
+ * or cfh_path().
+ */
+size_t
+CFCFile_path_buf_size(CFCFile *self, const char *base_dir);
+
+void
+CFCFile_c_path(CFCFile *self, char *buf, size_t buf_size,
+ const char *base_dir);
+
+void
+CFCFile_h_path(CFCFile *self, char *buf, size_t buf_size,
+ const char *base_dir);
+
+void
+CFCFile_cfh_path(CFCFile *self, char *buf, size_t buf_size,
+ const char *base_dir);
+
+struct CFCBase**
+CFCFile_blocks(CFCFile *self);
+
+struct CFCClass**
+CFCFile_classes(CFCFile *self);
+
+void
+CFCFile_set_modified(CFCFile *self, int modified);
+
+int
+CFCFile_get_modified(CFCFile *self);
+
+const char*
+CFCFile_get_source_class(CFCFile *self);
+
+const char*
+CFCFile_guard_name(CFCFile *self);
+
+const char*
+CFCFile_guard_start(CFCFile *self);
+
+const char*
+CFCFile_guard_close(CFCFile *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCFILE */
+
diff --git a/clownfish/src/CFCFunction.c b/clownfish/src/CFCFunction.c
new file mode 100644
index 0000000..77cf7a4
--- /dev/null
+++ b/clownfish/src/CFCFunction.c
@@ -0,0 +1,126 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_FUNCTION_STRUCT_DEF
+#include "CFCFunction.h"
+#include "CFCParcel.h"
+#include "CFCType.h"
+#include "CFCParamList.h"
+#include "CFCDocuComment.h"
+#include "CFCUtil.h"
+
+CFCFunction*
+CFCFunction_new(CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym, CFCType *return_type,
+ CFCParamList *param_list, CFCDocuComment *docucomment,
+ int is_inline) {
+ CFCFunction *self = (CFCFunction*)CFCBase_allocate(sizeof(CFCFunction),
+ "Clownfish::Function");
+ return CFCFunction_init(self, parcel, exposure, class_name, class_cnick,
+ micro_sym, return_type, param_list, docucomment,
+ is_inline);
+}
+
+static int
+S_validate_micro_sym(const char *micro_sym) {
+ size_t i;
+ size_t len = strlen(micro_sym);
+ if (!len) { return false; }
+ for (i = 0; i < len; i++) {
+ char c = micro_sym[i];
+ if (!islower(c) && !isdigit(c) && c != '_') { return false; }
+ }
+ return true;
+}
+
+CFCFunction*
+CFCFunction_init(CFCFunction *self, CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym, CFCType *return_type,
+ CFCParamList *param_list, CFCDocuComment *docucomment,
+ int is_inline) {
+
+ exposure = exposure ? exposure : "parcel";
+ CFCSymbol_init((CFCSymbol*)self, parcel, exposure, class_name,
+ class_cnick, micro_sym);
+ CFCUTIL_NULL_CHECK(class_name);
+ CFCUTIL_NULL_CHECK(return_type);
+ CFCUTIL_NULL_CHECK(param_list);
+ if (!S_validate_micro_sym(micro_sym)) {
+ croak("Invalid micro_sym: '%s'", micro_sym);
+ }
+ self->return_type = (CFCType*)CFCBase_incref((CFCBase*)return_type);
+ self->param_list = (CFCParamList*)CFCBase_incref((CFCBase*)param_list);
+ self->docucomment = (CFCDocuComment*)CFCBase_incref((CFCBase*)docucomment);
+ self->is_inline = is_inline;
+ return self;
+}
+
+void
+CFCFunction_destroy(CFCFunction *self) {
+ CFCBase_decref((CFCBase*)self->return_type);
+ CFCBase_decref((CFCBase*)self->param_list);
+ CFCBase_decref((CFCBase*)self->docucomment);
+ CFCSymbol_destroy((CFCSymbol*)self);
+}
+
+CFCType*
+CFCFunction_get_return_type(CFCFunction *self) {
+ return self->return_type;
+}
+
+CFCParamList*
+CFCFunction_get_param_list(CFCFunction *self) {
+ return self->param_list;
+}
+
+CFCDocuComment*
+CFCFunction_get_docucomment(CFCFunction *self) {
+ return self->docucomment;
+}
+
+int
+CFCFunction_inline(CFCFunction *self) {
+ return self->is_inline;
+}
+
+int
+CFCFunction_void(CFCFunction *self) {
+ return CFCType_is_void(self->return_type);
+}
+
+const char*
+CFCFunction_full_func_sym(CFCFunction *self) {
+ return CFCSymbol_full_sym((CFCSymbol*)self);
+}
+
+const char*
+CFCFunction_short_func_sym(CFCFunction *self) {
+ return CFCSymbol_short_sym((CFCSymbol*)self);
+}
+
diff --git a/clownfish/src/CFCFunction.h b/clownfish/src/CFCFunction.h
new file mode 100644
index 0000000..53f13d5
--- /dev/null
+++ b/clownfish/src/CFCFunction.h
@@ -0,0 +1,86 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCFUNCTION
+#define H_CFCFUNCTION
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCFunction CFCFunction;
+struct CFCParcel;
+struct CFCType;
+struct CFCDocuComment;
+struct CFCParamList;
+
+#ifdef CFC_NEED_FUNCTION_STRUCT_DEF
+#define CFC_NEED_SYMBOL_STRUCT_DEF
+#include "CFCSymbol.h"
+struct CFCFunction {
+ CFCSymbol symbol;
+ struct CFCType *return_type;
+ struct CFCParamList *param_list;
+ struct CFCDocuComment *docucomment;
+ int is_inline;
+};
+#endif
+
+
+CFCFunction*
+CFCFunction_new(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym, struct CFCType *return_type,
+ struct CFCParamList *param_list,
+ struct CFCDocuComment *docucomment, int is_inline);
+
+CFCFunction*
+CFCFunction_init(CFCFunction *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *class_cnick, const char *micro_sym,
+ struct CFCType *return_type, struct CFCParamList *param_list,
+ struct CFCDocuComment *docucomment, int is_inline);
+
+void
+CFCFunction_destroy(CFCFunction *self);
+
+struct CFCType*
+CFCFunction_get_return_type(CFCFunction *self);
+
+struct CFCParamList*
+CFCFunction_get_param_list(CFCFunction *self);
+
+struct CFCDocuComment*
+CFCFunction_get_docucomment(CFCFunction *self);
+
+int
+CFCFunction_inline(CFCFunction *self);
+
+int
+CFCFunction_void(CFCFunction *self);
+
+const char*
+CFCFunction_full_func_sym(CFCFunction *self);
+
+const char*
+CFCFunction_short_func_sym(CFCFunction *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCFUNCTION */
+
diff --git a/clownfish/src/CFCHierarchy.c b/clownfish/src/CFCHierarchy.c
new file mode 100644
index 0000000..c7814f6
--- /dev/null
+++ b/clownfish/src/CFCHierarchy.c
@@ -0,0 +1,577 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/stat.h>
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#ifdef WIN32
+ #define PATH_SEP "\\"
+#else
+ #define PATH_SEP "/"
+#endif
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCHierarchy.h"
+#include "CFCClass.h"
+#include "CFCFile.h"
+#include "CFCSymbol.h"
+#include "CFCUtil.h"
+
+struct CFCHierarchy {
+ CFCBase base;
+ char *source;
+ char *dest;
+ void *parser;
+ CFCClass **trees;
+ size_t num_trees;
+ CFCFile **files;
+ size_t num_files;
+};
+
+static void
+S_parse_cf_files(CFCHierarchy *self);
+
+static void
+S_add_file(CFCHierarchy *self, CFCFile *file);
+
+static void
+S_add_tree(CFCHierarchy *self, CFCClass *klass);
+
+static CFCFile*
+S_fetch_file(CFCHierarchy *self, const char *source_class);
+
+// Recursive helper function for CFCUtil_propagate_modified.
+static int
+S_do_propagate_modified(CFCHierarchy *self, CFCClass *klass, int modified);
+
+// Platform-agnostic opendir wrapper.
+static void*
+S_opendir(const char *dir);
+
+// Platform-agnostic readdir wrapper.
+static const char*
+S_next_entry(void *dirhandle);
+
+// Platform-agnostic closedir wrapper.
+static void
+S_closedir(void *dirhandle, const char *dir);
+
+// Indicate whether a path is a directory.
+// Note: this has to be defined before including the Perl headers because they
+// redefine stat() in an incompatible way on certain systems (Windows).
+static int
+S_is_dir(const char *path) {
+ struct stat stat_buf;
+ int stat_check = stat(path, &stat_buf);
+ if (stat_check == -1) {
+ CFCUtil_die("Stat failed for '%s': %s", path,
+ strerror(errno));
+ }
+ return (stat_buf.st_mode & S_IFDIR) ? true : false;
+}
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+CFCHierarchy*
+CFCHierarchy_new(const char *source, const char *dest, void *parser) {
+ CFCHierarchy *self
+ = (CFCHierarchy*)CFCBase_allocate(sizeof(CFCHierarchy),
+ "Clownfish::Hierarchy");
+ return CFCHierarchy_init(self, source, dest, parser);
+}
+
+CFCHierarchy*
+CFCHierarchy_init(CFCHierarchy *self, const char *source, const char *dest,
+ void *parser) {
+ if (!source || !strlen(source) || !dest || !strlen(dest)) {
+ croak("Both 'source' and 'dest' are required");
+ }
+ self->source = CFCUtil_strdup(source);
+ self->dest = CFCUtil_strdup(dest);
+ self->trees = (CFCClass**)CALLOCATE(1, sizeof(CFCClass*));
+ self->num_trees = 0;
+ self->files = (CFCFile**)CALLOCATE(1, sizeof(CFCFile*));
+ self->num_files = 0;
+ self->parser = newSVsv((SV*)parser);
+ return self;
+}
+
+void
+CFCHierarchy_destroy(CFCHierarchy *self) {
+ size_t i;
+ for (i = 0; self->trees[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->trees[i]);
+ }
+ for (i = 0; self->files[i] != NULL; i++) {
+ CFCBase_decref((CFCBase*)self->files[i]);
+ }
+ FREEMEM(self->trees);
+ FREEMEM(self->files);
+ FREEMEM(self->source);
+ FREEMEM(self->dest);
+ SvREFCNT_dec((SV*)self->parser);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+void
+CFCHierarchy_build(CFCHierarchy *self) {
+ S_parse_cf_files(self);
+ size_t i;
+ for (i = 0; self->trees[i] != NULL; i++) {
+ CFCClass_grow_tree(self->trees[i]);
+ }
+}
+
+static CFCFile*
+S_parse_file(void *parser, const char *content, const char *source_class) {
+ dSP;
+ ENTER;
+ SAVETMPS;
+ PUSHMARK(SP);
+ XPUSHs(sv_2mortal(newSVsv((SV*)parser)));
+ XPUSHs(sv_2mortal(newSVpvn(content, strlen(content))));
+ XPUSHs(sv_2mortal(newSVpvn(source_class, strlen(source_class))));
+ PUTBACK;
+
+ int count = call_pv("Clownfish::Hierarchy::_do_parse_file", G_SCALAR);
+
+ SPAGAIN;
+
+ if (count != 1) {
+ CFCUtil_die("call to _do_parse_file failed\n");
+ }
+
+ SV *got = POPs;
+ CFCFile *file = NULL;
+ if (sv_derived_from(got, "Clownfish::File")) {
+ IV tmp = SvIV(SvRV(got));
+ file = INT2PTR(CFCFile*, tmp);
+ CFCBase_incref((CFCBase*)file);
+ }
+
+ PUTBACK;
+ FREETMPS;
+ LEAVE;
+
+ return file;
+}
+
+static char**
+S_find_cfh(char *dir, char **cfh_list, size_t num_cfh) {
+ void *dirhandle = S_opendir(dir);
+ size_t full_path_cap = strlen(dir) * 2;
+ char *full_path = (char*)MALLOCATE(full_path_cap);
+ const char *entry = NULL;
+ while (NULL != (entry = S_next_entry(dirhandle))) {
+ // Ignore updirs and hidden files.
+ if (strncmp(entry, ".", 1) == 0) {
+ continue;
+ }
+
+ size_t name_len = strlen(entry);
+ size_t needed = strlen(dir) + 1 + name_len + 1;
+ if (needed > full_path_cap) {
+ full_path_cap = needed;
+ full_path = (char*)MALLOCATE(full_path_cap);
+ }
+ int full_path_len = sprintf(full_path, "%s" PATH_SEP "%s", dir, entry);
+ if (full_path_len < 0) { CFCUtil_die("sprintf failed"); }
+ const char *cfh_suffix = strstr(full_path, ".cfh");
+
+ if (cfh_suffix == full_path + (full_path_len - 4)) {
+ cfh_list = (char**)REALLOCATE(cfh_list,
+ (num_cfh + 2) * sizeof(char*));
+ cfh_list[num_cfh++] = CFCUtil_strdup(full_path);
+ cfh_list[num_cfh] = NULL;
+ }
+ else if (S_is_dir(full_path)) {
+ cfh_list = S_find_cfh(full_path, cfh_list, num_cfh);
+ num_cfh = 0;
+ if (cfh_list) {
+ while (cfh_list[num_cfh] != NULL) { num_cfh++; }
+ }
+ }
+ }
+
+ FREEMEM(full_path);
+ S_closedir(dirhandle, dir);
+ return cfh_list;
+}
+
+static void
+S_parse_cf_files(CFCHierarchy *self) {
+ char **all_source_paths = (char**)CALLOCATE(1, sizeof(char*));
+ all_source_paths = S_find_cfh(self->source, all_source_paths, 0);
+ const char *source_dir = self->source;
+ size_t source_dir_len = strlen(source_dir);
+ size_t all_classes_cap = 10;
+ size_t num_classes = 0;
+ CFCClass **all_classes = (CFCClass**)MALLOCATE(
+ (all_classes_cap + 1) * sizeof(CFCClass*));
+ char *source_class = NULL;
+ size_t source_class_max = 0;
+
+ // Process any file that has at least one class declaration.
+ int i;
+ for (i = 0; all_source_paths[i] != NULL; i++) {
+ // Derive the name of the class that owns the module file.
+ char *source_path = all_source_paths[i];
+ size_t source_path_len = strlen(source_path);
+ if (strncmp(source_path, source_dir, source_dir_len) != 0) {
+ CFCUtil_die("'%s' doesn't start with '%s'", source_path,
+ source_dir);
+ }
+ size_t j;
+ size_t source_class_len = 0;
+ if (source_class_max < source_path_len * 2 + 1) {
+ source_class_max = source_path_len * 2 + 1;
+ source_class = (char*)REALLOCATE(source_class, source_class_max);
+ }
+ for (j = source_dir_len; j < source_path_len - strlen(".cfh"); j++) {
+ char c = source_path[j];
+ if (isalnum(c)) {
+ source_class[source_class_len++] = c;
+ }
+ else {
+ if (source_class_len != 0) {
+ source_class[source_class_len++] = ':';
+ source_class[source_class_len++] = ':';
+ }
+ }
+ }
+ source_class[source_class_len] = '\0';
+
+ // Slurp, parse, add parsed file to pool.
+ size_t unused;
+ char *content = CFCUtil_slurp_text(source_path, &unused);
+ CFCFile *file = S_parse_file(self->parser, content, source_class);
+ if (!file) {
+ croak("parser error for %s", source_path);
+ }
+ S_add_file(self, file);
+
+ CFCClass **classes_in_file = CFCFile_classes(file);
+ for (j = 0; classes_in_file[j] != NULL; j++) {
+ if (num_classes == all_classes_cap) {
+ all_classes_cap += 10;
+ all_classes = (CFCClass**)REALLOCATE(
+ all_classes,
+ (all_classes_cap + 1) * sizeof(CFCClass*));
+ }
+ all_classes[num_classes++] = classes_in_file[j];
+ }
+ }
+ all_classes[num_classes] = NULL;
+
+ // Wrangle the classes into hierarchies and figure out inheritance.
+ for (i = 0; all_classes[i] != NULL; i++) {
+ CFCClass *klass = all_classes[i];
+ const char *parent_name = CFCClass_get_parent_class_name(klass);
+ if (parent_name) {
+ size_t j;
+ for (j = 0; ; j++) {
+ CFCClass *maybe_parent = all_classes[j];
+ if (!maybe_parent) {
+ CFCUtil_die("Parent class '%s' not defined", parent_name);
+ }
+ const char *maybe_parent_name
+ = CFCSymbol_get_class_name((CFCSymbol*)maybe_parent);
+ if (strcmp(parent_name, maybe_parent_name) == 0) {
+ CFCClass_add_child(maybe_parent, klass);
+ break;
+ }
+ }
+ }
+ else {
+ S_add_tree(self, klass);
+ }
+ }
+
+ FREEMEM(all_classes);
+ for (i = 0; all_source_paths[i] != NULL; i++) {
+ FREEMEM(all_source_paths[i]);
+ }
+ FREEMEM(all_source_paths);
+ FREEMEM(source_class);
+}
+
+int
+CFCHierarchy_propagate_modified(CFCHierarchy *self, int modified) {
+ // Seed the recursive write.
+ int somebody_is_modified = false;
+ size_t i;
+ for (i = 0; self->trees[i] != NULL; i++) {
+ CFCClass *tree = self->trees[i];
+ if (S_do_propagate_modified(self, tree, modified)) {
+ somebody_is_modified = true;
+ }
+ }
+ if (somebody_is_modified || modified) {
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+int
+S_do_propagate_modified(CFCHierarchy *self, CFCClass *klass, int modified) {
+ const char *source_class = CFCClass_get_source_class(klass);
+ CFCFile *file = S_fetch_file(self, source_class);
+ size_t cfh_buf_size = CFCFile_path_buf_size(file, self->source);
+ char *source_path = (char*)MALLOCATE(cfh_buf_size);
+ CFCFile_cfh_path(file, source_path, cfh_buf_size, self->source);
+ size_t h_buf_size = CFCFile_path_buf_size(file, self->dest);
+ char *h_path = (char*)MALLOCATE(h_buf_size);
+ CFCFile_h_path(file, h_path, h_buf_size, self->dest);
+
+ if (!CFCUtil_current(source_path, h_path)) {
+ modified = true;
+ }
+ if (modified) {
+ CFCFile_set_modified(file, modified);
+ }
+
+ // Proceed to the next generation.
+ int somebody_is_modified = modified;
+ size_t i;
+ CFCClass **children = CFCClass_children(klass);
+ for (i = 0; children[i] != NULL; i++) {
+ CFCClass *kid = children[i];
+ if (CFCClass_final(klass)) {
+ CFCUtil_die("Attempt to inherit from final class '%s' by '%s'",
+ CFCSymbol_get_class_name((CFCSymbol*)klass),
+ CFCSymbol_get_class_name((CFCSymbol*)kid));
+ }
+ if (S_do_propagate_modified(self, kid, modified)) {
+ somebody_is_modified = 1;
+ }
+ }
+
+ return somebody_is_modified;
+}
+
+static void
+S_add_tree(CFCHierarchy *self, CFCClass *klass) {
+ CFCUTIL_NULL_CHECK(klass);
+ const char *full_struct_sym = CFCClass_full_struct_sym(klass);
+ size_t i;
+ for (i = 0; self->trees[i] != NULL; i++) {
+ const char *existing = CFCClass_full_struct_sym(self->trees[i]);
+ if (strcmp(full_struct_sym, existing) == 0) {
+ CFCUtil_die("Tree '%s' alread added", full_struct_sym);
+ }
+ }
+ self->num_trees++;
+ size_t size = (self->num_trees + 1) * sizeof(CFCClass*);
+ self->trees = (CFCClass**)REALLOCATE(self->trees, size);
+ self->trees[self->num_trees - 1]
+ = (CFCClass*)CFCBase_incref((CFCBase*)klass);
+ self->trees[self->num_trees] = NULL;
+}
+
+CFCClass**
+CFCHierarchy_ordered_classes(CFCHierarchy *self) {
+ size_t num_classes = 0;
+ size_t max_classes = 10;
+ CFCClass **ladder = (CFCClass**)MALLOCATE(
+ (max_classes + 1) * sizeof(CFCClass*));
+ size_t i;
+ for (i = 0; self->trees[i] != NULL; i++) {
+ CFCClass *tree = self->trees[i];
+ CFCClass **child_ladder = CFCClass_tree_to_ladder(tree);
+ size_t j;
+ for (j = 0; child_ladder[j] != NULL; j++) {
+ if (num_classes == max_classes) {
+ max_classes += 10;
+ ladder = (CFCClass**)REALLOCATE(
+ ladder, (max_classes + 1) * sizeof(CFCClass*));
+ }
+ ladder[num_classes++] = child_ladder[j];
+ }
+ FREEMEM(child_ladder);
+ }
+ ladder[num_classes] = NULL;
+ return ladder;
+}
+
+static CFCFile*
+S_fetch_file(CFCHierarchy *self, const char *source_class) {
+ size_t i;
+ for (i = 0; self->files[i] != NULL; i++) {
+ const char *existing = CFCFile_get_source_class(self->files[i]);
+ if (strcmp(source_class, existing) == 0) {
+ return self->files[i];
+ }
+ }
+ return NULL;
+}
+
+static void
+S_add_file(CFCHierarchy *self, CFCFile *file) {
+ CFCUTIL_NULL_CHECK(file);
+ const char *source_class = CFCFile_get_source_class(file);
+ CFCClass **classes = CFCFile_classes(file);
+ size_t i;
+ for (i = 0; self->files[i] != NULL; i++) {
+ CFCFile *existing = self->files[i];
+ const char *old_source_class = CFCFile_get_source_class(existing);
+ if (strcmp(source_class, old_source_class) == 0) {
+ CFCUtil_die("File for source class %s already registered",
+ source_class);
+ }
+ CFCClass **existing_classes = CFCFile_classes(existing);
+ size_t j;
+ for (j = 0; classes[j] != NULL; j++) {
+ const char *new_class_name
+ = CFCSymbol_get_class_name((CFCSymbol*)classes[j]);
+ size_t k;
+ for (k = 0; existing_classes[k] != NULL; k++) {
+ const char *existing_class_name
+ = CFCSymbol_get_class_name((CFCSymbol*)existing_classes[k]);
+ if (strcmp(new_class_name, existing_class_name) == 0) {
+ CFCUtil_die("Class '%s' already registered",
+ new_class_name);
+ }
+ }
+ }
+ }
+ self->num_files++;
+ size_t size = (self->num_files + 1) * sizeof(CFCFile*);
+ self->files = (CFCFile**)REALLOCATE(self->files, size);
+ self->files[self->num_files - 1]
+ = (CFCFile*)CFCBase_incref((CFCBase*)file);
+ self->files[self->num_files] = NULL;
+}
+
+struct CFCFile**
+CFCHierarchy_files(CFCHierarchy *self) {
+ return self->files;
+}
+
+const char*
+CFCHierarchy_get_source(CFCHierarchy *self) {
+ return self->source;
+}
+
+const char*
+CFCHierarchy_get_dest(CFCHierarchy *self) {
+ return self->dest;
+}
+
+/******************************** WINDOWS **********************************/
+#ifdef WIN32
+
+#include <windows.h>
+
+typedef struct WinDH {
+ HANDLE handle;
+ WIN32_FIND_DATA *find_data;
+ char path[MAX_PATH + 1];
+ int first_time;
+} WinDH;
+
+static void*
+S_opendir(const char *dir) {
+ size_t dirlen = strlen(dir);
+ if (dirlen >= MAX_PATH - 2) {
+ CFCUtil_die("Exceeded MAX_PATH(%d): %s", (int)MAX_PATH, dir);
+ }
+ WinDH *dh = (WinDH*)CALLOCATE(1, sizeof(WinDH));
+ dh->find_data = (WIN32_FIND_DATA*)MALLOCATE(sizeof(WIN32_FIND_DATA));
+
+ // Tack on wildcard needed by FindFirstFile.
+ int check = sprintf(dh->path, "%s\\*", dir);
+ if (check < 0) { CFCUtil_die("sprintf failed"); }
+
+ dh->handle = FindFirstFile(dh->path, dh->find_data);
+ if (dh->handle == INVALID_HANDLE_VALUE) {
+ CFCUtil_die("Can't open dir '%s'", dh->path);
+ }
+ dh->first_time = true;
+
+ return dh;
+}
+
+static const char*
+S_next_entry(void *dirhandle) {
+ WinDH *dh = (WinDH*)dirhandle;
+ if (dh->first_time) {
+ dh->first_time = false;
+ }
+ else {
+ if ((FindNextFile(dh->handle, dh->find_data) == 0)) {
+ if (GetLastError() != ERROR_NO_MORE_FILES) {
+ CFCUtil_die("Error occurred while reading '%s'",
+ dh->path);
+ }
+ return NULL;
+ }
+ }
+ return dh->find_data->cFileName;
+}
+
+static void
+S_closedir(void *dirhandle, const char *dir) {
+ WinDH *dh = (WinDH*)dirhandle;
+ if (!FindClose(dh->handle)) {
+ CFCUtil_die("Error occurred while closing dir '%s'", dir);
+ }
+ FREEMEM(dh->find_data);
+ FREEMEM(dh);
+}
+
+/******************************** UNIXEN ***********************************/
+#else
+
+#include <dirent.h>
+
+static void*
+S_opendir(const char *dir) {
+ DIR *dirhandle = opendir(dir);
+ if (!dirhandle) {
+ CFCUtil_die("Failed to opendir for '%s': %s", dir, strerror(errno));
+ }
+ return dirhandle;
+}
+
+static const char*
+S_next_entry(void *dirhandle) {
+ struct dirent *entry = readdir((DIR*)dirhandle);
+ return entry ? entry->d_name : NULL;
+}
+
+static void
+S_closedir(void *dirhandle, const char *dir) {
+ if (closedir(dirhandle) == -1) {
+ CFCUtil_die("Error closing dir '%s': %s", dir, strerror(errno));
+ }
+}
+
+#endif
+
diff --git a/clownfish/src/CFCHierarchy.h b/clownfish/src/CFCHierarchy.h
new file mode 100644
index 0000000..a472c23
--- /dev/null
+++ b/clownfish/src/CFCHierarchy.h
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCHIERARCHY
+#define H_CFCHIERARCHY
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCHierarchy CFCHierarchy;
+struct CFCClass;
+struct CFCFile;
+
+CFCHierarchy*
+CFCHierarchy_new(const char *source, const char *dest, void *parser);
+
+CFCHierarchy*
+CFCHierarchy_init(CFCHierarchy *self, const char *source, const char *dest,
+ void *parser);
+
+void
+CFCHierarchy_destroy(CFCHierarchy *self);
+
+void
+CFCHierarchy_build(CFCHierarchy *self);
+
+int
+CFCHierarchy_propagate_modified(CFCHierarchy *self, int modified);
+
+struct CFCClass**
+CFCHierarchy_ordered_classes(CFCHierarchy *self);
+
+struct CFCFile**
+CFCHierarchy_files(CFCHierarchy *self);
+
+const char*
+CFCHierarchy_get_source(CFCHierarchy *self);
+
+const char*
+CFCHierarchy_get_dest(CFCHierarchy *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCHIERARCHY */
+
diff --git a/clownfish/src/CFCMethod.c b/clownfish/src/CFCMethod.c
new file mode 100644
index 0000000..36d99ad
--- /dev/null
+++ b/clownfish/src/CFCMethod.c
@@ -0,0 +1,355 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#define CFC_NEED_FUNCTION_STRUCT_DEF
+#include "CFCFunction.h"
+#include "CFCMethod.h"
+#include "CFCType.h"
+#include "CFCUtil.h"
+#include "CFCParamList.h"
+#include "CFCParcel.h"
+#include "CFCDocuComment.h"
+#include "CFCVariable.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+struct CFCMethod {
+ CFCFunction function;
+ char *macro_sym;
+ char *short_typedef;
+ char *full_typedef;
+ char *full_callback_sym;
+ char *full_override_sym;
+ int is_final;
+ int is_abstract;
+ int is_novel;
+};
+
+static void
+S_update_typedefs(CFCMethod *self, const char *short_sym);
+
+CFCMethod*
+CFCMethod_new(CFCParcel *parcel, const char *exposure, const char *class_name,
+ const char *class_cnick, const char *macro_sym,
+ CFCType *return_type, CFCParamList *param_list,
+ CFCDocuComment *docucomment, int is_final, int is_abstract) {
+ CFCMethod *self = (CFCMethod*)CFCBase_allocate(sizeof(CFCMethod),
+ "Clownfish::Method");
+ return CFCMethod_init(self, parcel, exposure, class_name, class_cnick,
+ macro_sym, return_type, param_list, docucomment,
+ is_final, is_abstract);
+}
+
+static int
+S_validate_macro_sym(const char *macro_sym) {
+ if (!macro_sym || !strlen(macro_sym)) { return false; }
+
+ int need_upper = true;
+ int need_letter = true;
+ for (;; macro_sym++) {
+ if (need_upper && !isupper(*macro_sym)) { return false; }
+ if (need_letter && !isalpha(*macro_sym)) { return false; }
+ need_upper = false;
+ need_letter = false;
+
+ // We've reached NULL-termination without problems, so succeed.
+ if (!*macro_sym) { return true; }
+
+ if (!isalnum(*macro_sym)) {
+ if (*macro_sym != '_') { return false; }
+ need_upper = true;
+ }
+ }
+}
+
+CFCMethod*
+CFCMethod_init(CFCMethod *self, CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *macro_sym, CFCType *return_type,
+ CFCParamList *param_list, CFCDocuComment *docucomment,
+ int is_final, int is_abstract) {
+ // Validate macro_sym, derive micro_sym.
+ if (!S_validate_macro_sym(macro_sym)) {
+ croak("Invalid macro_sym: '%s'", macro_sym ? macro_sym : "[NULL]");
+ }
+ char *micro_sym = CFCUtil_strdup(macro_sym);
+ size_t i;
+ for (i = 0; micro_sym[i] != '\0'; i++) {
+ micro_sym[i] = tolower(micro_sym[i]);
+ }
+
+ // Super-init and clean up derived micro_sym.
+ CFCFunction_init((CFCFunction*)self, parcel, exposure, class_name,
+ class_cnick, micro_sym, return_type, param_list,
+ docucomment, false);
+ FREEMEM(micro_sym);
+
+ // Verify that the first element in the arg list is a self.
+ CFCVariable **args = CFCParamList_get_variables(param_list);
+ if (!args[0]) { croak("Missing 'self' argument"); }
+ CFCType *type = CFCVariable_get_type(args[0]);
+ const char *specifier = CFCType_get_specifier(type);
+ const char *prefix = CFCSymbol_get_prefix((CFCSymbol*)self);
+ const char *last_colon = strrchr(class_name, ':');
+ const char *struct_sym = last_colon ? last_colon + 1 : class_name;
+ char *wanted = (char*)MALLOCATE(strlen(prefix) + strlen(struct_sym) + 1);
+ sprintf(wanted, "%s%s", prefix, struct_sym);
+ int mismatch = strcmp(wanted, specifier);
+ FREEMEM(wanted);
+ if (mismatch) {
+ croak("First arg type doesn't match class: '%s' '%s", class_name,
+ specifier);
+ }
+
+ self->macro_sym = CFCUtil_strdup(macro_sym);
+ self->short_typedef = NULL;
+ self->full_typedef = NULL;
+ self->is_final = is_final;
+ self->is_abstract = is_abstract;
+
+ // Derive more symbols.
+ const char *full_func_sym = CFCFunction_full_func_sym((CFCFunction*)self);
+ size_t amount = strlen(full_func_sym) + sizeof("_OVERRIDE") + 1;
+ self->full_callback_sym = (char*)MALLOCATE(amount);
+ self->full_override_sym = (char*)MALLOCATE(amount);
+ int check = sprintf(self->full_callback_sym, "%s_CALLBACK",
+ full_func_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ check = sprintf(self->full_override_sym, "%s_OVERRIDE", full_func_sym);
+ if (check < 0) { croak("sprintf failed"); }
+
+ // Assume that this method is novel until we discover when applying
+ // inheritance that it was overridden.
+ self->is_novel = 1;
+
+ // Cache typedefs.
+ S_update_typedefs(self, CFCSymbol_short_sym((CFCSymbol*)self));
+
+ return self;
+}
+
+void
+CFCMethod_destroy(CFCMethod *self) {
+ FREEMEM(self->macro_sym);
+ FREEMEM(self->short_typedef);
+ FREEMEM(self->full_typedef);
+ FREEMEM(self->full_callback_sym);
+ FREEMEM(self->full_override_sym);
+ CFCFunction_destroy((CFCFunction*)self);
+}
+
+int
+CFCMethod_compatible(CFCMethod *self, CFCMethod *other) {
+ if (!other) { return false; }
+ if (strcmp(self->macro_sym, other->macro_sym)) { return false; }
+ int my_public = CFCSymbol_public((CFCSymbol*)self);
+ int other_public = CFCSymbol_public((CFCSymbol*)self);
+ if (!!my_public != !!other_public) { return false; }
+
+ // Check arguments and initial values.
+ CFCParamList *my_param_list = self->function.param_list;
+ CFCParamList *other_param_list = other->function.param_list;
+ CFCVariable **my_args = CFCParamList_get_variables(my_param_list);
+ CFCVariable **other_args = CFCParamList_get_variables(other_param_list);
+ const char **my_vals = CFCParamList_get_initial_values(my_param_list);
+ const char **other_vals = CFCParamList_get_initial_values(other_param_list);
+ size_t i;
+ for (i = 1; ; i++) { // start at 1, skipping self
+ if (!!my_args[i] != !!other_args[i]) { return false; }
+ if (!!my_vals[i] != !!other_vals[i]) { return false; }
+ if (my_vals[i]) {
+ if (strcmp(my_vals[i], other_vals[i])) { return false; }
+ }
+ if (my_args[i]) {
+ if (!CFCVariable_equals(my_args[i], other_args[i])) {
+ return false;
+ }
+ }
+ else {
+ break;
+ }
+ }
+
+ // Check return types.
+ CFCType *type = CFCFunction_get_return_type((CFCFunction*)self);
+ CFCType *other_type = CFCFunction_get_return_type((CFCFunction*)other);
+ if (CFCType_is_object(type)) {
+ // Weak validation to allow covariant object return types.
+ if (!CFCType_is_object(other_type)) { return false; }
+ if (!CFCType_similar(type, other_type)) { return false; }
+ }
+ else {
+ if (!CFCType_equals(type, other_type)) { return false; }
+ }
+
+ return true;
+}
+
+void
+CFCMethod_override(CFCMethod *self, CFCMethod *orig) {
+ // Check that the override attempt is legal.
+ if (CFCMethod_final(orig)) {
+ const char *orig_class = CFCSymbol_get_class_name((CFCSymbol*)orig);
+ const char *my_class = CFCSymbol_get_class_name((CFCSymbol*)self);
+ croak("Attempt to override final method '%s' from '%s' by '%s'",
+ orig->macro_sym, orig_class, my_class);
+ }
+ if (!CFCMethod_compatible(self, orig)) {
+ const char *func = CFCFunction_full_func_sym((CFCFunction*)self);
+ const char *orig_func = CFCFunction_full_func_sym((CFCFunction*)orig);
+ croak("Non-matching signatures for %s and %s", func, orig_func);
+ }
+
+ // Mark the Method as no longer novel.
+ self->is_novel = false;
+}
+
+CFCMethod*
+CFCMethod_finalize(CFCMethod *self) {
+ CFCSymbol *self_sym = (CFCSymbol*)self;
+ CFCParcel *parcel = CFCSymbol_get_parcel(self_sym);
+ const char *exposure = CFCSymbol_get_exposure(self_sym);
+ const char *class_name = CFCSymbol_get_class_name(self_sym);
+ const char *class_cnick = CFCSymbol_get_class_cnick(self_sym);
+ CFCMethod *finalized
+ = CFCMethod_new(parcel, exposure, class_name, class_cnick,
+ self->macro_sym, self->function.return_type,
+ self->function.param_list,
+ self->function.docucomment, true,
+ self->is_abstract);
+ finalized->is_novel = self->is_final; // Is this right?
+ S_update_typedefs(finalized, CFCSymbol_short_sym((CFCSymbol*)self));
+ return finalized;
+}
+
+size_t
+CFCMethod_short_method_sym(CFCMethod *self, const char *invoker, char *buf,
+ size_t buf_size) {
+ CFCUTIL_NULL_CHECK(invoker);
+ size_t needed = strlen(invoker) + 1 + strlen(self->macro_sym) + 1;
+ if (buf_size >= needed) {
+ int check = sprintf(buf, "%s_%s", invoker, self->macro_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ return needed;
+}
+
+size_t
+CFCMethod_full_method_sym(CFCMethod *self, const char *invoker, char *buf,
+ size_t buf_size) {
+ CFCUTIL_NULL_CHECK(invoker);
+ const char *Prefix = CFCSymbol_get_Prefix((CFCSymbol*)self);
+ size_t needed = strlen(Prefix)
+ + strlen(invoker)
+ + 1
+ + strlen(self->macro_sym)
+ + 1;
+ if (buf_size >= needed) {
+ int check = sprintf(buf, "%s%s_%s", Prefix, invoker, self->macro_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ return needed;
+}
+
+size_t
+CFCMethod_full_offset_sym(CFCMethod *self, const char *invoker, char *buf,
+ size_t buf_size) {
+ CFCUTIL_NULL_CHECK(invoker);
+ size_t needed = CFCMethod_full_method_sym(self, invoker, NULL, 0)
+ + strlen("_OFFSET");
+ if (buf_size >= needed) {
+ CFCMethod_full_method_sym(self, invoker, buf, buf_size);
+ strcat(buf, "_OFFSET");
+ }
+ return needed;
+}
+
+const char*
+CFCMethod_get_macro_sym(CFCMethod *self) {
+ return self->macro_sym;
+}
+
+static void
+S_update_typedefs(CFCMethod *self, const char *short_sym) {
+ FREEMEM(self->short_typedef);
+ FREEMEM(self->full_typedef);
+ if (short_sym) {
+ const char *prefix = CFCSymbol_get_prefix((CFCSymbol*)self);
+ size_t amount = strlen(short_sym) + 3;
+ self->short_typedef = (char*)MALLOCATE(amount);
+ int check = sprintf(self->short_typedef, "%s_t", short_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ amount += strlen(prefix);
+ self->full_typedef = (char*)MALLOCATE(amount);
+ check = sprintf(self->full_typedef, "%s%s_t", prefix, short_sym);
+ if (check < 0) { croak("sprintf failed"); }
+ }
+ else {
+ self->short_typedef = NULL;
+ self->full_typedef = NULL;
+ }
+}
+
+const char*
+CFCMethod_short_typedef(CFCMethod *self) {
+ return self->short_typedef;
+}
+
+const char*
+CFCMethod_full_typedef(CFCMethod *self) {
+ return self->full_typedef;
+}
+
+const char*
+CFCMethod_full_callback_sym(CFCMethod *self) {
+ return self->full_callback_sym;
+}
+
+const char*
+CFCMethod_full_override_sym(CFCMethod *self) {
+ return self->full_override_sym;
+}
+
+int
+CFCMethod_final(CFCMethod *self) {
+ return self->is_final;
+}
+
+int
+CFCMethod_abstract(CFCMethod *self) {
+ return self->is_abstract;
+}
+
+int
+CFCMethod_novel(CFCMethod *self) {
+ return self->is_novel;
+}
+
+CFCType*
+CFCMethod_self_type(CFCMethod *self) {
+ CFCVariable **vars = CFCParamList_get_variables(self->function.param_list);
+ return CFCVariable_get_type(vars[0]);
+}
+
diff --git a/clownfish/src/CFCMethod.h b/clownfish/src/CFCMethod.h
new file mode 100644
index 0000000..ed3d5bf
--- /dev/null
+++ b/clownfish/src/CFCMethod.h
@@ -0,0 +1,111 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCMETHOD
+#define H_CFCMETHOD
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCMethod CFCMethod;
+struct CFCParcel;
+struct CFCType;
+struct CFCParamList;
+struct CFCDocuComment;
+
+CFCMethod*
+CFCMethod_new(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *macro_sym, struct CFCType *return_type,
+ struct CFCParamList *param_list,
+ struct CFCDocuComment *docucomment, int is_final,
+ int is_abstract);
+
+CFCMethod*
+CFCMethod_init(CFCMethod *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *class_cnick, const char *macro_sym,
+ struct CFCType *return_type, struct CFCParamList *param_list,
+ struct CFCDocuComment *docucomment, int is_final,
+ int is_abstract);
+
+void
+CFCMethod_destroy(CFCMethod *self);
+
+int
+CFCMethod_compatible(CFCMethod *self, CFCMethod *other);
+
+void
+CFCMethod_override(CFCMethod *self, CFCMethod *orig);
+
+CFCMethod*
+CFCMethod_finalize(CFCMethod *self);
+
+/**
+ * @return the number of bytes which the symbol would occupy.
+ */
+size_t
+CFCMethod_short_method_sym(CFCMethod *self, const char *invoker, char *buf,
+ size_t buf_size);
+
+/**
+ * @return the number of bytes which the symbol would occupy.
+ */
+size_t
+CFCMethod_full_method_sym(CFCMethod *self, const char *invoker, char *buf,
+ size_t buf_size);
+
+/**
+ * @return the number of bytes which the symbol would occupy.
+ */
+size_t
+CFCMethod_full_offset_sym(CFCMethod *self, const char *invoker, char *buf,
+ size_t buf_size);
+
+const char*
+CFCMethod_get_macro_sym(CFCMethod *self);
+
+const char*
+CFCMethod_short_typedef(CFCMethod *self);
+
+const char*
+CFCMethod_full_typedef(CFCMethod *self);
+
+const char*
+CFCMethod_full_callback_sym(CFCMethod *self);
+
+const char*
+CFCMethod_full_override_sym(CFCMethod *self);
+
+int
+CFCMethod_final(CFCMethod *self);
+
+int
+CFCMethod_abstract(CFCMethod *self);
+
+int
+CFCMethod_novel(CFCMethod *self);
+
+struct CFCType*
+CFCMethod_self_type(CFCMethod *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCMETHOD */
+
diff --git a/clownfish/src/CFCParamList.c b/clownfish/src/CFCParamList.c
new file mode 100644
index 0000000..3a0b3fc
--- /dev/null
+++ b/clownfish/src/CFCParamList.c
@@ -0,0 +1,162 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCParamList.h"
+#include "CFCVariable.h"
+#include "CFCSymbol.h"
+#include "CFCUtil.h"
+
+struct CFCParamList {
+ CFCBase base;
+ CFCVariable **variables;
+ char **values;
+ int variadic;
+ size_t num_vars;
+ char *c_string;
+ char *name_list;
+};
+
+//
+static void
+S_generate_c_strings(CFCParamList *self);
+
+CFCParamList*
+CFCParamList_new(int variadic) {
+ CFCParamList *self = (CFCParamList*)CFCBase_allocate(sizeof(CFCParamList),
+ "Clownfish::ParamList");
+ return CFCParamList_init(self, variadic);
+}
+
+CFCParamList*
+CFCParamList_init(CFCParamList *self, int variadic) {
+ self->variadic = variadic;
+ self->num_vars = 0;
+ self->variables = (CFCVariable**)CALLOCATE(1, sizeof(void*));
+ self->values = (char**)CALLOCATE(1, sizeof(char*));
+ S_generate_c_strings(self);
+ return self;
+}
+
+void
+CFCParamList_add_param(CFCParamList *self, CFCVariable *variable,
+ const char *value) {
+ CFCUTIL_NULL_CHECK(variable);
+ self->num_vars++;
+ size_t amount = (self->num_vars + 1) * sizeof(void*);
+ self->variables = (CFCVariable**)REALLOCATE(self->variables, amount);
+ self->values = (char**)REALLOCATE(self->values, amount);
+ self->variables[self->num_vars - 1]
+ = (CFCVariable*)CFCBase_incref((CFCBase*)variable);
+ self->values[self->num_vars - 1] = value ? CFCUtil_strdup(value) : NULL;
+ self->variables[self->num_vars] = NULL;
+ self->values[self->num_vars] = NULL;
+
+ S_generate_c_strings(self);
+}
+
+void
+CFCParamList_destroy(CFCParamList *self) {
+ size_t i;
+ for (i = 0; i < self->num_vars; i++) {
+ CFCBase_decref((CFCBase*)self->variables[i]);
+ FREEMEM(self->values[i]);
+ }
+ FREEMEM(self->variables);
+ FREEMEM(self->values);
+ FREEMEM(self->c_string);
+ FREEMEM(self->name_list);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+static void
+S_generate_c_strings(CFCParamList *self) {
+ size_t c_string_size = 1;
+ size_t name_list_size = 1;
+ size_t i;
+
+ // Calc space requirements and allocate memory.
+ for (i = 0; i < self->num_vars; i++) {
+ CFCVariable *var = self->variables[i];
+ c_string_size += sizeof(", ");
+ c_string_size += strlen(CFCVariable_local_c(var));
+ name_list_size += sizeof(", ");
+ name_list_size += strlen(CFCSymbol_micro_sym((CFCSymbol*)var));
+ }
+ if (self->variadic) {
+ c_string_size += sizeof(", ...");
+ }
+ FREEMEM(self->c_string);
+ FREEMEM(self->name_list);
+ self->c_string = (char*)MALLOCATE(c_string_size);
+ self->name_list = (char*)MALLOCATE(name_list_size);
+ self->c_string[0] = '\0';
+ self->name_list[0] = '\0';
+
+ // Build the strings.
+ for (i = 0; i < self->num_vars; i++) {
+ CFCVariable *var = self->variables[i];
+ strcat(self->c_string, CFCVariable_local_c(var));
+ strcat(self->name_list, CFCSymbol_micro_sym((CFCSymbol*)var));
+ if (i == self->num_vars - 1) {
+ if (self->variadic) {
+ strcat(self->c_string, ", ...");
+ }
+ }
+ else {
+ strcat(self->c_string, ", ");
+ strcat(self->name_list, ", ");
+ }
+ }
+}
+
+CFCVariable**
+CFCParamList_get_variables(CFCParamList *self) {
+ return self->variables;
+}
+
+const char**
+CFCParamList_get_initial_values(CFCParamList *self) {
+ return (const char**)self->values;
+}
+
+size_t
+CFCParamList_num_vars(CFCParamList *self) {
+ return self->num_vars;
+}
+
+int
+CFCParamList_variadic(CFCParamList *self) {
+ return self->variadic;
+}
+
+const char*
+CFCParamList_to_c(CFCParamList *self) {
+ return self->c_string;
+}
+
+const char*
+CFCParamList_name_list(CFCParamList *self) {
+ return self->name_list;
+}
+
diff --git a/clownfish/src/CFCParamList.h b/clownfish/src/CFCParamList.h
new file mode 100644
index 0000000..853bb90
--- /dev/null
+++ b/clownfish/src/CFCParamList.h
@@ -0,0 +1,63 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCPARAMLIST
+#define H_CFCPARAMLIST
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCParamList CFCParamList;
+struct CFCVariable;
+
+CFCParamList*
+CFCParamList_new(int variadic);
+
+CFCParamList*
+CFCParamList_init(CFCParamList *self, int variadic);
+
+void
+CFCParamList_destroy(CFCParamList *self);
+
+void
+CFCParamList_add_param(CFCParamList *self, struct CFCVariable *variable,
+ const char *value);
+
+struct CFCVariable**
+CFCParamList_get_variables(CFCParamList *self);
+
+const char**
+CFCParamList_get_initial_values(CFCParamList *self);
+
+int
+CFCParamList_variadic(CFCParamList *self);
+
+size_t
+CFCParamList_num_vars(CFCParamList *self);
+
+const char*
+CFCParamList_to_c(CFCParamList *self);
+
+const char*
+CFCParamList_name_list(CFCParamList *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCPARAMLIST */
+
diff --git a/clownfish/src/CFCParcel.c b/clownfish/src/CFCParcel.c
new file mode 100644
index 0000000..a8bb94f
--- /dev/null
+++ b/clownfish/src/CFCParcel.c
@@ -0,0 +1,217 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCParcel.h"
+#include "CFCUtil.h"
+
+struct CFCParcel {
+ CFCBase base;
+ char *name;
+ char *cnick;
+ char *prefix;
+ char *Prefix;
+ char *PREFIX;
+};
+
+#define MAX_PARCELS 100
+static CFCParcel *registry[MAX_PARCELS + 1];
+static int first_time = true;
+
+CFCParcel*
+CFCParcel_singleton(const char *name, const char *cnick) {
+ // Set up registry.
+ if (first_time) {
+ size_t i;
+ for (i = 1; i < MAX_PARCELS; i++) { registry[i] = NULL; }
+ first_time = false;
+ }
+
+ // Return the default parcel for either a blank name or a NULL name.
+ if (!name || !strlen(name)) {
+ return CFCParcel_default_parcel();
+ }
+
+ // Return an existing singleton if the parcel has already been registered.
+ size_t i;
+ for (i = 1; registry[i] != NULL; i++) {
+ CFCParcel *existing = registry[i];
+ if (strcmp(existing->name, name) == 0) {
+ if (cnick && strcmp(existing->cnick, cnick) != 0) {
+ croak("cnick '%s' for parcel '%s' conflicts with '%s'",
+ cnick, name, existing->cnick);
+ }
+ return existing;
+ }
+ }
+ if (i == MAX_PARCELS) {
+ croak("Exceeded maximum number of parcels (%d)", MAX_PARCELS);
+ }
+
+ // Register new parcel.
+ CFCParcel *singleton = CFCParcel_new(name, cnick);
+ registry[i] = singleton;
+
+ return singleton;
+}
+
+void
+CFCParcel_reap_singletons(void) {
+ if (registry[0]) {
+ // default parcel.
+ CFCBase_decref((CFCBase*)registry[0]);
+ }
+ int i;
+ for (i = 1; registry[i] != NULL; i++) {
+ CFCParcel *parcel = registry[i];
+ CFCBase_decref((CFCBase*)parcel);
+ }
+}
+
+static int
+S_validate_name_or_cnick(const char *orig) {
+ const char *ptr = orig;
+ for (; *ptr != 0; ptr++) {
+ if (!isalpha(*ptr)) { return false; }
+ }
+ return true;
+}
+
+CFCParcel*
+CFCParcel_new(const char *name, const char *cnick) {
+ CFCParcel *self = (CFCParcel*)CFCBase_allocate(sizeof(CFCParcel),
+ "Clownfish::Parcel");
+ return CFCParcel_init(self, name, cnick);
+}
+
+CFCParcel*
+CFCParcel_init(CFCParcel *self, const char *name, const char *cnick) {
+ // Validate name.
+ if (!name || !S_validate_name_or_cnick(name)) {
+ croak("Invalid name: '%s'", name ? name : "[NULL]");
+ }
+ self->name = CFCUtil_strdup(name);
+
+ // Validate or derive cnick.
+ if (cnick) {
+ if (!S_validate_name_or_cnick(cnick)) {
+ croak("Invalid cnick: '%s'", cnick);
+ }
+ self->cnick = CFCUtil_strdup(cnick);
+ }
+ else {
+ // Default cnick to name.
+ self->cnick = CFCUtil_strdup(name);
+ }
+
+ // Derive prefix, Prefix, PREFIX.
+ size_t cnick_len = strlen(self->cnick);
+ size_t prefix_len = cnick_len ? cnick_len + 1 : 0;
+ size_t amount = prefix_len + 1;
+ self->prefix = (char*)MALLOCATE(amount);
+ self->Prefix = (char*)MALLOCATE(amount);
+ self->PREFIX = (char*)MALLOCATE(amount);
+ memcpy(self->Prefix, self->cnick, cnick_len);
+ if (cnick_len) {
+ self->Prefix[cnick_len] = '_';
+ self->Prefix[cnick_len + 1] = '\0';
+ }
+ else {
+ self->Prefix[cnick_len] = '\0';
+ }
+ size_t i;
+ for (i = 0; i < amount; i++) {
+ self->prefix[i] = tolower(self->Prefix[i]);
+ self->PREFIX[i] = toupper(self->Prefix[i]);
+ }
+ self->prefix[prefix_len] = '\0';
+ self->Prefix[prefix_len] = '\0';
+ self->PREFIX[prefix_len] = '\0';
+
+ return self;
+}
+
+void
+CFCParcel_destroy(CFCParcel *self) {
+ FREEMEM(self->name);
+ FREEMEM(self->cnick);
+ FREEMEM(self->prefix);
+ FREEMEM(self->Prefix);
+ FREEMEM(self->PREFIX);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+static CFCParcel *default_parcel = NULL;
+
+CFCParcel*
+CFCParcel_default_parcel(void) {
+ if (default_parcel == NULL) {
+ default_parcel = CFCParcel_new("DEFAULT", "");
+ registry[0] = default_parcel;
+ }
+ return default_parcel;
+}
+
+CFCParcel*
+CFCParcel_clownfish_parcel(void) {
+ return CFCParcel_singleton("Lucy", "Lucy");
+}
+
+int
+CFCParcel_equals(CFCParcel *self, CFCParcel *other) {
+ if (strcmp(self->name, other->name)) { return false; }
+ if (strcmp(self->cnick, other->cnick)) { return false; }
+ return true;
+}
+
+const char*
+CFCParcel_get_name(CFCParcel *self) {
+ return self->name;
+}
+
+const char*
+CFCParcel_get_cnick(CFCParcel *self) {
+ return self->cnick;
+}
+
+const char*
+CFCParcel_get_prefix(CFCParcel *self) {
+ return self->prefix;
+}
+
+const char*
+CFCParcel_get_Prefix(CFCParcel *self) {
+ return self->Prefix;
+}
+
+const char*
+CFCParcel_get_PREFIX(CFCParcel *self) {
+ return self->PREFIX;
+}
+
diff --git a/clownfish/src/CFCParcel.h b/clownfish/src/CFCParcel.h
new file mode 100644
index 0000000..3b9f3c5
--- /dev/null
+++ b/clownfish/src/CFCParcel.h
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCPARCEL
+#define H_CFCPARCEL
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCParcel CFCParcel;
+
+CFCParcel*
+CFCParcel_singleton(const char *name, const char *cnick);
+
+/** Decref all singletons at shutdown.
+ */
+void
+CFCParcel_reap_singletons(void);
+
+CFCParcel*
+CFCParcel_new(const char *name, const char *cnick);
+
+CFCParcel*
+CFCParcel_init(CFCParcel *self, const char *name, const char *cnick);
+
+void
+CFCParcel_destroy(CFCParcel *self);
+
+CFCParcel*
+CFCParcel_default_parcel(void);
+
+/** Return the Parcel under which Obj, CharBuf, VArray, Hash, etc. live. At
+ * some point in the future, these core object types may move to the
+ * "Clownfish" Parcel, but for now they are within "Lucy".
+ */
+CFCParcel*
+CFCParcel_clownfish_parcel(void);
+
+int
+CFCParcel_equals(CFCParcel *self, CFCParcel *other);
+
+const char*
+CFCParcel_get_name(CFCParcel *self);
+
+const char*
+CFCParcel_get_cnick(CFCParcel *self);
+
+const char*
+CFCParcel_get_prefix(CFCParcel *self);
+
+const char*
+CFCParcel_get_Prefix(CFCParcel *self);
+
+const char*
+CFCParcel_get_PREFIX(CFCParcel *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCPARCEL */
+
diff --git a/clownfish/src/CFCSymbol.c b/clownfish/src/CFCSymbol.c
new file mode 100644
index 0000000..fc5b360
--- /dev/null
+++ b/clownfish/src/CFCSymbol.c
@@ -0,0 +1,295 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_SYMBOL_STRUCT_DEF
+#include "CFCSymbol.h"
+#include "CFCParcel.h"
+#include "CFCUtil.h"
+
+CFCSymbol*
+CFCSymbol_new(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym) {
+ CFCSymbol *self = (CFCSymbol*)CFCBase_allocate(sizeof(CFCSymbol),
+ "Clownfish::Symbol");
+ return CFCSymbol_init(self, parcel, exposure, class_name, class_cnick,
+ micro_sym);
+}
+
+static int
+S_validate_exposure(const char *exposure) {
+ if (!exposure) { return false; }
+ if (strcmp(exposure, "public")
+ && strcmp(exposure, "parcel")
+ && strcmp(exposure, "private")
+ && strcmp(exposure, "local")
+ ) {
+ return false;
+ }
+ return true;
+}
+
+static int
+S_validate_class_name(const char *class_name) {
+ const char *ptr;
+
+ // Must be UpperCamelCase, separated by "::".
+ for (ptr = class_name; *ptr != 0;) {
+ if (!isupper(*ptr)) { return false; }
+
+ // Each component must contain lowercase letters.
+ const char *substring;
+ for (substring = ptr; ; substring++) {
+ if (*substring == 0) { return false; }
+ else if (*substring == ':') { return false; }
+ else if (islower(*substring)) { break; }
+ }
+
+ while (*ptr != 0) {
+ if (*ptr == 0) { break; }
+ else if (*ptr == ':') {
+ ptr++;
+ if (*ptr != ':') { return false; }
+ ptr++;
+ if (*ptr == 0) { return false; }
+ break;
+ }
+ else if (!isalnum(*ptr)) { return false; }
+ ptr++;
+ }
+ }
+
+ return true;
+}
+
+int
+CFCSymbol_validate_class_name_component(const char *name) {
+ if (!S_validate_class_name(name)) { return false; }
+ if (strchr(name, ':') != NULL) { return false; }
+ return true;
+}
+
+static int
+S_validate_class_cnick(const char *class_cnick) {
+ // Allow all caps.
+ const char *ptr;
+ for (ptr = class_cnick; ; ptr++) {
+ if (*ptr == 0) {
+ if (strlen(class_cnick)) { return true; }
+ else { break; }
+ }
+ else if (!isupper(*ptr)) { break; }
+ }
+
+ // Same as one component of a class name.
+ return CFCSymbol_validate_class_name_component(class_cnick);
+}
+
+static int
+S_validate_identifier(const char *identifier) {
+ const char *ptr = identifier;
+ if (!isalpha(*ptr) && *ptr != '_') { return false; }
+ for (; *ptr != 0; ptr++) {
+ if (!isalnum(*ptr) && *ptr != '_') { return false; }
+ }
+ return true;
+}
+
+CFCSymbol*
+CFCSymbol_init(CFCSymbol *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *class_cnick, const char *micro_sym) {
+ // Validate.
+ CFCUTIL_NULL_CHECK(parcel);
+ if (!S_validate_exposure(exposure)) {
+ croak("Invalid exposure: '%s'", exposure ? exposure : "[NULL]");
+ }
+ if (class_name && !S_validate_class_name(class_name)) {
+ croak("Invalid class_name: '%s'", class_name);
+ }
+ if (!micro_sym || !S_validate_identifier(micro_sym)) {
+ croak("Invalid micro_sym: '%s'", micro_sym ? micro_sym : "[NULL]");
+ }
+
+ // Derive class_cnick if necessary, then validate.
+ const char *real_cnick = NULL;
+ if (class_name) {
+ if (class_cnick) {
+ real_cnick = class_cnick;
+ }
+ else {
+ const char *last_colon = strrchr(class_name, ':');
+ real_cnick = last_colon ? last_colon + 1 : class_name;
+ }
+ }
+ else if (class_cnick) {
+ // Sanity check class_cnick without class_name.
+ croak("Can't supply class_cnick without class_name");
+ }
+ else {
+ real_cnick = NULL;
+ }
+ if (real_cnick && !S_validate_class_cnick(real_cnick)) {
+ croak("Invalid class_cnick: '%s'", real_cnick);
+ }
+
+ // Assign.
+ self->parcel = (CFCParcel*)CFCBase_incref((CFCBase*)parcel);
+ self->exposure = CFCUtil_strdup(exposure);
+ self->class_name = CFCUtil_strdup(class_name);
+ self->class_cnick = CFCUtil_strdup(real_cnick);
+ self->micro_sym = CFCUtil_strdup(micro_sym);
+
+ // Derive short_sym.
+ size_t class_cnick_len = self->class_cnick
+ ? strlen(self->class_cnick)
+ : 0;
+ size_t short_sym_len = class_cnick_len
+ + strlen("_")
+ + strlen(self->micro_sym);
+ self->short_sym = (char*)MALLOCATE(short_sym_len + 1);
+ if (self->class_cnick) {
+ memcpy((void*)self->short_sym, self->class_cnick, class_cnick_len);
+ }
+ self->short_sym[class_cnick_len] = '_';
+ memcpy(&self->short_sym[class_cnick_len + 1],
+ self->micro_sym, strlen(micro_sym));
+ self->short_sym[short_sym_len] = '\0';
+
+ // Derive full_sym;
+ const char *prefix = CFCParcel_get_prefix(self->parcel);
+ size_t prefix_len = strlen(prefix);
+ size_t full_sym_len = prefix_len + short_sym_len;
+ self->full_sym = (char*)MALLOCATE(full_sym_len + 1);
+ memcpy(self->full_sym, prefix, prefix_len);
+ memcpy(&self->full_sym[prefix_len], self->short_sym, short_sym_len);
+ self->full_sym[full_sym_len] = '\0';
+
+ return self;
+}
+
+void
+CFCSymbol_destroy(CFCSymbol *self) {
+ CFCBase_decref((CFCBase*)self->parcel);
+ FREEMEM(self->exposure);
+ FREEMEM(self->class_name);
+ FREEMEM(self->class_cnick);
+ FREEMEM(self->micro_sym);
+ FREEMEM(self->short_sym);
+ FREEMEM(self->full_sym);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+int
+CFCSymbol_equals(CFCSymbol *self, CFCSymbol *other) {
+ if (strcmp(self->micro_sym, other->micro_sym) != 0) { return false; }
+ if (!CFCParcel_equals(self->parcel, other->parcel)) { return false; }
+ if (strcmp(self->exposure, other->exposure) != 0) { return false; }
+ if (self->class_name) {
+ if (!other->class_name) { return false; }
+ if (strcmp(self->class_name, other->class_name) != 0) {
+ return false;
+ }
+ }
+ else if (other->class_name) {
+ return false;
+ }
+ return true;
+}
+
+int
+CFCSymbol_public(CFCSymbol *self) {
+ return !strcmp(self->exposure, "public");
+}
+
+int
+CFCSymbol_parcel(CFCSymbol *self) {
+ return !strcmp(self->exposure, "parcel");
+}
+
+int
+CFCSymbol_private(CFCSymbol *self) {
+ return !strcmp(self->exposure, "private");
+}
+
+int
+CFCSymbol_local(CFCSymbol *self) {
+ return !strcmp(self->exposure, "local");
+}
+
+const char*
+CFCSymbol_full_sym(CFCSymbol *self) {
+ return self->full_sym;
+}
+
+const char*
+CFCSymbol_short_sym(CFCSymbol *self) {
+ return self->short_sym;
+}
+
+struct CFCParcel*
+CFCSymbol_get_parcel(CFCSymbol *self) {
+ return self->parcel;
+}
+
+const char*
+CFCSymbol_get_class_name(CFCSymbol *self) {
+ return self->class_name;
+}
+
+const char*
+CFCSymbol_get_class_cnick(CFCSymbol *self) {
+ return self->class_cnick;
+}
+
+const char*
+CFCSymbol_get_exposure(CFCSymbol *self) {
+ return self->exposure;
+}
+
+const char*
+CFCSymbol_micro_sym(CFCSymbol *self) {
+ return self->micro_sym;
+}
+
+const char*
+CFCSymbol_get_prefix(CFCSymbol *self) {
+ return CFCParcel_get_prefix(self->parcel);
+}
+
+const char*
+CFCSymbol_get_Prefix(CFCSymbol *self) {
+ return CFCParcel_get_Prefix(self->parcel);
+}
+
+const char*
+CFCSymbol_get_PREFIX(CFCSymbol *self) {
+ return CFCParcel_get_PREFIX(self->parcel);
+}
+
diff --git a/clownfish/src/CFCSymbol.h b/clownfish/src/CFCSymbol.h
new file mode 100644
index 0000000..7569b4a
--- /dev/null
+++ b/clownfish/src/CFCSymbol.h
@@ -0,0 +1,113 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCSYMBOL
+#define H_CFCSYMBOL
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCSymbol CFCSymbol;
+struct CFCParcel;
+
+#ifdef CFC_NEED_SYMBOL_STRUCT_DEF
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+struct CFCSymbol {
+ CFCBase base;
+ struct CFCParcel *parcel;
+ char *exposure;
+ char *class_name;
+ char *class_cnick;
+ char *micro_sym;
+ char *short_sym;
+ char *full_sym;
+};
+#endif
+
+/** Return true if the supplied string is comprised solely of alphanumeric
+ * characters, begins with an uppercase letter, and contains at least one
+ * lower case letter.
+ */
+int
+CFCSymbol_validate_class_name_component(const char *name);
+
+CFCSymbol*
+CFCSymbol_new(struct CFCParcel *parcel, const char *exposure, const char *class_name,
+ const char *class_cnick, const char *micro_sym);
+
+CFCSymbol*
+CFCSymbol_init(CFCSymbol *self, struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym);
+
+void
+CFCSymbol_destroy(CFCSymbol *self);
+
+int
+CFCSymbol_equals(CFCSymbol *self, CFCSymbol *other);
+
+struct CFCParcel*
+CFCSymbol_get_parcel(CFCSymbol *self);
+
+// May be NULL.
+const char*
+CFCSymbol_get_class_name(CFCSymbol *self);
+
+// May be NULL.
+const char*
+CFCSymbol_get_class_cnick(CFCSymbol *self);
+
+const char*
+CFCSymbol_get_exposure(CFCSymbol *self);
+
+int
+CFCSymbol_public(CFCSymbol *self);
+
+int
+CFCSymbol_parcel(CFCSymbol *self);
+
+int
+CFCSymbol_private(CFCSymbol *self);
+
+int
+CFCSymbol_local(CFCSymbol *self);
+
+const char*
+CFCSymbol_micro_sym(CFCSymbol *self);
+
+const char*
+CFCSymbol_short_sym(CFCSymbol *self);
+
+const char*
+CFCSymbol_full_sym(CFCSymbol *self);
+
+const char*
+CFCSymbol_get_prefix(CFCSymbol *self);
+
+const char*
+CFCSymbol_get_Prefix(CFCSymbol *self);
+
+const char*
+CFCSymbol_get_PREFIX(CFCSymbol *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCSYMBOL */
+
diff --git a/clownfish/src/CFCType.c b/clownfish/src/CFCType.c
new file mode 100644
index 0000000..7a3be2c
--- /dev/null
+++ b/clownfish/src/CFCType.c
@@ -0,0 +1,474 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <ctype.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_BASE_STRUCT_DEF
+#include "CFCBase.h"
+#include "CFCType.h"
+#include "CFCParcel.h"
+#include "CFCSymbol.h"
+#include "CFCUtil.h"
+
+struct CFCType {
+ CFCBase base;
+ int flags;
+ char *specifier;
+ int indirection;
+ struct CFCParcel *parcel;
+ char *c_string;
+ size_t width;
+ char *array;
+ struct CFCType *child;
+};
+
+CFCType*
+CFCType_new(int flags, struct CFCParcel *parcel, const char *specifier,
+ int indirection, const char *c_string) {
+ CFCType *self = (CFCType*)CFCBase_allocate(sizeof(CFCType),
+ "Clownfish::Type");
+ return CFCType_init(self, flags, parcel, specifier, indirection,
+ c_string);
+}
+
+CFCType*
+CFCType_init(CFCType *self, int flags, struct CFCParcel *parcel,
+ const char *specifier, int indirection, const char *c_string) {
+ self->flags = flags;
+ self->parcel = (CFCParcel*)CFCBase_incref((CFCBase*)parcel);
+ self->specifier = CFCUtil_strdup(specifier);
+ self->indirection = indirection;
+ self->c_string = c_string ? CFCUtil_strdup(c_string) : CFCUtil_strdup("");
+ self->width = 0;
+ self->array = NULL;
+ self->child = NULL;
+ return self;
+}
+
+CFCType*
+CFCType_new_integer(int flags, const char *specifier) {
+ // Validate specifier, find width.
+ size_t width;
+ if (!strcmp(specifier, "int8_t") || !strcmp(specifier, "uint8_t")) {
+ width = 1;
+ }
+ else if (!strcmp(specifier, "int16_t") || !strcmp(specifier, "uint16_t")) {
+ width = 2;
+ }
+ else if (!strcmp(specifier, "int32_t") || !strcmp(specifier, "uint32_t")) {
+ width = 4;
+ }
+ else if (!strcmp(specifier, "int64_t") || !strcmp(specifier, "uint64_t")) {
+ width = 8;
+ }
+ else if (!strcmp(specifier, "char")
+ || !strcmp(specifier, "short")
+ || !strcmp(specifier, "int")
+ || !strcmp(specifier, "long")
+ || !strcmp(specifier, "size_t")
+ || !strcmp(specifier, "bool_t") // Charmonizer type.
+ ) {
+ width = 0;
+ }
+ else {
+ croak("Unknown integer specifier: '%s'", specifier);
+ }
+
+ // Add Charmonizer prefix if necessary.
+ char full_specifier[32];
+ if (strcmp(specifier, "bool_t") == 0) {
+ strcpy(full_specifier, "chy_bool_t");
+ }
+ else {
+ strcpy(full_specifier, specifier);
+ }
+
+ // Cache the C representation of this type.
+ char c_string[32];
+ if (flags & CFCTYPE_CONST) {
+ sprintf(c_string, "const %s", full_specifier);
+ }
+ else {
+ strcpy(c_string, full_specifier);
+ }
+
+ // Add flags.
+ flags |= CFCTYPE_PRIMITIVE;
+ flags |= CFCTYPE_INTEGER;
+
+ CFCType *self = CFCType_new(flags, NULL, full_specifier, 0, c_string);
+ self->width = width;
+ return self;
+}
+
+static const char *float_specifiers[] = {
+ "float",
+ "double",
+ NULL
+};
+
+CFCType*
+CFCType_new_float(int flags, const char *specifier) {
+ // Validate specifier.
+ size_t i;
+ for (i = 0; ; i++) {
+ if (!float_specifiers[i]) {
+ croak("Unknown float specifier: '%s'", specifier);
+ }
+ if (strcmp(float_specifiers[i], specifier) == 0) {
+ break;
+ }
+ }
+
+ // Cache the C representation of this type.
+ char c_string[32];
+ if (flags & CFCTYPE_CONST) {
+ sprintf(c_string, "const %s", specifier);
+ }
+ else {
+ strcpy(c_string, specifier);
+ }
+
+ flags |= CFCTYPE_PRIMITIVE;
+ flags |= CFCTYPE_FLOATING;
+
+ return CFCType_new(flags, NULL, specifier, 0, c_string);
+}
+
+CFCType*
+CFCType_new_object(int flags, CFCParcel *parcel, const char *specifier,
+ int indirection) {
+ // Validate params.
+ if (indirection != 1) {
+ croak("Parameter 'indirection' can only be 1");
+ }
+ if (!specifier || !strlen(specifier)) {
+ croak("Missing required param 'specifier'");
+ }
+ if ((flags & CFCTYPE_INCREMENTED) && (flags & CFCTYPE_DECREMENTED)) {
+ croak("Can't be both incremented and decremented");
+ }
+ if (!parcel) {
+ croak("Missing required param 'parcel'");
+ }
+
+ // Add flags.
+ flags |= CFCTYPE_OBJECT;
+ if (strstr(specifier, "CharBuf")) {
+ // Determine whether this type is a string type.
+ flags |= CFCTYPE_STRING_TYPE;
+ }
+
+ const char *prefix = CFCParcel_get_prefix(parcel);
+ const size_t MAX_SPECIFIER_LEN = 256;
+ char full_specifier[MAX_SPECIFIER_LEN + 1];
+ char small_specifier[MAX_SPECIFIER_LEN + 1];
+ if (strlen(prefix) + strlen(specifier) > MAX_SPECIFIER_LEN) {
+ croak("Specifier and/or parcel prefix too long");
+ }
+ if (strstr(specifier, prefix) != specifier) {
+ sprintf(full_specifier, "%s%s", prefix, specifier);
+ strcpy(small_specifier, specifier);
+ }
+ else {
+ strcpy(full_specifier, specifier);
+ strcpy(small_specifier, specifier + strlen(prefix));
+ }
+ if (!CFCSymbol_validate_class_name_component(small_specifier)) {
+ croak("Invalid specifier: '%s'", specifier);
+ }
+
+ // Cache C representation.
+ char c_string[MAX_SPECIFIER_LEN + 10];
+ if (flags & CFCTYPE_CONST) {
+ sprintf(c_string, "const %s*", full_specifier);
+ }
+ else {
+ sprintf(c_string, "%s*", full_specifier);
+ }
+
+ return CFCType_new(flags, parcel, full_specifier, 1, c_string);
+}
+
+CFCType*
+CFCType_new_composite(int flags, CFCType *child, int indirection,
+ const char *array) {
+ if (!child) {
+ croak("Missing required param 'child'");
+ }
+ flags |= CFCTYPE_COMPOSITE;
+
+ // Cache C representation.
+ // NOTE: Array postfixes are NOT included.
+ const size_t MAX_LEN = 256;
+ const char *child_c_string = CFCType_to_c(child);
+ size_t child_c_len = strlen(child_c_string);
+ size_t amount = child_c_len + indirection;
+ if (amount > MAX_LEN) {
+ croak("C representation too long");
+ }
+ char c_string[MAX_LEN + 1];
+ strcpy(c_string, child_c_string);
+ int i;
+ for (i = 0; i < indirection; i++) {
+ strncat(c_string, "*", 1);
+ }
+
+ CFCType *self = CFCType_new(flags, NULL, CFCType_get_specifier(child),
+ indirection, c_string);
+ self->child = (CFCType*)CFCBase_incref((CFCBase*)child);
+
+ // Record array spec.
+ const char *array_spec = array ? array : "";
+ size_t array_spec_size = strlen(array_spec) + 1;
+ self->array = (char*)MALLOCATE(array_spec_size);
+ strcpy(self->array, array_spec);
+
+ return self;
+}
+
+CFCType*
+CFCType_new_void(int is_const) {
+ int flags = CFCTYPE_VOID;
+ const char *c_string = is_const ? "const void" : "void";
+ if (is_const) { flags |= CFCTYPE_CONST; }
+ return CFCType_new(flags, NULL, "void", 0, c_string);
+}
+
+CFCType*
+CFCType_new_va_list(void) {
+ return CFCType_new(CFCTYPE_VA_LIST, NULL, "va_list", 0, "va_list");
+}
+
+
+CFCType*
+CFCType_new_arbitrary(CFCParcel *parcel, const char *specifier) {
+ const size_t MAX_SPECIFIER_LEN = 256;
+
+ // Add parcel prefix to what appear to be namespaced types.
+ char full_specifier[MAX_SPECIFIER_LEN + 1];
+ if (isupper(*specifier) && parcel != NULL) {
+ const char *prefix = CFCParcel_get_prefix(parcel);
+ size_t full_len = strlen(prefix) + strlen(specifier);
+ if (full_len > MAX_SPECIFIER_LEN) {
+ croak("Illegal specifier: '%s'", specifier);
+ }
+ sprintf(full_specifier, "%s%s", prefix, specifier);
+ }
+ else {
+ if (strlen(specifier) > MAX_SPECIFIER_LEN) {
+ croak("Illegal specifier: '%s'", specifier);
+ }
+ strcpy(full_specifier, specifier);
+ }
+
+ // Validate specifier.
+ size_t i, max;
+ for (i = 0, max = strlen(full_specifier); i < max; i++) {
+ if (!isalnum(full_specifier[i]) && full_specifier[i] != '_') {
+ croak("Illegal specifier: '%s'", full_specifier);
+ }
+ }
+
+ return CFCType_new(CFCTYPE_ARBITRARY, parcel, full_specifier, 0,
+ full_specifier);
+}
+
+void
+CFCType_destroy(CFCType *self) {
+ if (self->child) {
+ CFCBase_decref((CFCBase*)self->child);
+ }
+ CFCBase_decref((CFCBase*)self->parcel);
+ FREEMEM(self->specifier);
+ FREEMEM(self->c_string);
+ FREEMEM(self->array);
+ CFCBase_destroy((CFCBase*)self);
+}
+
+int
+CFCType_equals(CFCType *self, CFCType *other) {
+ if ((CFCType_const(self) ^ CFCType_const(other))
+ || (CFCType_nullable(self) ^ CFCType_nullable(other))
+ || (CFCType_is_void(self) ^ CFCType_is_void(other))
+ || (CFCType_is_object(self) ^ CFCType_is_object(other))
+ || (CFCType_is_primitive(self) ^ CFCType_is_primitive(other))
+ || (CFCType_is_integer(self) ^ CFCType_is_integer(other))
+ || (CFCType_is_floating(self) ^ CFCType_is_floating(other))
+ || (CFCType_is_va_list(self) ^ CFCType_is_va_list(other))
+ || (CFCType_is_arbitrary(self) ^ CFCType_is_arbitrary(other))
+ || (CFCType_is_composite(self) ^ CFCType_is_composite(other))
+ || (CFCType_incremented(self) ^ CFCType_incremented(other))
+ || (CFCType_decremented(self) ^ CFCType_decremented(other))
+ || !!self->child ^ !!other->child
+ || !!self->array ^ !!other->array
+ ) {
+ return false;
+ }
+ if (self->indirection != other->indirection) { return false; }
+ if (strcmp(self->specifier, other->specifier) != 0) { return false; }
+ if (self->child) {
+ if (!CFCType_equals(self->child, other->child)) { return false; }
+ }
+ if (self->array) {
+ if (strcmp(self->array, other->array) != 0) { return false; }
+ }
+ return true;
+}
+
+int
+CFCType_similar(CFCType *self, CFCType *other) {
+ if (!CFCType_is_object(self)) {
+ croak("Attempt to call 'similar' on a non-object type");
+ }
+ if ((CFCType_const(self) ^ CFCType_const(other))
+ || (CFCType_nullable(self) ^ CFCType_nullable(other))
+ || (CFCType_incremented(self) ^ CFCType_incremented(other))
+ || (CFCType_decremented(self) ^ CFCType_decremented(other))
+ || (CFCType_is_object(self) ^ CFCType_is_object(other))
+ ) {
+ return false;
+ }
+ return true;
+}
+
+void
+CFCType_set_specifier(CFCType *self, const char *specifier) {
+ FREEMEM(self->specifier);
+ self->specifier = CFCUtil_strdup(specifier);
+}
+
+const char*
+CFCType_get_specifier(CFCType *self) {
+ return self->specifier;
+}
+
+int
+CFCType_get_indirection(CFCType *self) {
+ return self->indirection;
+}
+
+struct CFCParcel*
+CFCType_get_parcel(CFCType *self) {
+ return self->parcel;
+}
+
+void
+CFCType_set_c_string(CFCType *self, const char *c_string) {
+ FREEMEM(self->c_string);
+ self->c_string = CFCUtil_strdup(c_string);
+}
+
+const char*
+CFCType_to_c(CFCType *self) {
+ return self->c_string;
+}
+
+size_t
+CFCType_get_width(CFCType *self) {
+ return self->width;
+}
+
+const char*
+CFCType_get_array(CFCType *self) {
+ return self->array;
+}
+
+int
+CFCType_const(CFCType *self) {
+ return !!(self->flags & CFCTYPE_CONST);
+}
+
+void
+CFCType_set_nullable(CFCType *self, int nullable) {
+ if (nullable) {
+ self->flags |= CFCTYPE_NULLABLE;
+ }
+ else {
+ self->flags &= ~CFCTYPE_NULLABLE;
+ }
+}
+
+int
+CFCType_nullable(CFCType *self) {
+ return !!(self->flags & CFCTYPE_NULLABLE);
+}
+
+int
+CFCType_incremented(CFCType *self) {
+ return !!(self->flags & CFCTYPE_INCREMENTED);
+}
+
+int
+CFCType_decremented(CFCType *self) {
+ return !!(self->flags & CFCTYPE_DECREMENTED);
+}
+
+int
+CFCType_is_void(CFCType *self) {
+ return !!(self->flags & CFCTYPE_VOID);
+}
+
+int
+CFCType_is_object(CFCType *self) {
+ return !!(self->flags & CFCTYPE_OBJECT);
+}
+
+int
+CFCType_is_primitive(CFCType *self) {
+ return !!(self->flags & CFCTYPE_PRIMITIVE);
+}
+
+int
+CFCType_is_integer(CFCType *self) {
+ return !!(self->flags & CFCTYPE_INTEGER);
+}
+
+int
+CFCType_is_floating(CFCType *self) {
+ return !!(self->flags & CFCTYPE_FLOATING);
+}
+
+int
+CFCType_is_string_type(CFCType *self) {
+ return !!(self->flags & CFCTYPE_STRING_TYPE);
+}
+
+int
+CFCType_is_va_list(CFCType *self) {
+ return !!(self->flags & CFCTYPE_VA_LIST);
+}
+
+int
+CFCType_is_arbitrary(CFCType *self) {
+ return !!(self->flags & CFCTYPE_ARBITRARY);
+}
+
+int
+CFCType_is_composite(CFCType *self) {
+ return !!(self->flags & CFCTYPE_COMPOSITE);
+}
+
diff --git a/clownfish/src/CFCType.h b/clownfish/src/CFCType.h
new file mode 100644
index 0000000..03b93a6
--- /dev/null
+++ b/clownfish/src/CFCType.h
@@ -0,0 +1,152 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCTYPE
+#define H_CFCTYPE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCType CFCType;
+struct CFCParcel;
+
+#define CFCTYPE_CONST 0x00000001
+#define CFCTYPE_NULLABLE 0x00000002
+#define CFCTYPE_VOID 0x00000004
+#define CFCTYPE_INCREMENTED 0x00000008
+#define CFCTYPE_DECREMENTED 0x00000010
+#define CFCTYPE_OBJECT 0x00000020
+#define CFCTYPE_PRIMITIVE 0x00000040
+#define CFCTYPE_INTEGER 0x00000080
+#define CFCTYPE_FLOATING 0x00000100
+#define CFCTYPE_STRING_TYPE 0x00000200
+#define CFCTYPE_VA_LIST 0x00000400
+#define CFCTYPE_ARBITRARY 0x00000800
+#define CFCTYPE_COMPOSITE 0x00001000
+
+CFCType*
+CFCType_new(int flags, struct CFCParcel *parcel, const char *specifier,
+ int indirection, const char *c_string);
+
+CFCType*
+CFCType_init(CFCType *self, int flags, struct CFCParcel *parcel,
+ const char *specifier, int indirection, const char *c_string);
+
+CFCType*
+CFCType_new_integer(int flags, const char *specifier);
+
+CFCType*
+CFCType_new_float(int flags, const char *specifier);
+
+CFCType*
+CFCType_new_object(int flags, struct CFCParcel *parcel, const char *specifier,
+ int indirection);
+
+CFCType*
+CFCType_new_composite(int flags, CFCType *child, int indirection,
+ const char *array);
+
+CFCType*
+CFCType_new_void(int is_const);
+
+CFCType*
+CFCType_new_va_list(void);
+
+CFCType*
+CFCType_new_arbitrary(struct CFCParcel *parcel, const char *specifier);
+
+void
+CFCType_destroy(CFCType *self);
+
+int
+CFCType_equals(CFCType *self, CFCType *other);
+
+int
+CFCType_similar(CFCType *self, CFCType *other);
+
+void
+CFCType_set_specifier(CFCType *self, const char *specifier);
+
+const char*
+CFCType_get_specifier(CFCType *self);
+
+int
+CFCType_get_indirection(CFCType *self);
+
+struct CFCParcel*
+CFCType_get_parcel(CFCType *self);
+
+void
+CFCType_set_c_string(CFCType *self, const char *c_string);
+
+const char*
+CFCType_to_c(CFCType *self);
+
+size_t
+CFCType_get_width(CFCType *self);
+
+const char*
+CFCType_get_array(CFCType *self);
+
+int
+CFCType_const(CFCType *self);
+
+void
+CFCType_set_nullable(CFCType *self, int nullable);
+
+int
+CFCType_nullable(CFCType *self);
+
+int
+CFCType_incremented(CFCType *self);
+
+int
+CFCType_decremented(CFCType *self);
+
+int
+CFCType_is_void(CFCType *self);
+
+int
+CFCType_is_object(CFCType *self);
+
+int
+CFCType_is_primitive(CFCType *self);
+
+int
+CFCType_is_integer(CFCType *self);
+
+int
+CFCType_is_floating(CFCType *self);
+
+int
+CFCType_is_string_type(CFCType *self);
+
+int
+CFCType_is_va_list(CFCType *self);
+
+int
+CFCType_is_arbitrary(CFCType *self);
+
+int
+CFCType_is_composite(CFCType *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCTYPE */
+
diff --git a/clownfish/src/CFCUtil.c b/clownfish/src/CFCUtil.c
new file mode 100644
index 0000000..11555f2
--- /dev/null
+++ b/clownfish/src/CFCUtil.c
@@ -0,0 +1,288 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <errno.h>
+#include <sys/stat.h>
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#include "CFCUtil.h"
+
+void
+CFCUtil_null_check(const void *arg, const char *name, const char *file,
+ int line) {
+ if (!arg) {
+ CFCUtil_die("%s cannot be NULL at %s line %d", name, file, line);
+ }
+}
+
+char*
+CFCUtil_strdup(const char *string) {
+ if (!string) { return NULL; }
+ return CFCUtil_strndup(string, strlen(string));
+}
+
+char*
+CFCUtil_strndup(const char *string, size_t len) {
+ if (!string) { return NULL; }
+ char *copy = (char*)MALLOCATE(len + 1);
+ memcpy(copy, string, len);
+ copy[len] = '\0';
+ return copy;
+}
+
+void
+CFCUtil_trim_whitespace(char *text) {
+ if (!text) {
+ return;
+ }
+
+ // Find start.
+ char *ptr = text;
+ while (*ptr != '\0' && isspace(*ptr)) { ptr++; }
+
+ // Find end.
+ size_t orig_len = strlen(text);
+ char *limit = text + orig_len;
+ for (; limit > text; limit--) {
+ if (!isspace(*(limit - 1))) { break; }
+ }
+
+ // Modify string in place and NULL-terminate.
+ while (ptr < limit) {
+ *text++ = *ptr++;
+ }
+ *text = '\0';
+}
+
+void*
+CFCUtil_wrapped_malloc(size_t count, const char *file, int line) {
+ void *pointer = malloc(count);
+ if (pointer == NULL && count != 0) {
+ if (sizeof(long) >= sizeof(size_t)) {
+ fprintf(stderr, "Can't malloc %lu bytes at %s line %d\n",
+ (unsigned long)count, file, line);
+ }
+ else {
+ fprintf(stderr, "malloc failed at %s line %d\n", file, line);
+ }
+ exit(1);
+ }
+ return pointer;
+}
+
+void*
+CFCUtil_wrapped_calloc(size_t count, size_t size, const char *file, int line) {
+ void *pointer = calloc(count, size);
+ if (pointer == NULL && count != 0) {
+ if (sizeof(long) >= sizeof(size_t)) {
+ fprintf(stderr,
+ "Can't calloc %lu elements of size %lu at %s line %d\n",
+ (unsigned long)count, (unsigned long)size, file, line);
+ }
+ else {
+ fprintf(stderr, "calloc failed at %s line %d\n", file, line);
+ }
+ exit(1);
+ }
+ return pointer;
+}
+
+void*
+CFCUtil_wrapped_realloc(void *ptr, size_t size, const char *file, int line) {
+ void *pointer = realloc(ptr, size);
+ if (pointer == NULL && size != 0) {
+ if (sizeof(long) >= sizeof(size_t)) {
+ fprintf(stderr, "Can't realloc %lu bytes at %s line %d\n",
+ (unsigned long)size, file, line);
+ }
+ else {
+ fprintf(stderr, "realloc failed at %s line %d\n", file, line);
+ }
+ exit(1);
+ }
+ return pointer;
+}
+
+void
+CFCUtil_wrapped_free(void *ptr) {
+ free(ptr);
+}
+
+int
+CFCUtil_current(const char *orig, const char *dest) {
+ // If the destination file doesn't exist, we're not current.
+ struct stat dest_stat;
+ if (stat(dest, &dest_stat) == -1) {
+ return false;
+ }
+
+ // If the source file is newer than the dest, we're not current.
+ struct stat orig_stat;
+ if (stat(orig, &orig_stat) == -1) {
+ CFCUtil_die("Missing source file '%s'", orig);
+ }
+ if (orig_stat.st_mtime > dest_stat.st_mtime) {
+ return false;
+ }
+
+ // Current!
+ return 1;
+}
+
+void
+CFCUtil_write_file(const char *filename, const char *content, size_t len) {
+ FILE *fh = fopen(filename, "w+");
+ if (fh == NULL) {
+ CFCUtil_die("Couldn't open '%s': %s", filename, strerror(errno));
+ }
+ fwrite(content, sizeof(char), len, fh);
+ if (fclose(fh)) {
+ CFCUtil_die("Error when closing '%s': %s", filename, strerror(errno));
+ }
+}
+
+char*
+CFCUtil_slurp_text(const char *file_path, size_t *len_ptr) {
+ FILE *const file = fopen(file_path, "r");
+ char *contents;
+ size_t binary_len;
+ long text_len;
+
+ /* Sanity check. */
+ if (file == NULL) {
+ CFCUtil_die("Error opening file '%s': %s", file_path, strerror(errno));
+ }
+
+ /* Find length; return NULL if the file has a zero-length. */
+ binary_len = CFCUtil_flength(file);
+ if (binary_len == 0) {
+ *len_ptr = 0;
+ return NULL;
+ }
+
+ /* Allocate memory and read the file. */
+ contents = (char*)MALLOCATE(binary_len * sizeof(char) + 1);
+ text_len = fread(contents, sizeof(char), binary_len, file);
+
+ /* Weak error check, because CRLF might result in fewer chars read. */
+ if (text_len <= 0) {
+ CFCUtil_die("Tried to read %ld bytes of '%s', got return code %ld",
+ (long)binary_len, file_path, (long)text_len);
+ }
+
+ /* NULL-terminate. */
+ contents[text_len] = '\0';
+
+ /* Set length pointer for benefit of caller. */
+ *len_ptr = text_len;
+
+ /* Clean up. */
+ if (fclose(file)) {
+ CFCUtil_die("Error closing file '%s': %s", file_path, strerror(errno));
+ }
+
+ return contents;
+}
+
+void
+CFCUtil_write_if_changed(const char *path, const char *content, size_t len) {
+ FILE *f = fopen(path, "r");
+ if (f) { // Does file exist?
+ if (fclose(f)) {
+ CFCUtil_die("Error closing file '%s': %s", path, strerror(errno));
+ }
+ size_t existing_len;
+ char *existing = CFCUtil_slurp_text(path, &existing_len);
+ int changed = true;
+ if (existing_len == len && strcmp(content, existing) == 0) {
+ changed = false;
+ }
+ FREEMEM(existing);
+ if (changed == false) {
+ return;
+ }
+ }
+ CFCUtil_write_file(path, content, len);
+}
+
+long
+CFCUtil_flength(void *file) {
+ FILE *f = (FILE*)file;
+ const long bookmark = (long)ftell(f);
+ long check_val;
+ long len;
+
+ /* Seek to end of file and check length. */
+ check_val = fseek(f, 0, SEEK_END);
+ if (check_val == -1) { CFCUtil_die("fseek error : %s\n", strerror(errno)); }
+ len = (long)ftell(f);
+ if (len == -1) { CFCUtil_die("ftell error : %s\n", strerror(errno)); }
+
+ /* Return to where we were. */
+ check_val = fseek(f, bookmark, SEEK_SET);
+ if (check_val == -1) { CFCUtil_die("fseek error : %s\n", strerror(errno)); }
+
+ return len;
+}
+
+void
+CFCUtil_die(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+ fprintf(stderr, "\n");
+ exit(1);
+}
+
+void
+CFCUtil_warn(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ vfprintf(stderr, format, args);
+ va_end(args);
+ fprintf(stderr, "\n");
+}
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+void*
+CFCUtil_make_perl_obj(void *ptr, const char *klass) {
+ SV *inner_obj = newSV(0);
+ SvOBJECT_on(inner_obj);
+ PL_sv_objcount++;
+ SvUPGRADE(inner_obj, SVt_PVMG);
+ sv_setiv(inner_obj, PTR2IV(ptr));
+
+ // Connect class association.
+ HV *stash = gv_stashpvn((char*)klass, strlen(klass), TRUE);
+ SvSTASH_set(inner_obj, (HV*)SvREFCNT_inc(stash));
+
+ return inner_obj;
+}
+
diff --git a/clownfish/src/CFCUtil.h b/clownfish/src/CFCUtil.h
new file mode 100644
index 0000000..2495b44
--- /dev/null
+++ b/clownfish/src/CFCUtil.h
@@ -0,0 +1,123 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCUTIL
+#define H_CFCUTIL
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Create an inner Perl object with a refcount of 1. For use in actual
+ * Perl-space, it is necessary to wrap this inner object in an RV.
+ */
+void*
+CFCUtil_make_perl_obj(void *ptr, const char *klass);
+
+/** Throw an error if the supplied argument is NULL.
+ */
+void
+CFCUtil_null_check(const void *arg, const char *name, const char *file, int line);
+#define CFCUTIL_NULL_CHECK(arg) \
+ CFCUtil_null_check(arg, #arg, __FILE__, __LINE__)
+
+/** Portable, NULL-safe implementation of strdup().
+ */
+char*
+CFCUtil_strdup(const char *string);
+
+/** Portable, NULL-safe implementation of strndup().
+ */
+char*
+CFCUtil_strndup(const char *string, size_t len);
+
+/** Trim whitespace from the beginning and the end of a string.
+ */
+void
+CFCUtil_trim_whitespace(char *text);
+
+/** Attempt to allocate memory with malloc, but print an error and exit if the
+ * call fails.
+ */
+void*
+CFCUtil_wrapped_malloc(size_t count, const char *file, int line);
+
+/** Attempt to allocate memory with calloc, but print an error and exit if the
+ * call fails.
+ */
+void*
+CFCUtil_wrapped_calloc(size_t count, size_t size, const char *file, int line);
+
+/** Attempt to allocate memory with realloc, but print an error and exit if
+ * the call fails.
+ */
+void*
+CFCUtil_wrapped_realloc(void *ptr, size_t size, const char *file, int line);
+
+/** Free memory. (Wrapping is necessary in cases where memory allocated
+ * within Clownfish has to be freed in an external environment where "free"
+ * may have been redefined.)
+ */
+void
+CFCUtil_wrapped_free(void *ptr);
+
+#define MALLOCATE(_count) \
+ CFCUtil_wrapped_malloc((_count), __FILE__, __LINE__)
+#define CALLOCATE(_count, _size) \
+ CFCUtil_wrapped_calloc((_count), (_size), __FILE__, __LINE__)
+#define REALLOCATE(_ptr, _count) \
+ CFCUtil_wrapped_realloc((_ptr), (_count), __FILE__, __LINE__)
+#define FREEMEM(_ptr) \
+ CFCUtil_wrapped_free(_ptr)
+
+int
+CFCUtil_current(const char *orig, const char *dest);
+
+/* Open a file (truncating if necessary) and write [content] to it. CFCUtil_die() if
+ * an error occurs.
+ */
+void
+CFCUtil_write_file(const char *filename, const char *content, size_t len);
+
+void
+CFCUtil_write_if_changed(const char *path, const char *content, size_t len);
+
+/* Read an entire file (as text) into memory.
+ */
+char*
+CFCUtil_slurp_text(const char *file_path, size_t *len_ptr);
+
+/* Get the length of a file (may overshoot on text files under DOS).
+ */
+long
+CFCUtil_flength(void *file);
+
+/* Print an error message to stderr and exit.
+ */
+void
+CFCUtil_die(const char *format, ...);
+
+/* Print an error message to stderr.
+ */
+void
+CFCUtil_warn(const char *format, ...);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCUTIL */
+
diff --git a/clownfish/src/CFCVariable.c b/clownfish/src/CFCVariable.c
new file mode 100644
index 0000000..0c9ca61
--- /dev/null
+++ b/clownfish/src/CFCVariable.c
@@ -0,0 +1,133 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#ifndef true
+ #define true 1
+ #define false 0
+#endif
+
+#define CFC_NEED_SYMBOL_STRUCT_DEF
+#include "CFCSymbol.h"
+#include "CFCVariable.h"
+#include "CFCParcel.h"
+#include "CFCType.h"
+#include "CFCUtil.h"
+
+struct CFCVariable {
+ struct CFCSymbol symbol;
+ CFCType *type;
+ char *local_c;
+ char *global_c;
+ char *local_dec;
+};
+
+CFCVariable*
+CFCVariable_new(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym, struct CFCType *type) {
+ CFCVariable *self = (CFCVariable*)CFCBase_allocate(sizeof(CFCVariable),
+ "Clownfish::Variable");
+ return CFCVariable_init(self, parcel, exposure, class_name, class_cnick,
+ micro_sym, type);
+}
+
+CFCVariable*
+CFCVariable_init(CFCVariable *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *class_cnick, const char *micro_sym,
+ struct CFCType *type) {
+ // Validate type.
+ CFCUTIL_NULL_CHECK(type);
+
+ // Default exposure to "local".
+ const char *real_exposure = exposure ? exposure : "local";
+
+ CFCSymbol_init((CFCSymbol*)self, parcel, real_exposure, class_name,
+ class_cnick, micro_sym);
+
+ // Assign type.
+ self->type = (CFCType*)CFCBase_incref((CFCBase*)type);
+
+ // Cache various C string representations.
+ const char *type_str = CFCType_to_c(type);
+ const char *postfix = "";
+ if (CFCType_is_composite(type) && CFCType_get_array(type) != NULL) {
+ postfix = CFCType_get_array(type);
+ }
+ {
+ size_t size = strlen(type_str) + sizeof(" ") + strlen(micro_sym) +
+ strlen(postfix) + 1;
+ self->local_c = (char*)MALLOCATE(size);
+ sprintf(self->local_c, "%s %s%s", type_str, micro_sym, postfix);
+ }
+ {
+ self->local_dec = (char*)MALLOCATE(strlen(self->local_c) + sizeof(";\0"));
+ sprintf(self->local_dec, "%s;", self->local_c);
+ }
+ {
+ const char *full_sym = CFCSymbol_full_sym((CFCSymbol*)self);
+ size_t size = strlen(type_str) + sizeof(" ") + strlen(full_sym) +
+ strlen(postfix) + 1;
+ self->global_c = (char*)MALLOCATE(size);
+ sprintf(self->global_c, "%s %s%s", type_str, full_sym, postfix);
+ }
+
+ return self;
+}
+
+void
+CFCVariable_destroy(CFCVariable *self) {
+ CFCBase_decref((CFCBase*)self->type);
+ FREEMEM(self->local_c);
+ FREEMEM(self->global_c);
+ FREEMEM(self->local_dec);
+ CFCSymbol_destroy((CFCSymbol*)self);
+}
+
+int
+CFCVariable_equals(CFCVariable *self, CFCVariable *other) {
+ if (!CFCType_equals(self->type, other->type)) { return false; }
+ return CFCSymbol_equals((CFCSymbol*)self, (CFCSymbol*)other);
+}
+
+CFCType*
+CFCVariable_get_type(CFCVariable *self) {
+ return self->type;
+}
+
+const char*
+CFCVariable_local_c(CFCVariable *self) {
+ return self->local_c;
+}
+
+const char*
+CFCVariable_global_c(CFCVariable *self) {
+ return self->global_c;
+}
+
+const char*
+CFCVariable_local_declaration(CFCVariable *self) {
+ return self->local_dec;
+}
+
diff --git a/clownfish/src/CFCVariable.h b/clownfish/src/CFCVariable.h
new file mode 100644
index 0000000..37b953f
--- /dev/null
+++ b/clownfish/src/CFCVariable.h
@@ -0,0 +1,62 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFCVARIABLE
+#define H_CFCVARIABLE
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct CFCVariable CFCVariable;
+struct CFCParcel;
+struct CFCType;
+
+CFCVariable*
+CFCVariable_new(struct CFCParcel *parcel, const char *exposure,
+ const char *class_name, const char *class_cnick,
+ const char *micro_sym, struct CFCType *type);
+
+CFCVariable*
+CFCVariable_init(CFCVariable *self, struct CFCParcel *parcel,
+ const char *exposure, const char *class_name,
+ const char *class_cnick, const char *micro_sym,
+ struct CFCType *type);
+
+void
+CFCVariable_destroy(CFCVariable *self);
+
+int
+CFCVariable_equals(CFCVariable *self, CFCVariable *other);
+
+struct CFCType*
+CFCVariable_get_type(CFCVariable *self);
+
+const char*
+CFCVariable_local_c(CFCVariable *self);
+
+const char*
+CFCVariable_global_c(CFCVariable *self);
+
+const char*
+CFCVariable_local_declaration(CFCVariable *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H_CFCVARIABLE */
+
diff --git a/clownfish/t/000-load.t b/clownfish/t/000-load.t
new file mode 100644
index 0000000..4f450a2
--- /dev/null
+++ b/clownfish/t/000-load.t
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More 'no_plan';
+use File::Find 'find';
+
+my @modules;
+
+find(
+ { no_chdir => 1,
+ wanted => sub {
+ return unless $File::Find::name =~ /\.pm$/;
+ push @modules, $File::Find::name;
+ }
+ },
+ 'lib'
+);
+for (@modules) {
+ s/^.*?Clownfish/Clownfish/;
+ s/\.pm$//;
+ s/\W+/::/g;
+ use_ok($_);
+}
+
diff --git a/clownfish/t/001-util.t b/clownfish/t/001-util.t
new file mode 100644
index 0000000..b2f818c
--- /dev/null
+++ b/clownfish/t/001-util.t
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 15;
+use File::stat qw( stat );
+use Clownfish::Util qw(
+ slurp_text
+ current
+ verify_args
+ a_isa_b
+ write_if_changed
+);
+
+my $foo_txt = 'foo.txt';
+unlink $foo_txt;
+open( my $fh, '>', $foo_txt ) or die "Can't open '$foo_txt': $!";
+print $fh "foo";
+close $fh or die "Can't close '$foo_txt': $!";
+is( slurp_text($foo_txt), "foo", "slurp_text" );
+
+ok( current( $foo_txt, $foo_txt ), "current" );
+ok( !current( $foo_txt, 't' ), "not current" );
+ok( !current( $foo_txt, "nonexistent_file" ),
+ "not current when dest file mising"
+);
+
+my $one_second_ago = time() - 1;
+utime( $one_second_ago, $one_second_ago, $foo_txt )
+ or die "utime failed";
+write_if_changed( $foo_txt, "foo" );
+is( stat($foo_txt)->mtime, $one_second_ago,
+ "write_if_changed does nothing if contents not changed" );
+write_if_changed( $foo_txt, "foofoo" );
+ok( stat($foo_txt)->mtime != $one_second_ago,
+ "write_if_changed writes if contents changed"
+);
+
+unlink $foo_txt;
+
+my %defaults = ( foo => undef );
+sub test_verify_args { return verify_args( \%defaults, @_ ) }
+
+ok( test_verify_args( foo => 'foofoo' ), "args verified" );
+ok( !test_verify_args('foo'), "odd args fail verification" );
+like( $@, qr/odd/, 'set $@ on odd arg failure' );
+ok( !test_verify_args( bar => 'nope' ), "bad param doesn't verify" );
+like( $@, qr/param/, 'set $@ on invalid param failure' );
+
+my $foo = bless {}, 'Foo';
+ok( a_isa_b( $foo, 'Foo' ), "a_isa_b true" );
+ok( !a_isa_b( $foo, 'Bar' ), "a_isa_b false" );
+ok( !a_isa_b( 'Foo', 'Foo' ), "a_isa_b not blessed" );
+ok( !a_isa_b( undef, 'Foo' ), "a_isa_b undef" );
+
diff --git a/clownfish/t/050-docucomment.t b/clownfish/t/050-docucomment.t
new file mode 100644
index 0000000..3175b7f
--- /dev/null
+++ b/clownfish/t/050-docucomment.t
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 10;
+
+BEGIN { use_ok('Clownfish::DocuComment') }
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+isa_ok( $parser->docucomment('/** foo. */'), "Clownfish::DocuComment" );
+
+my $text = <<'END_COMMENT';
+/**
+ * Brief description. Long description.
+ *
+ * More long description.
+ *
+ * @param foo A foo.
+ * @param bar A bar.
+ *
+ * @param baz A baz.
+ * @return a return value.
+ */
+END_COMMENT
+
+my $docucomment = Clownfish::DocuComment->parse($text);
+
+like(
+ $docucomment->get_description,
+ qr/^Brief.*long description.\s*\Z/ims,
+ "get_description"
+);
+is( $docucomment->get_brief, "Brief description.", "brief" );
+like( $docucomment->get_long, qr/^Long.*long description.\s*\Z/ims, "long" );
+is_deeply( $docucomment->get_param_names, [qw( foo bar baz )],
+ "param names" );
+is( $docucomment->get_param_docs->[0], "A foo.", '@param terminated by @' );
+is( $docucomment->get_param_docs->[1],
+ "A bar.", '@param terminated by empty line' );
+is( $docucomment->get_param_docs->[2],
+ "A baz.", '@param terminated next element, @return' );
+is( $docucomment->get_retval, "a return value.", "get_retval" );
+
diff --git a/clownfish/t/051-symbol.t b/clownfish/t/051-symbol.t
new file mode 100644
index 0000000..65a5005
--- /dev/null
+++ b/clownfish/t/051-symbol.t
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package ClownfishyThing;
+use base qw( Clownfish::Symbol );
+
+sub new {
+ return shift->SUPER::new( micro_sym => 'sym', exposure => 'parcel', @_ );
+}
+
+package main;
+use Test::More tests => 44;
+
+for (qw( foo FOO 1Foo Foo_Bar FOOBAR 1FOOBAR )) {
+ eval { my $thing = ClownfishyThing->new( class_name => $_ ) };
+ like( $@, qr/class_name/, "Reject invalid class name $_" );
+ my $bogus_middle = "Foo::" . $_ . "::Bar";
+ eval { my $thing = ClownfishyThing->new( class_name => $bogus_middle ) };
+ like( $@, qr/class_name/, "Reject invalid class name $bogus_middle" );
+}
+
+my @exposures = qw( public private parcel local );
+for my $exposure (@exposures) {
+ my $thing = ClownfishyThing->new( exposure => $exposure );
+ ok( $thing->$exposure, "exposure $exposure" );
+ my @not_exposures = grep { $_ ne $exposure } @exposures;
+ ok( !$thing->$_, "$exposure means not $_" ) for @not_exposures;
+}
+
+my $foo = ClownfishyThing->new( class_name => 'Foo' );
+my $foo_jr = ClownfishyThing->new( class_name => 'Foo::FooJr' );
+ok( !$foo->equals($foo_jr), "different class_name spoils equals" );
+is( $foo_jr->get_class_name, "Foo::FooJr", "get_class_name" );
+is( $foo_jr->get_class_cnick, "FooJr", "derive class_cnick from class_name" );
+
+my $public_exposure = ClownfishyThing->new( exposure => 'public' );
+my $parcel_exposure = ClownfishyThing->new( exposure => 'parcel' );
+ok( !$public_exposure->equals($parcel_exposure),
+ "different exposure spoils equals"
+);
+
+my $lucifer_parcel = Clownfish::Parcel->singleton( name => 'Lucifer' );
+my $lucifer = ClownfishyThing->new( parcel => 'Lucifer' );
+ok( $lucifer_parcel == $lucifer->get_parcel, "derive parcel" );
+is( $lucifer->get_prefix, "lucifer_", "get_prefix" );
+is( $lucifer->get_Prefix, "Lucifer_", "get_Prefix" );
+is( $lucifer->get_PREFIX, "LUCIFER_", "get_PREFIX" );
+my $luser = ClownfishyThing->new( parcel => 'Luser' );
+ok( !$lucifer->equals($luser), "different parcel spoils equals" );
+
+for ( qw( 1foo * 0 ), "\x{263a}" ) {
+ eval { my $thing = ClownfishyThing->new( micro_sym => $_ ); };
+ like( $@, qr/micro_sym/, "reject bad micro_sym" );
+}
+
+my $ooga = ClownfishyThing->new( micro_sym => 'ooga' );
+my $booga = ClownfishyThing->new( micro_sym => 'booga' );
+ok( !$ooga->equals($booga), "Different micro_sym spoils equals()" );
+
+my $eep = ClownfishyThing->new(
+ parcel => 'Eep',
+ class_name => "Op::Ork",
+ micro_sym => 'ah_ah',
+);
+is( $eep->short_sym, "Ork_ah_ah", "short_sym" );
+is( $eep->full_sym, "eep_Ork_ah_ah", "full_sym" );
+
diff --git a/clownfish/t/100-type.t b/clownfish/t/100-type.t
new file mode 100644
index 0000000..c9df6e0
--- /dev/null
+++ b/clownfish/t/100-type.t
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MyType;
+use base qw( Clownfish::Type );
+
+package main;
+use Test::More tests => 11;
+use Clownfish::Parcel;
+
+my $neato_parcel = Clownfish::Parcel->singleton( name => 'Neato' );
+
+my $type = MyType->new( parcel => 'Neato', specifier => 'mytype_t' );
+is( $type->get_parcel, $neato_parcel,
+ "constructor changes parcel name to Parcel singleton" );
+
+is( $type->to_c, '', "to_c()" );
+$type->set_c_string("mytype_t");
+is( $type->to_c, "mytype_t", "set_c_string()" );
+ok( !$type->const, "const() is off by default" );
+is( $type->get_specifier, "mytype_t", "get_specifier()" );
+
+ok( !$type->is_object, "is_object() false by default" );
+ok( !$type->is_integer, "is_integer() false by default" );
+ok( !$type->is_floating, "is_floating() false by default" );
+ok( !$type->is_void, "is_void() false by default" );
+ok( !$type->is_composite, "is_composite() false by default" );
+ok( !$type->is_string_type, "is_string_type() false by default" );
+
diff --git a/clownfish/t/101-primitive_type.t b/clownfish/t/101-primitive_type.t
new file mode 100644
index 0000000..90764da
--- /dev/null
+++ b/clownfish/t/101-primitive_type.t
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MyPrimitiveType;
+use base qw( Clownfish::Type );
+
+sub new {
+ my $either = shift;
+ return $either->SUPER::new( @_, primitive => 1, );
+}
+
+package main;
+use Test::More tests => 4;
+
+my $type = MyPrimitiveType->new( specifier => 'hump_t' );
+ok( $type->is_primitive, "is_primitive" );
+
+my $other = MyPrimitiveType->new( specifier => 'hump_t' );
+ok( $type->equals($other), "equals()" );
+
+$other = MyPrimitiveType->new( specifier => 'dump_t' );
+ok( !$type->equals($other), "equals() spoiled by specifier" );
+
+$other = MyPrimitiveType->new( specifier => 'hump_t', const => 1 );
+ok( !$type->equals($other), "equals() spoiled by const" );
+
diff --git a/clownfish/t/102-integer_type.t b/clownfish/t/102-integer_type.t
new file mode 100644
index 0000000..4688880
--- /dev/null
+++ b/clownfish/t/102-integer_type.t
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 101;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $integer_type = Clownfish::Type->new_integer(
+ specifier => 'int32_t',
+ const => 1,
+);
+ok( $integer_type->const, "const" );
+is( $integer_type->get_specifier, "int32_t" );
+like( $integer_type->to_c, qr/const/, "'const' in C representation" );
+
+my $parser = Clownfish::Parser->new;
+
+my @chy_specifiers = qw(
+ bool_t
+);
+my @c_specifiers = qw(
+ char
+ short
+ int
+ long
+ size_t
+ int8_t
+ int16_t
+ int32_t
+ int64_t
+ uint8_t
+ uint16_t
+ uint32_t
+ uint64_t
+);
+
+for my $chy_specifier (@chy_specifiers) {
+ is( $parser->chy_integer_specifier($chy_specifier),
+ $chy_specifier, "chy_integer_specifier: $chy_specifier" );
+ my $type = $parser->chy_integer_type($chy_specifier);
+ isa_ok( $type, "Clownfish::Type" );
+ ok( $type && $type->is_integer, "parsed Type is_integer()" );
+ $type = $parser->chy_integer_type("const $chy_specifier");
+ isa_ok( $type, "Clownfish::Type" );
+ ok( $type && $type->is_integer, "parsed const Type is_integer()" );
+ ok( $type && $type->const, "parsed const Type is const()" );
+ my $bogus = $chy_specifier . "oot_toot";
+ ok( !$parser->chy_integer_specifier($bogus),
+ "chy_integer_specifier guards against partial word matches" );
+}
+
+for my $c_specifier (@c_specifiers) {
+ is( $parser->c_integer_specifier($c_specifier),
+ $c_specifier, "c_integer_specifier: $c_specifier" );
+ my $type = $parser->c_integer_type($c_specifier);
+ isa_ok( $type, "Clownfish::Type" );
+ ok( $type && $type->is_integer, "parsed Type is_integer()" );
+ $type = $parser->c_integer_type("const $c_specifier");
+ isa_ok( $type, "Clownfish::Type" );
+ ok( $type && $type->is_integer, "parsed const Type is_integer()" );
+ ok( $type && $type->const, "parsed const Type is const()" );
+ my $bogus = $c_specifier . "y";
+ ok( !$parser->c_integer_specifier($bogus),
+ "c_integer_specifier guards against partial word matches" );
+}
diff --git a/clownfish/t/103-float_type.t b/clownfish/t/103-float_type.t
new file mode 100644
index 0000000..d0be4ca
--- /dev/null
+++ b/clownfish/t/103-float_type.t
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 17;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $float_type = Clownfish::Type->new_float(
+ specifier => 'float',
+ const => 1,
+);
+ok( $float_type->const, "const" );
+is( $float_type->get_specifier, "float" );
+like( $float_type->to_c, qr/const/, "'const' in C representation" );
+
+my $parser = Clownfish::Parser->new;
+
+for my $specifier (qw( float double)) {
+ is( $parser->c_float_specifier($specifier),
+ $specifier, "c_float_specifier: $specifier" );
+ my $type = $parser->float_type($specifier);
+ isa_ok( $type, "Clownfish::Type" );
+ ok( $type && $type->is_floating, "parsed specifier is_floating()" );
+ $type = $parser->float_type("const $specifier");
+ isa_ok( $type, "Clownfish::Type" );
+ ok( $type && $type->is_floating, "parsed const specifier is_floating()" );
+ ok( $type && $type->const, "parsed const specifier is_floating()" );
+ my $bogus = $specifier . "y";
+ ok( !$parser->c_float_specifier($bogus),
+ "c_float_specifier guards against partial word matches" );
+}
+
diff --git a/clownfish/t/104-void_type.t b/clownfish/t/104-void_type.t
new file mode 100644
index 0000000..99753af
--- /dev/null
+++ b/clownfish/t/104-void_type.t
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 12;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $void_type = Clownfish::Type->new_void;
+is( $void_type->get_specifier, "void", "specifier defaults to 'void'" );
+is( $void_type->to_c, "void", "to_c" );
+ok( $void_type->is_void, "is_void" );
+
+$void_type = Clownfish::Type->new_void( const => 1 );
+ok( $void_type->const, "const" );
+like( $void_type->to_c, qr/const/, "'const' in C representation" );
+
+my $parser = Clownfish::Parser->new;
+
+is( $parser->void_type_specifier('void'), 'void', 'void_type_specifier' );
+$void_type = $parser->void_type('void');
+isa_ok( $void_type, "Clownfish::Type" );
+ok( $void_type && $void_type->is_void,
+ "Parser calls new_void() when parsing 'void'" );
+my $const_void_type = $parser->void_type('const void');
+isa_ok( $const_void_type, "Clownfish::Type" );
+ok( $const_void_type && $const_void_type->is_void,
+ "Parser calls new_void() when parsing 'const void'"
+);
+ok( $const_void_type && $const_void_type->const,
+ "Parser preserves const when parsing 'const void'"
+);
+ok( !$parser->void_type_specifier('voidable'),
+ "void_type_specifier guards against partial word matches" );
+
diff --git a/clownfish/t/105-object_type.t b/clownfish/t/105-object_type.t
new file mode 100644
index 0000000..48dbe1d
--- /dev/null
+++ b/clownfish/t/105-object_type.t
@@ -0,0 +1,118 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 57;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+
+# Set and leave parcel.
+my $parcel = $parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+for my $bad_specifier (qw( foo fooBar Foo_Bar FOOBAR 1Foo 1FOO )) {
+ ok( !$parser->object_type_specifier($bad_specifier),
+ "reject bad object_type_specifier $bad_specifier"
+ );
+ eval {
+ my $type = Clownfish::Type->new_object(
+ parcel => 'Neato',
+ specifier => $bad_specifier,
+ );
+ };
+ like( $@, qr/specifier/,
+ "constructor rejects bad specifier $bad_specifier" );
+}
+
+for my $specifier (qw( Foo FooJr FooIII Foo4th )) {
+ is( $parser->object_type_specifier($specifier),
+ $specifier, "object_type_specifier: $specifier" );
+ is( $parser->object_type_specifier("neato_$specifier"),
+ "neato_$specifier", "object_type_specifier: neato_$specifier" );
+ my $type = $parser->object_type("$specifier*");
+ ok( $type && $type->is_object, "$specifier*" );
+ $type = $parser->object_type("neato_$specifier*");
+ ok( $type && $type->is_object, "neato_$specifier*" );
+ $type = $parser->object_type("const $specifier*");
+ ok( $type && $type->is_object, "const $specifier*" );
+ $type = $parser->object_type("incremented $specifier*");
+ ok( $type && $type->is_object, "incremented $specifier*" );
+ $type = $parser->object_type("decremented $specifier*");
+ ok( $type && $type->is_object, "decremented $specifier*" );
+}
+
+eval { my $type = Clownfish::Type->new_object };
+like( $@, qr/specifier/i, "specifier required" );
+
+for ( 0, 2 ) {
+ eval {
+ my $type = Clownfish::Type->new_object(
+ specifier => 'Foo',
+ indirection => $_,
+ );
+ };
+ like( $@, qr/indirection/i, "invalid indirection of $_" );
+}
+
+my $foo_type = Clownfish::Type->new_object( specifier => 'Foo' );
+my $another_foo = Clownfish::Type->new_object( specifier => 'Foo' );
+ok( $foo_type->equals($another_foo), "equals" );
+
+my $bar_type = Clownfish::Type->new_object( specifier => 'Bar' );
+ok( !$foo_type->equals($bar_type), "different specifier spoils equals" );
+
+my $foreign_foo = Clownfish::Type->new_object(
+ specifier => 'Foo',
+ parcel => 'Foreign',
+);
+ok( !$foo_type->equals($foreign_foo), "different parcel spoils equals" );
+is( $foreign_foo->get_specifier, "foreign_Foo",
+ "prepend parcel prefix to specifier" );
+
+my $incremented_foo = Clownfish::Type->new_object(
+ specifier => 'Foo',
+ incremented => 1,
+);
+ok( $incremented_foo->incremented, "incremented" );
+ok( !$foo_type->incremented, "not incremented" );
+ok( !$foo_type->equals($incremented_foo),
+ "different incremented spoils equals"
+);
+
+my $decremented_foo = Clownfish::Type->new_object(
+ specifier => 'Foo',
+ decremented => 1,
+);
+ok( $decremented_foo->decremented, "decremented" );
+ok( !$foo_type->decremented, "not decremented" );
+ok( !$foo_type->equals($decremented_foo),
+ "different decremented spoils equals"
+);
+
+my $const_foo = Clownfish::Type->new_object(
+ specifier => 'Foo',
+ const => 1,
+);
+ok( !$foo_type->equals($const_foo), "different const spoils equals" );
+like( $const_foo->to_c, qr/const/, "const included in C representation" );
+
+my $string_type = Clownfish::Type->new_object( specifier => 'CharBuf', );
+ok( !$foo_type->is_string_type, "Not is_string_type" );
+ok( $string_type->is_string_type, "is_string_type" );
+
diff --git a/clownfish/t/106-va_list_type.t b/clownfish/t/106-va_list_type.t
new file mode 100644
index 0000000..3914ce8
--- /dev/null
+++ b/clownfish/t/106-va_list_type.t
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 5;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $va_list_type = Clownfish::Type->new_va_list;
+is( $va_list_type->get_specifier,
+ "va_list", "specifier defaults to 'va_list'" );
+is( $va_list_type->to_c, "va_list", "to_c" );
+
+my $parser = Clownfish::Parser->new;
+
+is( $parser->va_list_type_specifier('va_list'),
+ 'va_list', 'va_list_type_specifier' );
+my $type = $parser->va_list_type('va_list');
+ok( $type && $type->is_va_list, "parse va_list" );
+ok( !$parser->va_list_type_specifier('va_listable'),
+ "va_list_type_specifier guards against partial word matches"
+);
+
diff --git a/clownfish/t/107-arbitrary_type.t b/clownfish/t/107-arbitrary_type.t
new file mode 100644
index 0000000..61a08ac
--- /dev/null
+++ b/clownfish/t/107-arbitrary_type.t
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 12;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $foo_type = Clownfish::Type->new_arbitrary(
+ parcel => 'Neato',
+ specifier => "foo_t",
+);
+is( $foo_type->get_specifier, "foo_t", "get_specifier" );
+is( $foo_type->to_c, "foo_t", "to_c" );
+
+my $compare_t_type = Clownfish::Type->new_arbitrary(
+ parcel => 'Neato',
+ specifier => "Sort_compare_t",
+);
+is( $compare_t_type->get_specifier,
+ "neato_Sort_compare_t", "Prepend prefix to specifier" );
+is( $compare_t_type->to_c, "neato_Sort_compare_t", "to_c" );
+
+my $twin = Clownfish::Type->new_arbitrary(
+ parcel => 'Neato',
+ specifier => "foo_t",
+);
+ok( $foo_type->equals($twin), "equals" );
+ok( !$foo_type->equals($compare_t_type),
+ "equals spoiled by different specifier"
+);
+
+my $parser = Clownfish::Parser->new;
+
+for my $specifier (qw( foo_t Sort_compare_t )) {
+ is( $parser->arbitrary_type_specifier($specifier),
+ $specifier, 'arbitrary_type_specifier' );
+ my $type = $parser->arbitrary_type($specifier);
+ ok( $type && $type->is_arbitrary, "arbitrary_type '$specifier'" );
+ ok( !$parser->arbitrary_type_specifier( $specifier . "_y_p_e" ),
+ "arbitrary_type_specifier guards against partial word matches"
+ );
+}
diff --git a/clownfish/t/108-composite_type.t b/clownfish/t/108-composite_type.t
new file mode 100644
index 0000000..ee6040c
--- /dev/null
+++ b/clownfish/t/108-composite_type.t
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 24;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+
+is( $parser->type_postfix($_), $_, "postfix: $_" )
+ for ( '[]', '[A_CONSTANT]', '*' );
+is( $parser->type_postfix('[ FOO ]'), '[FOO]', "type_postfix: [ FOO ]" );
+
+my @composite_type_strings = (
+ qw(
+ char*
+ char**
+ char***
+ int32_t*
+ Obj**
+ neato_method_t[]
+ neato_method_t[1]
+ multi_dimensional_t[SOME][A_FEW]
+ ),
+ 'char * * ',
+ 'const Obj**',
+ 'const void*',
+);
+
+for my $input (@composite_type_strings) {
+ my $type = $parser->type($input);
+ ok( $type && $type->is_composite, $input );
+}
+
+eval { my $type = Clownfish::Type->new_composite };
+like( $@, qr/child/i, "child required" );
+
+my $foo_type = Clownfish::Type->new_object( specifier => 'Foo' );
+my $composite_type = Clownfish::Type->new_composite(
+ child => $foo_type,
+ indirection => 1,
+);
+is( $composite_type->get_specifier,
+ 'Foo', "get_specifier delegates to child" );
+
+my $other = Clownfish::Type->new_composite(
+ child => $foo_type,
+ indirection => 1,
+);
+ok( $composite_type->equals($other), "equals" );
+ok( $composite_type->is_composite, "is_composite" );
+
+my $bar_type = Clownfish::Type->new_object( specifier => 'Bar' );
+my $bar_composite = Clownfish::Type->new_composite(
+ child => $bar_type,
+ indirection => 1,
+);
+ok( !$composite_type->equals($bar_composite),
+ "equals spoiled by different child"
+);
+
+my $foo_array = $parser->type("foo_t[]")
+ or die "Can't parse foo_t[]";
+is( $foo_array->get_array, '[]', "get_array" );
+unlike( $foo_array->to_c, qr/\[\]/, "array subscripts not included by to_c" );
+
+my $foo_array_array = $parser->type("foo_t[][]")
+ or die "Can't parse foo_t[][]";
+ok( !$foo_array->equals($foo_array_array),
+ "equals spoiled by different array postfixes"
+);
+
+my $foo_star = $parser->type("foo_t*")
+ or die "Can't parse foo_t*";
+my $foo_star_star = $parser->type("foo_t**")
+ or die "Can't parse foo_t**";
+ok( !$foo_star->equals($foo_star_star),
+ "equals spoiled by different levels of indirection" );
diff --git a/clownfish/t/200-function.t b/clownfish/t/200-function.t
new file mode 100644
index 0000000..cdd03af
--- /dev/null
+++ b/clownfish/t/200-function.t
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 6;
+
+BEGIN { use_ok('Clownfish::Function') }
+use Clownfish::Parser;
+use Clownfish::Parcel;
+
+my $parser = Clownfish::Parser->new;
+$parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+my %args = (
+ parcel => 'Neato',
+ return_type => $parser->type('Obj*'),
+ class_name => 'Neato::Foo',
+ class_cnick => 'Foo',
+ param_list => $parser->param_list('(int32_t some_num)'),
+ micro_sym => 'return_an_obj',
+);
+
+my $func = Clownfish::Function->new(%args);
+isa_ok( $func, "Clownfish::Function" );
+
+eval { my $death = Clownfish::Function->new( %args, extra_arg => undef ) };
+like( $@, qr/extra_arg/, "Extra arg kills constructor" );
+
+eval { Clownfish::Function->new( %args, micro_sym => 'Uh_Oh' ); };
+like( $@, qr/Uh_Oh/, "invalid micro_sym kills constructor" );
+
+my %sub_args = ( class => 'Neato::Obj', cnick => 'Obj' );
+
+isa_ok(
+ $parser->subroutine_declaration_statement( $_, 0, %sub_args, inert => 1 )
+ ->{declared},
+ "Clownfish::Function",
+ "function declaration: $_"
+ )
+ for (
+ 'inert int running_count(int biscuit);',
+ 'public inert Hash* init_fave_hash(int32_t num_buckets, bool_t o_rly);',
+ );
diff --git a/clownfish/t/201-method.t b/clownfish/t/201-method.t
new file mode 100644
index 0000000..83dec40
--- /dev/null
+++ b/clownfish/t/201-method.t
@@ -0,0 +1,134 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 30;
+
+BEGIN { use_ok('Clownfish::Method') }
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+$parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+my %args = (
+ parcel => 'Neato',
+ return_type => $parser->type('Obj*'),
+ class_name => 'Neato::Foo',
+ class_cnick => 'Foo',
+ param_list => $parser->param_list('(Foo *self, int32_t count = 0)'),
+ macro_sym => 'Return_An_Obj',
+);
+
+my $method = Clownfish::Method->new(%args);
+isa_ok( $method, "Clownfish::Method" );
+
+ok( $method->parcel, "parcel exposure by default" );
+
+eval { my $death = Clownfish::Method->new( %args, extra_arg => undef ) };
+like( $@, qr/extra_arg/, "Extra arg kills constructor" );
+
+eval { Clownfish::Method->new( %args, macro_sym => 'return_an_obj' ); };
+like( $@, qr/macro_sym/, "Invalid macro_sym kills constructor" );
+
+my $dupe = Clownfish::Method->new(%args);
+ok( $method->compatible($dupe), "compatible()" );
+
+my $macro_sym_differs = Clownfish::Method->new( %args, macro_sym => 'Eat' );
+ok( !$method->compatible($macro_sym_differs),
+ "different macro_sym spoils compatible()"
+);
+ok( !$macro_sym_differs->compatible($method), "... reversed" );
+
+my $extra_param = Clownfish::Method->new( %args,
+ param_list =>
+ $parser->param_list('(Foo *self, int32_t count = 0, int b)'), );
+ok( !$method->compatible($macro_sym_differs),
+ "extra param spoils compatible()"
+);
+ok( !$extra_param->compatible($method), "... reversed" );
+
+my $default_differs = Clownfish::Method->new( %args,
+ param_list => $parser->param_list('(Foo *self, int32_t count = 1)'), );
+ok( !$method->compatible($default_differs),
+ "different initial_value spoils compatible()"
+);
+ok( !$default_differs->compatible($method), "... reversed" );
+
+my $missing_default = Clownfish::Method->new( %args,
+ param_list => $parser->param_list('(Foo *self, int32_t count)'), );
+ok( !$method->compatible($missing_default),
+ "missing initial_value spoils compatible()"
+);
+ok( !$missing_default->compatible($method), "... reversed" );
+
+my $param_name_differs = Clownfish::Method->new( %args,
+ param_list => $parser->param_list('(Foo *self, int32_t countess)'), );
+ok( !$method->compatible($param_name_differs),
+ "different param name spoils compatible()"
+);
+ok( !$param_name_differs->compatible($method), "... reversed" );
+
+my $param_type_differs = Clownfish::Method->new( %args,
+ param_list => $parser->param_list('(Foo *self, uint32_t count)'), );
+ok( !$method->compatible($param_type_differs),
+ "different param type spoils compatible()"
+);
+ok( !$param_type_differs->compatible($method), "... reversed" );
+
+my $self_type_differs = Clownfish::Method->new(
+ %args,
+ class_name => 'Neato::Bar',
+ class_cnick => 'Bar',
+ param_list => $parser->param_list('(Bar *self, int32_t count = 0)'),
+);
+ok( $method->compatible($self_type_differs),
+ "different self type still compatible(), since can't test inheritance" );
+ok( $self_type_differs->compatible($method), "... reversed" );
+
+my $not_final = Clownfish::Method->new(%args);
+my $final = $not_final->finalize;
+
+eval { $method->override($final); };
+like( $@, qr/final/i, "Can't override final method" );
+
+ok( $not_final->compatible($final), "Finalize clones properly" );
+
+for my $meth_meth (qw( short_method_sym full_method_sym full_offset_sym)) {
+ eval { my $blah = $method->$meth_meth; };
+ like( $@, qr/invoker/, "$meth_meth requires invoker" );
+}
+
+my %sub_args = ( class => 'Neato::Obj', cnick => 'Obj' );
+
+isa_ok(
+ $parser->subroutine_declaration_statement( $_, 0, %sub_args )->{declared},
+ "Clownfish::Method",
+ "method declaration: $_"
+ )
+ for (
+ 'public int Do_Foo(Obj *self);',
+ 'parcel Obj* Gimme_An_Obj(Obj *self);',
+ 'void Do_Whatever(Obj *self, uint32_t a_num, float real);',
+ 'private Foo* Fetch_Foo(Obj *self, int num);',
+ );
+
+ok( $parser->subroutine_declaration_statement( $_, 0, %sub_args )->{declared}
+ ->final,
+ "final method: $_"
+) for ( 'public final void The_End(Obj *self);', );
+
diff --git a/clownfish/t/202-overridden_method.t b/clownfish/t/202-overridden_method.t
new file mode 100644
index 0000000..c166320
--- /dev/null
+++ b/clownfish/t/202-overridden_method.t
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 1;
+
+use Clownfish::Method;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+$parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+my %args = (
+ return_type => $parser->type('Obj*'),
+ class_name => 'Neato::Foo',
+ class_cnick => 'Foo',
+ param_list => $parser->param_list('(Foo *self)'),
+ macro_sym => 'Return_An_Obj',
+ parcel => 'Neato',
+);
+
+my $orig = Clownfish::Method->new(%args);
+my $overrider = Clownfish::Method->new(
+ %args,
+ param_list => $parser->param_list('(FooJr *self)'),
+ class_name => 'Neato::Foo::FooJr',
+ class_cnick => 'FooJr'
+);
+$overrider->override($orig);
+ok( !$overrider->novel );
+
diff --git a/clownfish/t/203-final_method.t b/clownfish/t/203-final_method.t
new file mode 100644
index 0000000..7cd8aeb
--- /dev/null
+++ b/clownfish/t/203-final_method.t
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 2;
+
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+$parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+my %args = (
+ return_type => $parser->type('Obj*'),
+ class_name => 'Neato::Foo',
+ class_cnick => 'Foo',
+ param_list => $parser->param_list('(Foo* self)'),
+ macro_sym => 'Return_An_Obj',
+ parcel => 'Neato',
+);
+
+my $not_final_method = Clownfish::Method->new(%args);
+my $final_method = $not_final_method->finalize;
+ok( !$not_final_method->final, "not final by default" );
+ok( $final_method->final, "finalize" );
+
diff --git a/clownfish/t/300-variable.t b/clownfish/t/300-variable.t
new file mode 100644
index 0000000..ee8c14d
--- /dev/null
+++ b/clownfish/t/300-variable.t
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 13;
+use Clownfish::Type;
+use Clownfish::Parser;
+
+BEGIN { use_ok('Clownfish::Variable') }
+
+my $parser = Clownfish::Parser->new;
+$parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+sub new_type { $parser->type(shift) }
+
+eval {
+ my $death = Clownfish::Variable->new(
+ micro_sym => 'foo',
+ type => new_type('int'),
+ extra_arg => undef,
+ );
+};
+like( $@, qr/extra_arg/, "Extra arg kills constructor" );
+
+eval { my $death = Clownfish::Variable->new( micro_sym => 'foo' ) };
+like( $@, qr/type/, "type is required" );
+eval { my $death = Clownfish::Variable->new( type => new_type('int32_t') ) };
+like( $@, qr/micro_sym/, "micro_sym is required" );
+
+my $var = Clownfish::Variable->new(
+ micro_sym => 'foo',
+ type => new_type('float*')
+);
+is( $var->local_c, 'float* foo', "local_c" );
+is( $var->local_declaration, 'float* foo;', "declaration" );
+ok( $var->local, "default to local access" );
+
+$var = Clownfish::Variable->new(
+ micro_sym => 'foo',
+ type => new_type('float[1]')
+);
+is( $var->local_c, 'float foo[1]',
+ "to_c appends array to var name rather than type specifier" );
+
+$var = Clownfish::Variable->new(
+ parcel => 'Neato',
+ micro_sym => 'foo',
+ type => new_type("Foo*"),
+ class_name => 'Crustacean::Lobster::LobsterClaw',
+ class_cnick => 'LobClaw',
+);
+is( $var->global_c, 'neato_Foo* neato_LobClaw_foo', "global_c" );
+
+isa_ok( $parser->var_declaration_statement($_)->{declared},
+ "Clownfish::Variable", "var_declaration_statement: $_" )
+ for (
+ 'parcel int foo;',
+ 'private Obj *obj;',
+ 'public inert int32_t **foo;',
+ 'Dog *fido;'
+ );
diff --git a/clownfish/t/301-param_list.t b/clownfish/t/301-param_list.t
new file mode 100644
index 0000000..4110dae
--- /dev/null
+++ b/clownfish/t/301-param_list.t
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 13;
+
+BEGIN { use_ok('Clownfish::ParamList') }
+use Clownfish::Type;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+$parser->parcel_definition('parcel Neato;')
+ or die "failed to process parcel_definition";
+
+isa_ok( $parser->param_variable($_),
+ "Clownfish::Variable", "param_variable: $_" )
+ for ( 'uint32_t baz', 'CharBuf *stuff', 'float **ptr', );
+
+my $param_list = $parser->param_list("(Obj *self, int num)");
+isa_ok( $param_list, "Clownfish::ParamList" );
+ok( !$param_list->variadic, "not variadic" );
+is( $param_list->to_c, 'neato_Obj* self, int num', "to_c" );
+is( $param_list->name_list, 'self, num', "name_list" );
+
+$param_list = $parser->param_list("(Obj *self=NULL, int num, ...)");
+ok( $param_list->variadic, "variadic" );
+is_deeply(
+ $param_list->get_initial_values,
+ [ "NULL", undef ],
+ "initial_values"
+);
+is( $param_list->to_c, 'neato_Obj* self, int num, ...', "to_c" );
+is( $param_list->num_vars, 2, "num_vars" );
+isa_ok( $param_list->get_variables->[0],
+ "Clownfish::Variable", "get_variables..." );
+
diff --git a/clownfish/t/400-class.t b/clownfish/t/400-class.t
new file mode 100644
index 0000000..ad0938f
--- /dev/null
+++ b/clownfish/t/400-class.t
@@ -0,0 +1,230 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 52;
+use Clownfish::Class;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+
+my $thing = Clownfish::Variable->new(
+ parcel => 'Neato',
+ class_name => 'Foo',
+ type => $parser->type('Thing*'),
+ micro_sym => 'thing',
+);
+my $widget = Clownfish::Variable->new(
+ class_name => 'Widget',
+ type => $parser->type('Widget*'),
+ micro_sym => 'widget',
+);
+my $tread_water = Clownfish::Function->new(
+ parcel => 'Neato',
+ class_name => 'Foo',
+ return_type => $parser->type('void'),
+ micro_sym => 'tread_water',
+ param_list => $parser->param_list('()'),
+);
+my %foo_create_args = (
+ parcel => 'Neato',
+ class_name => 'Foo',
+);
+
+my $foo = Clownfish::Class->create(%foo_create_args);
+$foo->add_function($tread_water);
+$foo->add_member_var($thing);
+$foo->add_inert_var($widget);
+eval { Clownfish::Class->create(%foo_create_args) };
+like( $@, qr/conflict/i,
+ "Can't call create for the same class more than once" );
+my $should_be_foo = Clownfish::Class->fetch_singleton(
+ parcel => 'Neato',
+ class_name => 'Foo',
+);
+is( $foo, $should_be_foo, "fetch_singleton" );
+
+my $foo_jr = Clownfish::Class->create(
+ parcel => 'Neato',
+ class_name => 'Foo::FooJr',
+ parent_class_name => 'Foo',
+);
+$foo_jr->add_attribute( dumpable => 1 );
+
+ok( $foo_jr->has_attribute('dumpable'), 'has_attribute' );
+is( $foo_jr->get_struct_sym, 'FooJr', "struct_sym" );
+is( $foo_jr->full_struct_sym, 'neato_FooJr', "full_struct_sym" );
+
+my $final_foo = Clownfish::Class->create(
+ parcel => 'Neato',
+ class_name => 'Foo::FooJr::FinalFoo',
+ parent_class_name => 'Foo::FooJr',
+ source_class => 'Foo::FooJr',
+ final => 1,
+);
+$final_foo->add_attribute( dumpable => 1 );
+ok( $final_foo->final, "final" );
+is( $final_foo->include_h, 'Foo/FooJr.h', "inlude_h uses source_class" );
+is( $final_foo->get_parent_class_name, 'Foo::FooJr',
+ "get_parent_class_name" );
+
+my $do_stuff
+ = $parser->subroutine_declaration_statement( 'void Do_Stuff(Foo *self);',
+ 0, class => 'Foo' )->{declared}
+ or die "parsing failure";
+$foo->add_method($do_stuff);
+
+my $inert_do_stuff
+ = $parser->subroutine_declaration_statement(
+ 'void Do_Stuff(InertFoo *self);',
+ 0, class => 'InertFoo' )->{declared}
+ or die "parsing failure";
+my %inert_args = (
+ parcel => 'Neato',
+ class_name => 'InertFoo',
+ inert => 1,
+);
+eval {
+ my $class = Clownfish::Class->create(%inert_args);
+ $class->add_method($inert_do_stuff);
+};
+like(
+ $@,
+ qr/inert class/i,
+ "Error out on conflict between inert attribute and object method"
+);
+
+$foo->add_child($foo_jr);
+$foo_jr->add_child($final_foo);
+$foo->grow_tree;
+eval { $foo->grow_tree };
+like( $@, qr/grow_tree/, "call grow_tree only once." );
+eval { $foo_jr->add_method($do_stuff) };
+like( $@, qr/grow_tree/, "Forbid add_method after grow_tree." );
+
+is( $foo_jr->get_parent, $foo, "grow_tree, one level" );
+is( $final_foo->get_parent, $foo_jr, "grow_tree, two levels" );
+is( $foo->novel_method("Do_Stuff"), $do_stuff, 'novel_method' );
+is( $foo_jr->method("Do_Stuff"), $do_stuff, "inherited method" );
+ok( !$foo_jr->novel_method("Do_Stuff"), 'inherited method not novel' );
+ok( $final_foo->method("Do_Stuff")->final, "Finalize inherited method" );
+ok( !$foo_jr->method("Do_Stuff")->final, "Don't finalize method in parent" );
+is_deeply( $foo->inert_vars, [$widget], "inert vars" );
+is_deeply( $foo->functions, [$tread_water], "inert funcs" );
+is_deeply( $foo->methods, [$do_stuff], "methods" );
+is_deeply( $foo->novel_methods, [$do_stuff], "novel_methods" );
+is_deeply( $foo->novel_member_vars, [$thing], "novel_member_vars" );
+is_deeply( $foo_jr->member_vars, [$thing], "inherit member vars" );
+is_deeply( $foo_jr->functions, [], "don't inherit inert funcs" );
+is_deeply( $foo_jr->novel_member_vars, [], "novel_member_vars" );
+is_deeply( $foo_jr->inert_vars, [], "don't inherit inert vars" );
+is_deeply( $final_foo->novel_methods, [], "novel_methods" );
+
+like( $foo_jr->get_autocode, qr/load/i, "autogenerate Dump/Load" );
+is_deeply( $foo->tree_to_ladder, [ $foo, $foo_jr, $final_foo ],
+ 'tree_to_ladder' );
+
+ok( $parser->class_modifier($_), "class_modifier: $_" )
+ for (qw( abstract inert ));
+
+ok( $parser->class_inheritance($_), "class_inheritance: $_" )
+ for ( 'inherits Foo', 'inherits Foo::FooJr::FooIII' );
+
+my $class_content
+ = 'public class Foo::FooJr cnick FooJr inherits Foo { private int num; }';
+my $class = $parser->class_declaration($class_content);
+isa_ok( $class, "Clownfish::Class", "class_declaration FooJr" );
+ok( ( scalar grep { $_->micro_sym eq 'num' } @{ $class->member_vars } ),
+ "parsed private member var" );
+
+$class_content = q|
+ /**
+ * Bow wow.
+ *
+ * Wow wow wow.
+ */
+ public class Animal::Dog inherits Animal : lovable : drooly {
+ public inert Dog* init(Dog *self, CharBuf *name, CharBuf *fave_food);
+ inert uint32_t count();
+ inert uint64_t num_dogs;
+
+ private CharBuf *name;
+ private bool_t likes_to_go_fetch;
+ private void Chase_Tail(Dog *self);
+
+ ChewToy *squishy;
+
+ void Destroy(Dog *self);
+ public CharBuf* Bark(Dog *self);
+ public void Eat(Dog *self);
+ public void Bite(Dog *self, Enemy *enemy);
+ public Thing *Fetch(Dog *self, Thing *thing);
+ public final void Bury(Dog *self, Bone *bone);
+ public Owner *mom;
+ public abstract incremented nullable Thing*
+ Scratch(Dog *self);
+
+ int32_t[1] flexible_array_at_end_of_struct;
+ }
+|;
+
+$class = $parser->class_declaration($class_content);
+isa_ok( $class, "Clownfish::Class", "class_declaration Dog" );
+ok( ( scalar grep { $_->micro_sym eq 'num_dogs' } @{ $class->inert_vars } ),
+ "parsed inert var" );
+ok( ( scalar grep { $_->micro_sym eq 'mom' } @{ $class->member_vars } ),
+ "parsed public member var" );
+ok( ( scalar grep { $_->micro_sym eq 'squishy' } @{ $class->member_vars } ),
+ "parsed parcel member var" );
+ok( ( scalar grep { $_->micro_sym eq 'init' } @{ $class->functions } ),
+ "parsed function" );
+ok( ( scalar grep { $_->micro_sym eq 'chase_tail' } @{ $class->methods } ),
+ "parsed private method" );
+ok( ( scalar grep { $_->micro_sym eq 'destroy' } @{ $class->methods } ),
+ "parsed parcel method" );
+ok( ( scalar grep { $_->micro_sym eq 'bury' } @{ $class->methods } ),
+ "parsed public method" );
+ok( ( scalar grep { $_->micro_sym eq 'scratch' } @{ $class->methods } ),
+ "parsed public abstract nullable method" );
+
+for my $method ( @{ $class->methods } ) {
+ if ( $method->micro_sym eq 'scratch' ) {
+ ok( $method->get_return_type->nullable,
+ "public abstract incremented nullable flagged as nullable" );
+ }
+}
+is( ( scalar grep { $_->public } @{ $class->methods } ),
+ 6, "pass acl to Method constructor" );
+ok( $class->has_attribute('lovable'), "parsed class attribute" );
+ok( $class->has_attribute('drooly'), "parsed second class attribute" );
+
+$class_content = qq|
+ parcel inert class Rigor::Mortis cnick Mort {
+ parcel inert void lie_still();
+ }|;
+$class = $parser->class_declaration($class_content);
+isa_ok( $class, "Clownfish::Class", "inert class_declaration" );
+ok( $class->inert, "inert modifier parsed and passed to constructor" );
+
+$class_content = qq|
+ final class Ultimo {
+ /** Throws an error.
+ */
+ void Say_Never(Ultimo *self);
+ }|;
+$class = $parser->class_declaration($class_content);
+ok( $class->final, "final class_declaration" );
diff --git a/clownfish/t/401-c_block.t b/clownfish/t/401-c_block.t
new file mode 100644
index 0000000..610d8b8
--- /dev/null
+++ b/clownfish/t/401-c_block.t
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 5;
+
+use Clownfish::CBlock;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+
+my $block = Clownfish::CBlock->new( contents => 'int foo;' );
+isa_ok( $block, "Clownfish::CBlock" );
+is( $block->get_contents, 'int foo;', "get_contents" );
+eval { Clownfish::CBlock->new };
+like( $@, qr/contents/, "content required" );
+
+$block = $parser->embed_c(qq| __C__\n#define FOO 1\n__END_C__ |);
+
+isa_ok( $block, "Clownfish::CBlock" );
+is( $block->get_contents, "#define FOO 1\n", "parse embed_c" );
+
diff --git a/clownfish/t/402-parcel.t b/clownfish/t/402-parcel.t
new file mode 100644
index 0000000..91d080f
--- /dev/null
+++ b/clownfish/t/402-parcel.t
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 7;
+
+BEGIN { use_ok('Clownfish::Parcel') }
+
+package ClownfishyThing;
+use base qw( Clownfish::Symbol );
+
+sub new {
+ return shift->SUPER::new( micro_sym => 'sym', exposure => 'parcel', @_ );
+}
+
+package main;
+
+# Register singleton.
+Clownfish::Parcel->singleton( name => 'Crustacean', cnick => 'Crust', );
+
+my $thing = ClownfishyThing->new;
+is( $thing->get_prefix, '', 'get_prefix with no parcel' );
+is( $thing->get_Prefix, '', 'get_Prefix with no parcel' );
+is( $thing->get_PREFIX, '', 'get_PREFIx with no parcel' );
+
+$thing = ClownfishyThing->new( parcel => 'Crustacean' );
+is( $thing->get_prefix, 'crust_', 'get_prefix with parcel' );
+is( $thing->get_Prefix, 'Crust_', 'get_Prefix with parcel' );
+is( $thing->get_PREFIX, 'CRUST_', 'get_PREFIx with parcel' );
+
diff --git a/clownfish/t/403-file.t b/clownfish/t/403-file.t
new file mode 100644
index 0000000..957090f
--- /dev/null
+++ b/clownfish/t/403-file.t
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 17;
+
+use Clownfish::File;
+use Clownfish::Parser;
+
+my $parser = Clownfish::Parser->new;
+
+my $parcel_declaration = "parcel Stuff;";
+my $class_content = qq|
+ class Stuff::Thing {
+ Foo *foo;
+ Bar *bar;
+ }
+|;
+my $c_block = "__C__\nint foo;\n__END_C__\n";
+
+my $file = $parser->file( "$parcel_declaration\n$class_content\n$c_block",
+ 0, source_class => 'Stuff::Thing' );
+
+is( $file->get_source_class, "Stuff::Thing", "get_source_class" );
+
+my $guard_name = $file->guard_name;
+is( $guard_name, "H_STUFF_THING", "guard_name" );
+like( $file->guard_start, qr/$guard_name/, "guard_start" );
+like( $file->guard_close, qr/$guard_name/,
+ "guard_close includes guard_name" );
+
+ok( !$file->get_modified, "modified false at start" );
+$file->set_modified(1);
+ok( $file->get_modified, "set_modified, get_modified" );
+
+my $path_sep = $^O =~ /^mswin/i ? '\\' : '/';
+my $path_to_stuff_thing = join( $path_sep, qw( path to Stuff Thing ) );
+is( $file->cfh_path('path/to'), "$path_to_stuff_thing.cfh", "cfh_path" );
+is( $file->c_path('path/to'), "$path_to_stuff_thing.c", "c_path" );
+is( $file->h_path('path/to'), "$path_to_stuff_thing.h", "h_path" );
+
+my $classes = $file->classes;
+is( scalar @$classes, 1, "classes() filters blocks" );
+my $class = $classes->[0];
+my ( $foo, $bar ) = @{ $class->member_vars };
+is( $foo->get_type->get_specifier,
+ 'stuff_Foo', 'file production picked up parcel def' );
+is( $bar->get_type->get_specifier, 'stuff_Bar', 'parcel def is sticky' );
+
+my $blocks = $file->blocks;
+is( scalar @$blocks, 3, "all three blocks" );
+isa_ok( $blocks->[0], "Clownfish::Parcel" );
+isa_ok( $blocks->[1], "Clownfish::Class" );
+isa_ok( $blocks->[2], "Clownfish::CBlock" );
+
+$file = $parser->file( $class_content, 0, source_class => 'Stuff::Thing' );
+($class) = @{ $file->classes };
+( $foo, $bar ) = @{ $class->member_vars };
+is( $foo->get_type->get_specifier, 'Foo', 'file production resets parcel' );
+
diff --git a/clownfish/t/500-hierarchy.t b/clownfish/t/500-hierarchy.t
new file mode 100644
index 0000000..fdb16d7
--- /dev/null
+++ b/clownfish/t/500-hierarchy.t
@@ -0,0 +1,92 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 18;
+
+use Clownfish::Hierarchy;
+use Clownfish::Util qw( a_isa_b );
+use File::Spec::Functions qw( catfile splitpath );
+use Fcntl;
+use File::Path qw( rmtree mkpath );
+
+my %args = (
+ source => 't/cfsource',
+ dest => 't/cfdest',
+);
+
+# Clean up.
+rmtree( $args{dest} );
+
+eval { my $death = Clownfish::Hierarchy->new( %args, extra_arg => undef ) };
+like( $@, qr/extra_arg/, "Extra arg kills constructor" );
+
+my $hierarchy = Clownfish::Hierarchy->new(%args);
+isa_ok( $hierarchy, "Clownfish::Hierarchy" );
+is( $hierarchy->get_source, $args{source}, "get_source" );
+is( $hierarchy->get_dest, $args{dest}, "get_dest" );
+
+$hierarchy->build;
+
+my @files = @{ $hierarchy->files };
+is( scalar @files, 3, "recursed and found all three files" );
+my %files;
+for my $file (@files) {
+ die "not a File" unless isa_ok( $file, "Clownfish::File" );
+ ok( !$file->get_modified, "start off not modified" );
+ my ($class)
+ = grep { a_isa_b( $_, "Clownfish::Class" ) } @{ $file->blocks };
+ die "no class" unless $class;
+ $files{ $class->get_class_name } = $file;
+}
+my $animal = $files{'Animal'} or die "No Animal";
+my $dog = $files{'Animal::Dog'} or die "No Dog";
+my $util = $files{'Animal::Util'} or die "No Util";
+
+my $classes = $hierarchy->ordered_classes;
+is( scalar @$classes, 3, "all classes" );
+for my $class (@$classes) {
+ die "not a Class" unless isa_ok( $class, "Clownfish::Class" );
+}
+
+# Generate fake C files, with times set to one second ago.
+my $one_second_ago = time() - 1;
+for my $file (@files) {
+ my $h_path = $file->h_path( $args{dest} );
+ my ( undef, $dir, undef ) = splitpath($h_path);
+ mkpath($dir);
+ sysopen( my $fh, $h_path, O_CREAT | O_EXCL | O_WRONLY )
+ or die "Can't open '$h_path': $!";
+ print $fh "#include <stdio.h>\n"; # fake content.
+ close $fh or die "Can't close '$h_path': $!";
+ utime( $one_second_ago, $one_second_ago, $h_path )
+ or die "utime failed for '$h_path': $!";
+}
+
+my $path_to_animal_cf = $animal->cfh_path( $args{source} );
+utime( undef, undef, $path_to_animal_cf )
+ or die "utime for '$path_to_animal_cf' failed"; # touch
+
+$hierarchy->propagate_modified;
+
+ok( $animal->get_modified, "Animal modified" );
+ok( $dog->get_modified, "Parent's modification propagates to child's file" );
+ok( !$util->get_modified, "modification doesn't propagate to inert class" );
+
+# Clean up.
+rmtree( $args{dest} );
+
diff --git a/clownfish/t/600-parser.t b/clownfish/t/600-parser.t
new file mode 100644
index 0000000..fe8f34e
--- /dev/null
+++ b/clownfish/t/600-parser.t
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 116;
+
+BEGIN { use_ok('Clownfish::Parser') }
+
+my $parser = Clownfish::Parser->new;
+isa_ok( $parser, "Clownfish::Parser" );
+
+isa_ok( $parser->parcel_definition("parcel Fish;"),
+ "Clownfish::Parcel", "parcel_definition" );
+isa_ok( $parser->parcel_definition("parcel Crustacean cnick Crust;"),
+ "Clownfish::Parcel", "parcel_definition with cnick" );
+
+# Set and leave parcel.
+my $parcel = $parser->parcel_definition('parcel Crustacean cnick Crust;')
+ or die "failed to process parcel_definition";
+is( $Clownfish::Parser::parcel, $parcel,
+ "parcel_definition sets internal \$parcel var" );
+
+is( $parser->strip_plain_comments("/*x*/"),
+ " ", "comments replaced by spaces" );
+is( $parser->strip_plain_comments("/**x*/"),
+ "/**x*/", "docu-comment untouched" );
+is( $parser->strip_plain_comments("/*\n*/"), " \n ", "newline preserved" );
+
+for (qw( foo _foo foo_yoo FOO Foo fOO f00 )) {
+ is( $parser->identifier($_), $_, "identifier: $_" );
+}
+
+for (qw( void unsigned float uint32_t int64_t uint8_t bool_t )) {
+ ok( !$parser->identifier($_), "reserved word not an identifier: $_" );
+}
+
+is( $parser->chy_integer_specifier($_), $_, "Charmony integer specifier $_" )
+ for qw( bool_t );
+
+is( $parser->object_type_specifier($_), $_, "object_type_specifier $_" )
+ for qw( ByteBuf Obj ANDMatcher );
+
+is( $parser->type_specifier($_), $_, "type_specifier $_" )
+ for qw( uint32_t char int short long float double void ANDMatcher );
+
+is( $parser->type_qualifier($_), $_, "type_qualifier $_" ) for qw( const );
+
+is( $parser->exposure_specifier($_), $_, "exposure_specifier $_" )
+ for qw( public private parcel );
+
+is( $parser->type_postfix($_), $_, "postfix: $_" )
+ for ( '[]', '[A_CONSTANT]', '*' );
+is( $parser->type_postfix('[ FOO ]'), '[FOO]', "type_postfix: [ FOO ]" );
+
+isa_ok( $parser->type($_), "Clownfish::Type", "type $_" )
+ for ( 'const char *', 'Obj*', 'i32_t', 'char[]', 'long[1]',
+ 'i64_t[FOO]' );
+
+is( $parser->declarator($_), $_, "declarator: $_" )
+ for ( 'foo', 'bar_bar_bar' );
+
+isa_ok( $parser->param_variable($_),
+ "Clownfish::Variable", "param_variable: $_" )
+ for ( 'uint32_t baz;', 'CharBuf *stuff;', 'float **ptr;', );
+
+isa_ok( $parser->var_declaration_statement($_)->{declared},
+ "Clownfish::Variable", "var_declaration_statement: $_" )
+ for (
+ 'parcel int foo;',
+ 'private Obj *obj;',
+ 'public inert i32_t **foo;',
+ 'Dog *fido;'
+ );
+
+is( $parser->hex_constant($_), $_, "hex_constant: $_" )
+ for (qw( 0x1 0x0a 0xFFFFFFFF ));
+
+is( $parser->integer_constant($_), $_, "integer_constant: $_" )
+ for (qw( 1 -9999 0 10000 ));
+
+is( $parser->float_constant($_), $_, "float_constant: $_" )
+ for (qw( 1.0 -9999.999 0.1 0.0 ));
+
+is( $parser->string_literal($_), $_, "string_literal: $_" )
+ for ( q|"blah"|, q|"blah blah"|, q|"\\"blah\\" \\"blah\\""| );
+
+is( $parser->scalar_constant($_), $_, "scalar_constant: $_" )
+ for ( q|"blah"|, 1, 1.2, "0xFC" );
+
+my @composites = ( 'int[]', "i32_t **", "Foo **", "Foo ***", "const void *" );
+for my $composite (@composites) {
+ my $parsed = $parser->type($composite);
+ ok( $parsed && $parsed->is_composite, "composite_type: $composite" );
+}
+
+my @object_types = ( 'Obj *', "incremented Foo*", "decremented CharBuf *" );
+for my $object_type (@object_types) {
+ my $parsed = $parser->object_type($object_type);
+ ok( $parsed && $parsed->is_object, "object_type: $object_type" );
+}
+
+my %param_lists = (
+ '(int foo)' => 1,
+ '(Obj *foo, Foo **foo_ptr)' => 2,
+ '()' => 0,
+);
+while ( my ( $param_list, $num_params ) = each %param_lists ) {
+ my $parsed = $parser->param_list($param_list);
+ isa_ok( $parsed, "Clownfish::ParamList", "param_list: $param_list" );
+}
+ok( $parser->param_list("(int foo, ...)")->variadic, "variadic param list" );
+my $param_list = $parser->param_list(q|(int foo = 0xFF, char *bar ="blah")|);
+is_deeply(
+ $param_list->get_initial_values,
+ [ '0xFF', '"blah"' ],
+ "initial values"
+);
+
+my %sub_args = ( class => 'Stuff::Obj', cnick => 'Obj' );
+
+ok( $parser->declaration_statement( $_, 0, %sub_args, inert => 1 ),
+ "declaration_statment: $_" )
+ for (
+ 'public Foo* Spew_Foo(Obj *self, uint32_t *how_many);',
+ 'private Hash *hash;',
+ );
+
+is( $parser->object_type_specifier($_), $_, "object_type_specifier: $_" )
+ for (qw( Foo FooJr FooIII Foo4th ));
+
+ok( !$parser->object_type_specifier($_), "illegal object_type_specifier: $_" )
+ for (qw( foo fooBar Foo_Bar FOOBAR 1Foo 1FOO ));
+
+is( $parser->class_name($_), $_, "class_name: $_" )
+ for (qw( Foo Foo::FooJr Foo::FooJr::FooIII Foo::FooJr::FooIII::Foo4th ));
+
+ok( !$parser->class_name($_), "illegal class_name: $_" )
+ for (qw( foo fooBar Foo_Bar FOOBAR 1Foo 1FOO ));
+
+is( $parser->cnick(qq|cnick $_|), $_, "cnick: $_" ) for (qw( Foo FF ));
+
+ok( !$parser->cnick(qq|cnick $_|), "Illegal cnick: $_" )
+ for (qw( foo fOO 1Foo ));
diff --git a/clownfish/t/cfsource/Animal.cfh b/clownfish/t/cfsource/Animal.cfh
new file mode 100644
index 0000000..a4d689a
--- /dev/null
+++ b/clownfish/t/cfsource/Animal.cfh
@@ -0,0 +1,19 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Animal;
+
+abstract class Animal { }
diff --git a/clownfish/t/cfsource/Animal/Dog.cfh b/clownfish/t/cfsource/Animal/Dog.cfh
new file mode 100644
index 0000000..6d54baa
--- /dev/null
+++ b/clownfish/t/cfsource/Animal/Dog.cfh
@@ -0,0 +1,28 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Animal;
+
+class Animal::Dog inherits Animal {
+ public inert incremented Dog*
+ new();
+
+ public inert Dog*
+ init(Dog *self);
+
+ public void
+ Bark(Dog *self);
+}
diff --git a/clownfish/t/cfsource/Animal/Util.cfh b/clownfish/t/cfsource/Animal/Util.cfh
new file mode 100644
index 0000000..f5688a5
--- /dev/null
+++ b/clownfish/t/cfsource/Animal/Util.cfh
@@ -0,0 +1,23 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Animal;
+
+inert class Animal::Util {
+ inert void
+ groom(Animal *animal);
+}
+
diff --git a/clownfish/typemap b/clownfish/typemap
new file mode 100644
index 0000000..cbd7c1f
--- /dev/null
+++ b/clownfish/typemap
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+TYPEMAP
+CFCBase* CLOWNFISH_TYPE
+CFCCBlock* CLOWNFISH_TYPE
+CFCClass* CLOWNFISH_TYPE
+CFCDocuComment* CLOWNFISH_TYPE
+CFCDumpable* CLOWNFISH_TYPE
+CFCFile* CLOWNFISH_TYPE
+CFCFunction* CLOWNFISH_TYPE
+CFCHierarchy* CLOWNFISH_TYPE
+CFCMethod* CLOWNFISH_TYPE
+CFCParamList* CLOWNFISH_TYPE
+CFCParcel* CLOWNFISH_TYPE
+CFCSymbol* CLOWNFISH_TYPE
+CFCType* CLOWNFISH_TYPE
+CFCVariable* CLOWNFISH_TYPE
+
+INPUT
+
+CLOWNFISH_TYPE
+ if (!SvOK($arg)) {
+ $var = NULL;
+ }
+ else if (sv_derived_from($arg, \"${(my $t = $type) =~ s/CFC(\w+).*/Clownfish::$1/;\$t}\")) {
+ IV objint = SvIV((SV*)SvRV($arg));
+ $var = INT2PTR($type, objint);
+ }
+ else {
+ croak(\"Not a ${(my $t = $type) =~ s/CFC(\w+).*/Clownfish::$1/;\$t}\");
+ }
+
+OUTPUT
+
+CLOWNFISH_TYPE
+ sv_setref_pv($arg, \"${(my $t = $type) =~ s/CFC(\w+).*/Clownfish::$1/;\$t}\", (void*)$var);
+
diff --git a/core/Lucy/Analysis/Analyzer.c b/core/Lucy/Analysis/Analyzer.c
new file mode 100644
index 0000000..2ba60e5
--- /dev/null
+++ b/core/Lucy/Analysis/Analyzer.c
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ANALYZER
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+
+Analyzer*
+Analyzer_init(Analyzer *self) {
+ ABSTRACT_CLASS_CHECK(self, ANALYZER);
+ return self;
+}
+
+Inversion*
+Analyzer_transform_text(Analyzer *self, CharBuf *text) {
+ size_t token_len = CB_Get_Size(text);
+ Token *seed = Token_new((char*)CB_Get_Ptr8(text), token_len, 0,
+ token_len, 1.0, 1);
+ Inversion *starter = Inversion_new(seed);
+ Inversion *retval = Analyzer_Transform(self, starter);
+ DECREF(seed);
+ DECREF(starter);
+ return retval;
+}
+
+VArray*
+Analyzer_split(Analyzer *self, CharBuf *text) {
+ Inversion *inversion = Analyzer_Transform_Text(self, text);
+ VArray *out = VA_new(0);
+ Token *token;
+
+ while ((token = Inversion_Next(inversion)) != NULL) {
+ VA_Push(out, (Obj*)CB_new_from_trusted_utf8(token->text, token->len));
+ }
+
+ DECREF(inversion);
+
+ return out;
+}
+
+
diff --git a/core/Lucy/Analysis/Analyzer.cfh b/core/Lucy/Analysis/Analyzer.cfh
new file mode 100644
index 0000000..7074acd
--- /dev/null
+++ b/core/Lucy/Analysis/Analyzer.cfh
@@ -0,0 +1,54 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Tokenize/modify/filter text.
+ *
+ * An Analyzer is a filter which processes text, transforming it from one form
+ * into another. For instance, an analyzer might break up a long text into
+ * smaller pieces (L<RegexTokenizer|Lucy::Analysis::RegexTokenizer>), or it might
+ * perform case folding to facilitate case-insensitive search
+ * (L<CaseFolder|Lucy::Analysis::CaseFolder>).
+ */
+abstract class Lucy::Analysis::Analyzer
+ inherits Lucy::Object::Obj : dumpable {
+
+ public inert Analyzer*
+ init(Analyzer *self);
+
+ /** Take a single L<Inversion|Lucy::Analysis::Inversion> as input
+ * and returns an Inversion, either the same one (presumably transformed
+ * in some way), or a new one.
+ */
+ public abstract incremented Inversion*
+ Transform(Analyzer *self, Inversion *inversion);
+
+ /** Kick off an analysis chain, creating an Inversion from string input.
+ * The default implementation simply creates an initial Inversion with a
+ * single Token, then calls Transform(), but occasionally subclasses will
+ * provide an optimized implementation which minimizes string copies.
+ */
+ public incremented Inversion*
+ Transform_Text(Analyzer *self, CharBuf *text);
+
+ /** Analyze text and return an array of token texts.
+ */
+ public incremented VArray*
+ Split(Analyzer *self, CharBuf *text);
+}
+
+
diff --git a/core/Lucy/Analysis/CaseFolder.c b/core/Lucy/Analysis/CaseFolder.c
new file mode 100644
index 0000000..02f8361
--- /dev/null
+++ b/core/Lucy/Analysis/CaseFolder.c
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_CASEFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/CaseFolder.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+
+CaseFolder*
+CaseFolder_new() {
+ CaseFolder *self = (CaseFolder*)VTable_Make_Obj(CASEFOLDER);
+ return CaseFolder_init(self);
+}
+
+CaseFolder*
+CaseFolder_init(CaseFolder *self) {
+ Analyzer_init((Analyzer*)self);
+ self->work_buf = BB_new(0);
+ return self;
+}
+
+void
+CaseFolder_destroy(CaseFolder *self) {
+ DECREF(self->work_buf);
+ SUPER_DESTROY(self, CASEFOLDER);
+}
+
+bool_t
+CaseFolder_equals(CaseFolder *self, Obj *other) {
+ CaseFolder *const twin = (CaseFolder*)other;
+ if (twin == self) { return true; }
+ UNUSED_VAR(self);
+ if (!Obj_Is_A(other, CASEFOLDER)) { return false; }
+ return true;
+}
+
+Hash*
+CaseFolder_dump(CaseFolder *self) {
+ CaseFolder_dump_t super_dump
+ = (CaseFolder_dump_t)SUPER_METHOD(CASEFOLDER, CaseFolder, Dump);
+ return super_dump(self);
+}
+
+CaseFolder*
+CaseFolder_load(CaseFolder *self, Obj *dump) {
+ CaseFolder_load_t super_load
+ = (CaseFolder_load_t)SUPER_METHOD(CASEFOLDER, CaseFolder, Load);
+ CaseFolder *loaded = super_load(self, dump);
+ return CaseFolder_init(loaded);
+}
+
+
diff --git a/core/Lucy/Analysis/CaseFolder.cfh b/core/Lucy/Analysis/CaseFolder.cfh
new file mode 100644
index 0000000..0c5214e
--- /dev/null
+++ b/core/Lucy/Analysis/CaseFolder.cfh
@@ -0,0 +1,57 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Normalize case, facilitating case-insensitive search.
+ *
+ * CaseFolder normalizes text according to Unicode case-folding rules, so that
+ * searches will be case-insensitive.
+ */
+
+class Lucy::Analysis::CaseFolder
+ inherits Lucy::Analysis::Analyzer : dumpable {
+
+ ByteBuf *work_buf;
+
+ inert incremented CaseFolder*
+ new();
+
+ /** Constructor. Takes no arguments.
+ */
+ public inert CaseFolder*
+ init(CaseFolder *self);
+
+ public incremented Inversion*
+ Transform(CaseFolder *self, Inversion *inversion);
+
+ public incremented Inversion*
+ Transform_Text(CaseFolder *self, CharBuf *text);
+
+ public bool_t
+ Equals(CaseFolder *self, Obj *other);
+
+ public incremented Hash*
+ Dump(CaseFolder *self);
+
+ public incremented CaseFolder*
+ Load(CaseFolder *self, Obj *dump);
+
+ public void
+ Destroy(CaseFolder *self);
+}
+
+
diff --git a/core/Lucy/Analysis/Inversion.c b/core/Lucy/Analysis/Inversion.c
new file mode 100644
index 0000000..23db3a0
--- /dev/null
+++ b/core/Lucy/Analysis/Inversion.c
@@ -0,0 +1,204 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INVERSION
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Util/SortUtils.h"
+
+#ifndef SIZE_MAX
+ #define SIZE_MAX ((size_t)-1)
+#endif
+
+// After inversion, record how many like tokens occur in each group.
+static void
+S_count_clusters(Inversion *self);
+
+Inversion*
+Inversion_new(Token *seed_token) {
+ Inversion *self = (Inversion*)VTable_Make_Obj(INVERSION);
+
+ // Init.
+ self->cap = 16;
+ self->size = 0;
+ self->tokens = (Token**)CALLOCATE(self->cap, sizeof(Token*));
+ self->cur = 0;
+ self->inverted = false;
+ self->cluster_counts = NULL;
+ self->cluster_counts_size = 0;
+
+ // Process the seed token.
+ if (seed_token != NULL) {
+ Inversion_append(self, (Token*)INCREF(seed_token));
+ }
+
+ return self;
+}
+
+void
+Inversion_destroy(Inversion *self) {
+ if (self->tokens) {
+ Token **tokens = self->tokens;
+ Token **const limit = tokens + self->size;
+ for (; tokens < limit; tokens++) {
+ DECREF(*tokens);
+ }
+ FREEMEM(self->tokens);
+ }
+ FREEMEM(self->cluster_counts);
+ SUPER_DESTROY(self, INVERSION);
+}
+
+uint32_t
+Inversion_get_size(Inversion *self) {
+ return self->size;
+}
+
+Token*
+Inversion_next(Inversion *self) {
+ // Kill the iteration if we're out of tokens.
+ if (self->cur == self->size) {
+ return NULL;
+ }
+ return self->tokens[self->cur++];
+}
+
+void
+Inversion_reset(Inversion *self) {
+ self->cur = 0;
+}
+
+static void
+S_grow(Inversion *self, size_t size) {
+ if (size > self->cap) {
+ uint64_t amount = size * sizeof(Token*);
+ // Clip rather than wrap.
+ if (amount > SIZE_MAX || amount < size) { amount = SIZE_MAX; }
+ self->tokens = (Token**)REALLOCATE(self->tokens, (size_t)amount);
+ self->cap = size;
+ memset(self->tokens + self->size, 0,
+ (size - self->size) * sizeof(Token*));
+ }
+}
+
+void
+Inversion_append(Inversion *self, Token *token) {
+ if (self->inverted) {
+ THROW(ERR, "Can't append tokens after inversion");
+ }
+ if (self->size >= self->cap) {
+ size_t new_capacity = Memory_oversize(self->size + 1, sizeof(Token*));
+ S_grow(self, new_capacity);
+ }
+ self->tokens[self->size] = token;
+ self->size++;
+}
+
+Token**
+Inversion_next_cluster(Inversion *self, uint32_t *count) {
+ Token **cluster = self->tokens + self->cur;
+
+ if (self->cur == self->size) {
+ *count = 0;
+ return NULL;
+ }
+
+ // Don't read past the end of the cluster counts array.
+ if (!self->inverted) {
+ THROW(ERR, "Inversion not yet inverted");
+ }
+ if (self->cur > self->cluster_counts_size) {
+ THROW(ERR, "Tokens were added after inversion");
+ }
+
+ // Place cluster count in passed-in var, advance bookmark.
+ *count = self->cluster_counts[self->cur];
+ self->cur += *count;
+
+ return cluster;
+}
+
+void
+Inversion_invert(Inversion *self) {
+ Token **tokens = self->tokens;
+ Token **limit = tokens + self->size;
+ int32_t token_pos = 0;
+
+ // Thwart future attempts to append.
+ if (self->inverted) {
+ THROW(ERR, "Inversion has already been inverted");
+ }
+ self->inverted = true;
+
+ // Assign token positions.
+ for (; tokens < limit; tokens++) {
+ Token *const cur_token = *tokens;
+ cur_token->pos = token_pos;
+ token_pos += cur_token->pos_inc;
+ if (token_pos < cur_token->pos) {
+ THROW(ERR, "Token positions out of order: %i32 %i32",
+ cur_token->pos, token_pos);
+ }
+ }
+
+ // Sort the tokens lexically, and hand off to cluster counting routine.
+ Sort_quicksort(self->tokens, self->size, sizeof(Token*), Token_compare,
+ NULL);
+ S_count_clusters(self);
+}
+
+static void
+S_count_clusters(Inversion *self) {
+ Token **tokens = self->tokens;
+ uint32_t *counts
+ = (uint32_t*)CALLOCATE(self->size + 1, sizeof(uint32_t));
+
+ // Save the cluster counts.
+ self->cluster_counts_size = self->size;
+ self->cluster_counts = counts;
+
+ for (uint32_t i = 0; i < self->size;) {
+ Token *const base_token = tokens[i];
+ char *const base_text = base_token->text;
+ const size_t base_len = base_token->len;
+ uint32_t j = i + 1;
+
+ // Iterate through tokens until text doesn't match.
+ while (j < self->size) {
+ Token *const candidate = tokens[j];
+
+ if ((candidate->len == base_len)
+ && (memcmp(candidate->text, base_text, base_len) == 0)
+ ) {
+ j++;
+ }
+ else {
+ break;
+ }
+ }
+
+ // Record count at the position of the first token in the cluster.
+ counts[i] = j - i;
+
+ // Start the next loop at the next token we haven't seen.
+ i = j;
+ }
+}
+
+
diff --git a/core/Lucy/Analysis/Inversion.cfh b/core/Lucy/Analysis/Inversion.cfh
new file mode 100644
index 0000000..f31deaa
--- /dev/null
+++ b/core/Lucy/Analysis/Inversion.cfh
@@ -0,0 +1,81 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * A collection of Tokens.
+ *
+ * An Inversion is a collection of Token objects which you can add to, then
+ * iterate over.
+ */
+class Lucy::Analysis::Inversion inherits Lucy::Object::Obj {
+
+ Token **tokens;
+ uint32_t size;
+ uint32_t cap;
+ uint32_t cur; /* pointer to current token */
+ bool_t inverted; /* inversion has been inverted */
+ uint32_t *cluster_counts; /* counts per unique text */
+ uint32_t cluster_counts_size; /* num unique texts */
+
+ /**
+ * @param seed An initial Token to start things off, which may be NULL.
+ */
+ inert incremented Inversion*
+ new(Token *seed = NULL);
+
+ /** Tack a token onto the end of the Inversion.
+ *
+ * @param token A Token.
+ */
+ void
+ Append(Inversion *self, decremented Token *token);
+
+ /** Return the next token in the Inversion until out of tokens.
+ */
+ nullable Token*
+ Next(Inversion *self);
+
+ /** Reset the Inversion's iterator, so that the next call to next()
+ * returns the first Token in the inversion.
+ */
+ void
+ Reset(Inversion *self);
+
+ /** Assign positions to constituent Tokens, tallying up the position
+ * increments. Sort the tokens first by token text and then by position
+ * ascending.
+ */
+ void
+ Invert(Inversion *self);
+
+ /** Return a pointer to the next group of like Tokens. The number of
+ * tokens in the cluster will be placed into <code>count</code>.
+ *
+ * @param count The number of tokens in the cluster.
+ */
+ nullable Token**
+ Next_Cluster(Inversion *self, uint32_t *count);
+
+ uint32_t
+ Get_Size(Inversion *self);
+
+ public void
+ Destroy(Inversion *self);
+}
+
+
diff --git a/core/Lucy/Analysis/PolyAnalyzer.c b/core/Lucy/Analysis/PolyAnalyzer.c
new file mode 100644
index 0000000..43b049e
--- /dev/null
+++ b/core/Lucy/Analysis/PolyAnalyzer.c
@@ -0,0 +1,136 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POLYANALYZER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/PolyAnalyzer.h"
+#include "Lucy/Analysis/CaseFolder.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Analysis/SnowballStemmer.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+
+PolyAnalyzer*
+PolyAnalyzer_new(const CharBuf *language, VArray *analyzers) {
+ PolyAnalyzer *self = (PolyAnalyzer*)VTable_Make_Obj(POLYANALYZER);
+ return PolyAnalyzer_init(self, language, analyzers);
+}
+
+PolyAnalyzer*
+PolyAnalyzer_init(PolyAnalyzer *self, const CharBuf *language,
+ VArray *analyzers) {
+ Analyzer_init((Analyzer*)self);
+ if (analyzers) {
+ for (uint32_t i = 0, max = VA_Get_Size(analyzers); i < max; i++) {
+ CERTIFY(VA_Fetch(analyzers, i), ANALYZER);
+ }
+ self->analyzers = (VArray*)INCREF(analyzers);
+ }
+ else if (language) {
+ self->analyzers = VA_new(3);
+ VA_Push(self->analyzers, (Obj*)CaseFolder_new());
+ VA_Push(self->analyzers, (Obj*)RegexTokenizer_new(NULL));
+ VA_Push(self->analyzers, (Obj*)SnowStemmer_new(language));
+ }
+ else {
+ THROW(ERR, "Must specify either 'language' or 'analyzers'");
+ }
+
+ return self;
+}
+
+void
+PolyAnalyzer_destroy(PolyAnalyzer *self) {
+ DECREF(self->analyzers);
+ SUPER_DESTROY(self, POLYANALYZER);
+}
+
+VArray*
+PolyAnalyzer_get_analyzers(PolyAnalyzer *self) {
+ return self->analyzers;
+}
+
+Inversion*
+PolyAnalyzer_transform(PolyAnalyzer *self, Inversion *inversion) {
+ VArray *const analyzers = self->analyzers;
+ (void)INCREF(inversion);
+
+ // Iterate through each of the analyzers in order.
+ for (uint32_t i = 0, max = VA_Get_Size(analyzers); i < max; i++) {
+ Analyzer *analyzer = (Analyzer*)VA_Fetch(analyzers, i);
+ Inversion *new_inversion = Analyzer_Transform(analyzer, inversion);
+ DECREF(inversion);
+ inversion = new_inversion;
+ }
+
+ return inversion;
+}
+
+Inversion*
+PolyAnalyzer_transform_text(PolyAnalyzer *self, CharBuf *text) {
+ VArray *const analyzers = self->analyzers;
+ const uint32_t num_analyzers = VA_Get_Size(analyzers);
+ Inversion *retval;
+
+ if (num_analyzers == 0) {
+ size_t token_len = CB_Get_Size(text);
+ char *buf = (char*)CB_Get_Ptr8(text);
+ Token *seed = Token_new(buf, token_len, 0, token_len, 1.0f, 1);
+ retval = Inversion_new(seed);
+ DECREF(seed);
+ }
+ else {
+ Analyzer *first_analyzer = (Analyzer*)VA_Fetch(analyzers, 0);
+ retval = Analyzer_Transform_Text(first_analyzer, text);
+ for (uint32_t i = 1; i < num_analyzers; i++) {
+ Analyzer *analyzer = (Analyzer*)VA_Fetch(analyzers, i);
+ Inversion *new_inversion = Analyzer_Transform(analyzer, retval);
+ DECREF(retval);
+ retval = new_inversion;
+ }
+ }
+
+ return retval;
+}
+
+bool_t
+PolyAnalyzer_equals(PolyAnalyzer *self, Obj *other) {
+ PolyAnalyzer *const twin = (PolyAnalyzer*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, POLYANALYZER)) { return false; }
+ if (!VA_Equals(twin->analyzers, (Obj*)self->analyzers)) { return false; }
+ return true;
+}
+
+PolyAnalyzer*
+PolyAnalyzer_load(PolyAnalyzer *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ PolyAnalyzer_load_t super_load = (PolyAnalyzer_load_t)SUPER_METHOD(
+ POLYANALYZER, PolyAnalyzer, Load);
+ PolyAnalyzer *loaded = super_load(self, dump);
+ VArray *analyzer_dumps = (VArray*)CERTIFY(
+ Hash_Fetch_Str(source, "analyzers", 9),
+ VARRAY);
+ VArray *analyzers = (VArray*)CERTIFY(
+ VA_Load(analyzer_dumps, (Obj*)analyzer_dumps),
+ VARRAY);
+ PolyAnalyzer_init(loaded, NULL, analyzers);
+ DECREF(analyzers);
+ return loaded;
+}
+
+
diff --git a/core/Lucy/Analysis/PolyAnalyzer.cfh b/core/Lucy/Analysis/PolyAnalyzer.cfh
new file mode 100644
index 0000000..2e01238
--- /dev/null
+++ b/core/Lucy/Analysis/PolyAnalyzer.cfh
@@ -0,0 +1,89 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Multiple Analyzers in series.
+ *
+ * A PolyAnalyzer is a series of L<Analyzers|Lucy::Analysis::Analyzer>,
+ * each of which will be called upon to "analyze" text in turn. You can
+ * either provide the Analyzers yourself, or you can specify a supported
+ * language, in which case a PolyAnalyzer consisting of a
+ * L<CaseFolder|Lucy::Analysis::CaseFolder>, a
+ * L<RegexTokenizer|Lucy::Analysis::RegexTokenizer>, and a
+ * L<SnowballStemmer|Lucy::Analysis::SnowballStemmer> will be generated for you.
+ *
+ * Supported languages:
+ *
+ * en => English,
+ * da => Danish,
+ * de => German,
+ * es => Spanish,
+ * fi => Finnish,
+ * fr => French,
+ * hu => Hungarian,
+ * it => Italian,
+ * nl => Dutch,
+ * no => Norwegian,
+ * pt => Portuguese,
+ * ro => Romanian,
+ * ru => Russian,
+ * sv => Swedish,
+ * tr => Turkish,
+ */
+class Lucy::Analysis::PolyAnalyzer
+ inherits Lucy::Analysis::Analyzer : dumpable {
+
+ VArray *analyzers;
+
+ inert incremented PolyAnalyzer*
+ new(const CharBuf *language = NULL, VArray *analyzers = NULL);
+
+ /**
+ * @param language An ISO code from the list of supported languages.
+ * @param analyzers An array of Analyzers. The order of the analyzers
+ * matters. Don't put a SnowballStemmer before a RegexTokenizer (can't stem whole
+ * documents or paragraphs -- just individual words), or a SnowballStopFilter
+ * after a SnowballStemmer (stemmed words, e.g. "themselv", will not appear in a
+ * stoplist). In general, the sequence should be: normalize, tokenize,
+ * stopalize, stem.
+ */
+ public inert PolyAnalyzer*
+ init(PolyAnalyzer *self, const CharBuf *language = NULL,
+ VArray *analyzers = NULL);
+
+ /** Getter for "analyzers" member.
+ */
+ public VArray*
+ Get_Analyzers(PolyAnalyzer *self);
+
+ public incremented Inversion*
+ Transform(PolyAnalyzer *self, Inversion *inversion);
+
+ public incremented Inversion*
+ Transform_Text(PolyAnalyzer *self, CharBuf *text);
+
+ public bool_t
+ Equals(PolyAnalyzer *self, Obj *other);
+
+ public incremented PolyAnalyzer*
+ Load(PolyAnalyzer *self, Obj *dump);
+
+ public void
+ Destroy(PolyAnalyzer *self);
+}
+
+
diff --git a/core/Lucy/Analysis/RegexTokenizer.c b/core/Lucy/Analysis/RegexTokenizer.c
new file mode 100644
index 0000000..ae222b1
--- /dev/null
+++ b/core/Lucy/Analysis/RegexTokenizer.c
@@ -0,0 +1,81 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_REGEXTOKENIZER
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/RegexTokenizer.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+
+RegexTokenizer*
+RegexTokenizer_new(const CharBuf *pattern) {
+ RegexTokenizer *self = (RegexTokenizer*)VTable_Make_Obj(REGEXTOKENIZER);
+ return RegexTokenizer_init(self, pattern);
+}
+
+Inversion*
+RegexTokenizer_transform(RegexTokenizer *self, Inversion *inversion) {
+ Inversion *new_inversion = Inversion_new(NULL);
+ Token *token;
+
+ while (NULL != (token = Inversion_Next(inversion))) {
+ RegexTokenizer_Tokenize_Str(self, token->text, token->len,
+ new_inversion);
+ }
+
+ return new_inversion;
+}
+
+Inversion*
+RegexTokenizer_transform_text(RegexTokenizer *self, CharBuf *text) {
+ Inversion *new_inversion = Inversion_new(NULL);
+ RegexTokenizer_Tokenize_Str(self, (char*)CB_Get_Ptr8(text),
+ CB_Get_Size(text), new_inversion);
+ return new_inversion;
+}
+
+Obj*
+RegexTokenizer_dump(RegexTokenizer *self) {
+ RegexTokenizer_dump_t super_dump
+ = (RegexTokenizer_dump_t)SUPER_METHOD(REGEXTOKENIZER, RegexTokenizer, Dump);
+ Hash *dump = (Hash*)CERTIFY(super_dump(self), HASH);
+ Hash_Store_Str(dump, "pattern", 7, CB_Dump(self->pattern));
+ return (Obj*)dump;
+}
+
+RegexTokenizer*
+RegexTokenizer_load(RegexTokenizer *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ RegexTokenizer_load_t super_load
+ = (RegexTokenizer_load_t)SUPER_METHOD(REGEXTOKENIZER, RegexTokenizer, Load);
+ RegexTokenizer *loaded = super_load(self, dump);
+ CharBuf *pattern
+ = (CharBuf*)CERTIFY(Hash_Fetch_Str(source, "pattern", 7), CHARBUF);
+ return RegexTokenizer_init(loaded, pattern);
+}
+
+bool_t
+RegexTokenizer_equals(RegexTokenizer *self, Obj *other) {
+ RegexTokenizer *const twin = (RegexTokenizer*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, REGEXTOKENIZER)) { return false; }
+ if (!CB_Equals(twin->pattern, (Obj*)self->pattern)) { return false; }
+ return true;
+}
+
+
diff --git a/core/Lucy/Analysis/RegexTokenizer.cfh b/core/Lucy/Analysis/RegexTokenizer.cfh
new file mode 100644
index 0000000..5531836
--- /dev/null
+++ b/core/Lucy/Analysis/RegexTokenizer.cfh
@@ -0,0 +1,100 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Split a string into tokens.
+ *
+ * Generically, "tokenizing" is a process of breaking up a string into an
+ * array of "tokens". For instance, the string "three blind mice" might be
+ * tokenized into "three", "blind", "mice".
+ *
+ * Lucy::Analysis::RegexTokenizer decides where it should break up the text
+ * based on a regular expression compiled from a supplied <code>pattern</code>
+ * matching one token. If our source string is...
+ *
+ * "Eats, Shoots and Leaves."
+ *
+ * ... then a "whitespace tokenizer" with a <code>pattern</code> of
+ * <code>"\\S+"</code> produces...
+ *
+ * Eats,
+ * Shoots
+ * and
+ * Leaves.
+ *
+ * ... while a "word character tokenizer" with a <code>pattern</code> of
+ * <code>"\\w+"</code> produces...
+ *
+ * Eats
+ * Shoots
+ * and
+ * Leaves
+ *
+ * ... the difference being that the word character tokenizer skips over
+ * punctuation as well as whitespace when determining token boundaries.
+ */
+class Lucy::Analysis::RegexTokenizer
+ inherits Lucy::Analysis::Analyzer {
+
+ CharBuf *pattern;
+ void *token_re;
+
+ inert incremented RegexTokenizer*
+ new(const CharBuf *pattern = NULL);
+
+ /**
+ * @param pattern A string specifying a Perl-syntax regular expression
+ * which should match one token. The default value is
+ * <code>\w+(?:[\x{2019}']\w+)*</code>, which matches "it's" as well as
+ * "it" and "O'Henry's" as well as "Henry".
+ */
+ public inert RegexTokenizer*
+ init(RegexTokenizer *self, const CharBuf *pattern = NULL);
+
+ public incremented Inversion*
+ Transform(RegexTokenizer *self, Inversion *inversion);
+
+ public incremented Inversion*
+ Transform_Text(RegexTokenizer *self, CharBuf *text);
+
+ /** Tokenize the supplied string and add any Tokens generated to the
+ * supplied Inversion.
+ */
+ void
+ Tokenize_Str(RegexTokenizer *self, const char *text, size_t len,
+ Inversion *inversion);
+
+ /** Set the compiled regular expression for matching a token. Also sets
+ * <code>pattern</code> as a side effect.
+ */
+ void
+ Set_Token_RE(RegexTokenizer *self, void *token_re);
+
+ public incremented Obj*
+ Dump(RegexTokenizer *self);
+
+ public incremented RegexTokenizer*
+ Load(RegexTokenizer *self, Obj *dump);
+
+ public bool_t
+ Equals(RegexTokenizer *self, Obj *other);
+
+ public void
+ Destroy(RegexTokenizer *self);
+}
+
+
diff --git a/core/Lucy/Analysis/SnowballStemmer.c b/core/Lucy/Analysis/SnowballStemmer.c
new file mode 100644
index 0000000..1cb6304
--- /dev/null
+++ b/core/Lucy/Analysis/SnowballStemmer.c
@@ -0,0 +1,111 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SNOWBALLSTEMMER
+#define C_LUCY_TOKEN
+#include <ctype.h>
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/SnowballStemmer.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+
+#include "libstemmer.h"
+
+SnowballStemmer*
+SnowStemmer_new(const CharBuf *language) {
+ SnowballStemmer *self = (SnowballStemmer*)VTable_Make_Obj(SNOWBALLSTEMMER);
+ return SnowStemmer_init(self, language);
+}
+
+SnowballStemmer*
+SnowStemmer_init(SnowballStemmer *self, const CharBuf *language) {
+ char lang_buf[3];
+ Analyzer_init((Analyzer*)self);
+ self->language = CB_Clone(language);
+
+ // Get a Snowball stemmer. Be case-insensitive.
+ lang_buf[0] = tolower(CB_Code_Point_At(language, 0));
+ lang_buf[1] = tolower(CB_Code_Point_At(language, 1));
+ lang_buf[2] = '\0';
+ self->snowstemmer = sb_stemmer_new(lang_buf, "UTF_8");
+ if (!self->snowstemmer) {
+ THROW(ERR, "Can't find a Snowball stemmer for %o", language);
+ }
+
+ return self;
+}
+
+void
+SnowStemmer_destroy(SnowballStemmer *self) {
+ if (self->snowstemmer) {
+ sb_stemmer_delete((struct sb_stemmer*)self->snowstemmer);
+ }
+ DECREF(self->language);
+ SUPER_DESTROY(self, SNOWBALLSTEMMER);
+}
+
+Inversion*
+SnowStemmer_transform(SnowballStemmer *self, Inversion *inversion) {
+ Token *token;
+ struct sb_stemmer *const snowstemmer
+ = (struct sb_stemmer*)self->snowstemmer;
+
+ while (NULL != (token = Inversion_Next(inversion))) {
+ const sb_symbol *stemmed_text
+ = sb_stemmer_stem(snowstemmer, (sb_symbol*)token->text, token->len);
+ size_t len = sb_stemmer_length(snowstemmer);
+ if (len > token->len) {
+ FREEMEM(token->text);
+ token->text = (char*)MALLOCATE(len + 1);
+ }
+ memcpy(token->text, stemmed_text, len + 1);
+ token->len = len;
+ }
+ Inversion_Reset(inversion);
+ return (Inversion*)INCREF(inversion);
+}
+
+Hash*
+SnowStemmer_dump(SnowballStemmer *self) {
+ SnowStemmer_dump_t super_dump
+ = (SnowStemmer_dump_t)SUPER_METHOD(SNOWBALLSTEMMER, SnowStemmer, Dump);
+ Hash *dump = super_dump(self);
+ Hash_Store_Str(dump, "language", 8, (Obj*)CB_Clone(self->language));
+ return dump;
+}
+
+SnowballStemmer*
+SnowStemmer_load(SnowballStemmer *self, Obj *dump) {
+ SnowStemmer_load_t super_load
+ = (SnowStemmer_load_t)SUPER_METHOD(SNOWBALLSTEMMER, SnowStemmer, Load);
+ SnowballStemmer *loaded = super_load(self, dump);
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *language
+ = (CharBuf*)CERTIFY(Hash_Fetch_Str(source, "language", 8), CHARBUF);
+ return SnowStemmer_init(loaded, language);
+}
+
+bool_t
+SnowStemmer_equals(SnowballStemmer *self, Obj *other) {
+ SnowballStemmer *const twin = (SnowballStemmer*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, SNOWBALLSTEMMER)) { return false; }
+ if (!CB_Equals(twin->language, (Obj*)self->language)) { return false; }
+ return true;
+}
+
+
diff --git a/core/Lucy/Analysis/SnowballStemmer.cfh b/core/Lucy/Analysis/SnowballStemmer.cfh
new file mode 100644
index 0000000..9a173d6
--- /dev/null
+++ b/core/Lucy/Analysis/SnowballStemmer.cfh
@@ -0,0 +1,60 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Reduce related words to a shared root.
+ *
+ * SnowballStemmer is an L<Analyzer|Lucy::Analysis::Analyzer> which reduces
+ * related words to a root form (using the "Snowball" stemming library). For
+ * instance, "horse", "horses", and "horsing" all become "hors" -- so that a
+ * search for 'horse' will also match documents containing 'horses' and
+ * 'horsing'.
+ */
+
+class Lucy::Analysis::SnowballStemmer cnick SnowStemmer
+ inherits Lucy::Analysis::Analyzer : dumpable {
+
+ void *snowstemmer;
+ CharBuf *language;
+
+ inert incremented SnowballStemmer*
+ new(const CharBuf *language);
+
+ /**
+ * @param language A two-letter ISO code identifying a language supported
+ * by Snowball.
+ */
+ public inert SnowballStemmer*
+ init(SnowballStemmer *self, const CharBuf *language);
+
+ public incremented Inversion*
+ Transform(SnowballStemmer *self, Inversion *inversion);
+
+ public incremented Hash*
+ Dump(SnowballStemmer *self);
+
+ public incremented SnowballStemmer*
+ Load(SnowballStemmer *self, Obj *dump);
+
+ public bool_t
+ Equals(SnowballStemmer *self, Obj *other);
+
+ public void
+ Destroy(SnowballStemmer *self);
+}
+
+
diff --git a/core/Lucy/Analysis/SnowballStopFilter.c b/core/Lucy/Analysis/SnowballStopFilter.c
new file mode 100644
index 0000000..30c24b0
--- /dev/null
+++ b/core/Lucy/Analysis/SnowballStopFilter.c
@@ -0,0 +1,141 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SNOWBALLSTOPFILTER
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+#include <ctype.h>
+
+#include "Lucy/Analysis/SnowballStopFilter.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+
+SnowballStopFilter*
+SnowStop_new(const CharBuf *language, Hash *stoplist) {
+ SnowballStopFilter *self = (SnowballStopFilter*)VTable_Make_Obj(SNOWBALLSTOPFILTER);
+ return SnowStop_init(self, language, stoplist);
+}
+
+SnowballStopFilter*
+SnowStop_init(SnowballStopFilter *self, const CharBuf *language,
+ Hash *stoplist) {
+ Analyzer_init((Analyzer*)self);
+
+ if (stoplist) {
+ if (language) { THROW(ERR, "Can't have both stoplist and language"); }
+ self->stoplist = (Hash*)INCREF(stoplist);
+ }
+ else if (language) {
+ self->stoplist = SnowStop_gen_stoplist(language);
+ if (!self->stoplist) {
+ THROW(ERR, "Can't get a stoplist for '%o'", language);
+ }
+ }
+ else {
+ THROW(ERR, "Either stoplist or language is required");
+ }
+
+ return self;
+}
+
+void
+SnowStop_destroy(SnowballStopFilter *self) {
+ DECREF(self->stoplist);
+ SUPER_DESTROY(self, SNOWBALLSTOPFILTER);
+}
+
+Inversion*
+SnowStop_transform(SnowballStopFilter *self, Inversion *inversion) {
+ Token *token;
+ Inversion *new_inversion = Inversion_new(NULL);
+ Hash *const stoplist = self->stoplist;
+
+ while (NULL != (token = Inversion_Next(inversion))) {
+ if (!Hash_Fetch_Str(stoplist, token->text, token->len)) {
+ Inversion_Append(new_inversion, (Token*)INCREF(token));
+ }
+ }
+
+ return new_inversion;
+}
+
+bool_t
+SnowStop_equals(SnowballStopFilter *self, Obj *other) {
+ SnowballStopFilter *const twin = (SnowballStopFilter*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, SNOWBALLSTOPFILTER)) { return false; }
+ if (!Hash_Equals(twin->stoplist, (Obj*)self->stoplist)) {
+ return false;
+ }
+ return true;
+}
+
+Hash*
+SnowStop_gen_stoplist(const CharBuf *language) {
+ CharBuf *lang = CB_new(3);
+ CB_Cat_Char(lang, tolower(CB_Code_Point_At(language, 0)));
+ CB_Cat_Char(lang, tolower(CB_Code_Point_At(language, 1)));
+ const uint8_t **words = NULL;
+ if (CB_Equals_Str(lang, "da", 2)) { words = SnowStop_snow_da; }
+ else if (CB_Equals_Str(lang, "de", 2)) { words = SnowStop_snow_de; }
+ else if (CB_Equals_Str(lang, "en", 2)) { words = SnowStop_snow_en; }
+ else if (CB_Equals_Str(lang, "es", 2)) { words = SnowStop_snow_es; }
+ else if (CB_Equals_Str(lang, "fi", 2)) { words = SnowStop_snow_fi; }
+ else if (CB_Equals_Str(lang, "fr", 2)) { words = SnowStop_snow_fr; }
+ else if (CB_Equals_Str(lang, "hu", 2)) { words = SnowStop_snow_hu; }
+ else if (CB_Equals_Str(lang, "it", 2)) { words = SnowStop_snow_it; }
+ else if (CB_Equals_Str(lang, "nl", 2)) { words = SnowStop_snow_nl; }
+ else if (CB_Equals_Str(lang, "no", 2)) { words = SnowStop_snow_no; }
+ else if (CB_Equals_Str(lang, "pt", 2)) { words = SnowStop_snow_pt; }
+ else if (CB_Equals_Str(lang, "ru", 2)) { words = SnowStop_snow_ru; }
+ else if (CB_Equals_Str(lang, "sv", 2)) { words = SnowStop_snow_sv; }
+ else {
+ DECREF(lang);
+ return NULL;
+ }
+ size_t num_stopwords = 0;
+ for (uint32_t i = 0; words[i] != NULL; i++) { num_stopwords++; }
+ NoCloneHash *stoplist = NoCloneHash_new(num_stopwords);
+ for (uint32_t i = 0; words[i] != NULL; i++) {
+ char *word = (char*)words[i];
+ ViewCharBuf *stop = ViewCB_new_from_trusted_utf8(word, strlen(word));
+ NoCloneHash_Store(stoplist, (Obj*)stop, INCREF(&EMPTY));
+ DECREF(stop);
+ }
+ DECREF(lang);
+ return (Hash*)stoplist;
+}
+
+/***************************************************************************/
+
+NoCloneHash*
+NoCloneHash_new(uint32_t capacity) {
+ NoCloneHash *self = (NoCloneHash*)VTable_Make_Obj(NOCLONEHASH);
+ return NoCloneHash_init(self, capacity);
+}
+
+NoCloneHash*
+NoCloneHash_init(NoCloneHash *self, uint32_t capacity) {
+ return (NoCloneHash*)Hash_init((Hash*)self, capacity);
+}
+
+Obj*
+NoCloneHash_make_key(NoCloneHash *self, Obj *key, int32_t hash_sum) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(hash_sum);
+ return INCREF(key);
+}
+
diff --git a/core/Lucy/Analysis/SnowballStopFilter.cfh b/core/Lucy/Analysis/SnowballStopFilter.cfh
new file mode 100644
index 0000000..077f8e2
--- /dev/null
+++ b/core/Lucy/Analysis/SnowballStopFilter.cfh
@@ -0,0 +1,112 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Suppress a "stoplist" of common words.
+ *
+ * A "stoplist" is collection of "stopwords": words which are common enough to
+ * be of little value when determining search results. For example, so many
+ * documents in English contain "the", "if", and "maybe" that it may improve
+ * both performance and relevance to block them.
+ *
+ * Before filtering stopwords:
+ *
+ * ("i", "am", "the", "walrus")
+ *
+ * After filtering stopwords:
+ *
+ * ("walrus")
+ *
+ * SnowballStopFilter provides default stoplists for several languages, courtesy of
+ * the Snowball project (<http://snowball.tartarus.org>), or you may supply
+ * your own.
+ *
+ * |-----------------------|
+ * | ISO CODE | LANGUAGE |
+ * |-----------------------|
+ * | da | Danish |
+ * | de | German |
+ * | en | English |
+ * | es | Spanish |
+ * | fi | Finnish |
+ * | fr | French |
+ * | hu | Hungarian |
+ * | it | Italian |
+ * | nl | Dutch |
+ * | no | Norwegian |
+ * | pt | Portuguese |
+ * | sv | Swedish |
+ * | ru | Russian |
+ * |-----------------------|
+ */
+
+class Lucy::Analysis::SnowballStopFilter cnick SnowStop
+ inherits Lucy::Analysis::Analyzer : dumpable {
+
+ Hash *stoplist;
+
+ inert const uint8_t** snow_da;
+ inert const uint8_t** snow_de;
+ inert const uint8_t** snow_en;
+ inert const uint8_t** snow_es;
+ inert const uint8_t** snow_fi;
+ inert const uint8_t** snow_fr;
+ inert const uint8_t** snow_hu;
+ inert const uint8_t** snow_it;
+ inert const uint8_t** snow_nl;
+ inert const uint8_t** snow_no;
+ inert const uint8_t** snow_pt;
+ inert const uint8_t** snow_ru;
+ inert const uint8_t** snow_sv;
+
+ inert incremented SnowballStopFilter*
+ new(const CharBuf *language = NULL, Hash *stoplist = NULL);
+
+ /**
+ * @param stoplist A hash with stopwords as the keys.
+ * @param language The ISO code for a supported language.
+ */
+ public inert SnowballStopFilter*
+ init(SnowballStopFilter *self, const CharBuf *language = NULL,
+ Hash *stoplist = NULL);
+
+ /** Return a Hash with the Snowball stoplist for the supplied language.
+ */
+ inert incremented Hash*
+ gen_stoplist(const CharBuf *language);
+
+ public incremented Inversion*
+ Transform(SnowballStopFilter *self, Inversion *inversion);
+
+ public bool_t
+ Equals(SnowballStopFilter *self, Obj *other);
+
+ public void
+ Destroy(SnowballStopFilter *self);
+}
+
+class Lucy::Analysis::SnowballStopFilter::NoCloneHash inherits Lucy::Object::Hash {
+ inert incremented NoCloneHash*
+ new(uint32_t capacity = 0);
+
+ inert NoCloneHash*
+ init(NoCloneHash *self, uint32_t capacity = 0);
+
+ public incremented Obj*
+ Make_Key(NoCloneHash *self, Obj *key, int32_t hash_sum);
+}
+
diff --git a/core/Lucy/Analysis/Token.c b/core/Lucy/Analysis/Token.c
new file mode 100644
index 0000000..5b53043
--- /dev/null
+++ b/core/Lucy/Analysis/Token.c
@@ -0,0 +1,118 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Analysis/Token.h"
+
+Token*
+Token_new(const char* text, size_t len, uint32_t start_offset,
+ uint32_t end_offset, float boost, int32_t pos_inc) {
+ Token *self = (Token*)VTable_Make_Obj(TOKEN);
+ return Token_init(self, text, len, start_offset, end_offset, boost,
+ pos_inc);
+}
+
+Token*
+Token_init(Token *self, const char* text, size_t len, uint32_t start_offset,
+ uint32_t end_offset, float boost, int32_t pos_inc) {
+ // Allocate and assign.
+ self->text = (char*)MALLOCATE(len + 1);
+ self->text[len] = '\0';
+ memcpy(self->text, text, len);
+
+ // Assign.
+ self->len = len;
+ self->start_offset = start_offset;
+ self->end_offset = end_offset;
+ self->boost = boost;
+ self->pos_inc = pos_inc;
+
+ // Init.
+ self->pos = -1;
+
+ return self;
+}
+
+void
+Token_destroy(Token *self) {
+ FREEMEM(self->text);
+ SUPER_DESTROY(self, TOKEN);
+}
+
+int
+Token_compare(void *context, const void *va, const void *vb) {
+ Token *const a = *((Token**)va);
+ Token *const b = *((Token**)vb);
+ size_t min_len = a->len < b->len ? a->len : b->len;
+ int comparison = memcmp(a->text, b->text, min_len);
+ UNUSED_VAR(context);
+
+ if (comparison == 0) {
+ if (a->len != b->len) {
+ comparison = a->len < b->len ? -1 : 1;
+ }
+ else {
+ comparison = a->pos < b->pos ? -1 : 1;
+ }
+ }
+
+ return comparison;
+}
+
+uint32_t
+Token_get_start_offset(Token *self) {
+ return self->start_offset;
+}
+
+uint32_t
+Token_get_end_offset(Token *self) {
+ return self->end_offset;
+}
+
+float
+Token_get_boost(Token *self) {
+ return self->boost;
+}
+
+int32_t
+Token_get_pos_inc(Token *self) {
+ return self->pos_inc;
+}
+
+char*
+Token_get_text(Token *self) {
+ return self->text;
+}
+
+size_t
+Token_get_len(Token *self) {
+ return self->len;
+}
+
+void
+Token_set_text(Token *self, char *text, size_t len) {
+ if (len > self->len) {
+ FREEMEM(self->text);
+ self->text = (char*)MALLOCATE(len + 1);
+ }
+ memcpy(self->text, text, len);
+ self->text[len] = '\0';
+ self->len = len;
+}
+
+
diff --git a/core/Lucy/Analysis/Token.cfh b/core/Lucy/Analysis/Token.cfh
new file mode 100644
index 0000000..eb2a2e5
--- /dev/null
+++ b/core/Lucy/Analysis/Token.cfh
@@ -0,0 +1,101 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Unit of text.
+ *
+ * Token is the fundamental unit used by Apache Lucy's Analyzer subclasses.
+ * Each Token has 5 attributes: <code>text</code>, <code>start_offset</code>,
+ * <code>end_offset</code>, <code>boost</code>, and <code>pos_inc</code>.
+ *
+ * The <code>text</code> attribute is a Unicode string encoded as UTF-8.
+ *
+ * <code>start_offset</code> is the start point of the token text, measured in
+ * Unicode code points from the top of the stored field;
+ * <code>end_offset</code> delimits the corresponding closing boundary.
+ * <code>start_offset</code> and <code>end_offset</code> locate the Token
+ * within a larger context, even if the Token's text attribute gets modified
+ * -- by stemming, for instance. The Token for "beating" in the text "beating
+ * a dead horse" begins life with a start_offset of 0 and an end_offset of 7;
+ * after stemming, the text is "beat", but the start_offset is still 0 and the
+ * end_offset is still 7. This allows "beating" to be highlighted correctly
+ * after a search matches "beat".
+ *
+ * <code>boost</code> is a per-token weight. Use this when you want to assign
+ * more or less importance to a particular token, as you might for emboldened
+ * text within an HTML document, for example. (Note: The field this token
+ * belongs to must be spec'd to use a posting of type
+ * L<Lucy::Index::Posting::RichPosting>.)
+ *
+ * <code>pos_inc</code is the POSition INCrement, measured in Tokens. This
+ * attribute, which defaults to 1, is a an advanced tool for manipulating
+ * phrase matching. Ordinarily, Tokens are assigned consecutive position
+ * numbers: 0, 1, and 2 for <code>"three blind mice"</code>. However, if you
+ * set the position increment for "blind" to, say, 1000, then the three tokens
+ * will end up assigned to positions 0, 1, and 1001 -- and will no longer
+ * produce a phrase match for the query <code>"three blind mice"</code>.
+ */
+class Lucy::Analysis::Token inherits Lucy::Object::Obj {
+
+ char *text;
+ size_t len;
+ uint32_t start_offset;
+ uint32_t end_offset;
+ float boost;
+ int32_t pos_inc;
+ int32_t pos;
+
+ inert incremented Token*
+ new(const char *text, size_t len, uint32_t start_offset,
+ uint32_t end_offset, float boost = 1.0, int32_t pos_inc = 1);
+
+ inert Token*
+ init(Token *self, const char *text, size_t len,
+ uint32_t start_offset, uint32_t end_offset,
+ float boost = 1.0, int32_t pos_inc = 1);
+
+ /** Sort_quicksort-compatible comparison routine.
+ */
+ inert int
+ compare(void *context, const void *va, const void *vb);
+
+ uint32_t
+ Get_Start_Offset(Token *self);
+
+ uint32_t
+ Get_End_Offset(Token *self);
+
+ float
+ Get_Boost(Token *self);
+
+ int32_t
+ Get_Pos_Inc(Token *self);
+
+ char*
+ Get_Text(Token *self);
+
+ size_t
+ Get_Len(Token *self);
+
+ void
+ Set_Text(Token *self, char *text, size_t len);
+
+ public void
+ Destroy(Token *self);
+}
+
+
diff --git a/core/Lucy/Docs/DevGuide.cfh b/core/Lucy/Docs/DevGuide.cfh
new file mode 100644
index 0000000..dfab5ff
--- /dev/null
+++ b/core/Lucy/Docs/DevGuide.cfh
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Quick-start guide to hacking on Apache Lucy.
+ *
+ * The Apache Lucy code base is organized into roughly four layers:
+ *
+ * * Charmonizer - compiler and OS configuration probing.
+ * * Clownfish - header files.
+ * * C - implementation files.
+ * * Host - binding language.
+ *
+ * Charmonizer is a configuration prober which writes a single header file,
+ * "charmony.h", describing the build environment and facilitating
+ * cross-platform development. It's similar to Autoconf or Metaconfig, but
+ * written in pure C.
+ *
+ * The ".cfh" files within the Lucy core are Clownfish header files.
+ * Clownfish is a purpose-built, declaration-only language which superimposes
+ * a single-inheritance object model on top of C which is specifically
+ * designed to co-exist happily with variety of "host" languages and to allow
+ * limited run-time dynamic subclassing. For more information see the
+ * Clownfish docs, but if there's one thing you should know about Clownfish OO
+ * before you start hacking, it's that method calls are differentiated from
+ * functions by capitalization:
+ *
+ * Indexer_Add_Doc <-- Method, typically uses dynamic dispatch.
+ * Indexer_add_doc <-- Function, always a direct invocation.
+ *
+ * The C files within the Lucy core are where most of Lucy's low-level
+ * functionality lies. They implement the interface defined by the Clownfish
+ * header files.
+ *
+ * The C core is intentionally left incomplete, however; to be usable, it must
+ * be bound to a "host" language. (In this context, even C is considered a
+ * "host" which must implement the missing pieces and be "bound" to the core.)
+ * Some of the binding code is autogenerated by Clownfish on a spec customized
+ * for each language. Other pieces are hand-coded in either C (using the
+ * host's C API) or the host language itself.
+ */
+
+inert class Lucy::Docs::DevGuide { }
+
+
diff --git a/core/Lucy/Docs/FileLocking.cfh b/core/Lucy/Docs/FileLocking.cfh
new file mode 100644
index 0000000..4e96309
--- /dev/null
+++ b/core/Lucy/Docs/FileLocking.cfh
@@ -0,0 +1,83 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Manage indexes on shared volumes.
+ *
+ * Normally, index locking is an invisible process. Exclusive write access is
+ * controlled via lockfiles within the index directory and problems only arise
+ * if multiple processes attempt to acquire the write lock simultaneously;
+ * search-time processes do not ordinarily require locking at all.
+ *
+ * On shared volumes, however, the default locking mechanism fails, and manual
+ * intervention becomes necessary.
+ *
+ * Both read and write applications accessing an index on a shared volume need
+ * to identify themselves with a unique <code>host</code> id, e.g. hostname or
+ * ip address. Knowing the host id makes it possible to tell which lockfiles
+ * belong to other machines and therefore must not be removed when the
+ * lockfile's pid number appears not to correspond to an active process.
+ *
+ * At index-time, the danger is that multiple indexing processes from
+ * different machines which fail to specify a unique <code>host</code> id can
+ * delete each others' lockfiles and then attempt to modify the index at the
+ * same time, causing index corruption. The search-time problem is more
+ * complex.
+ *
+ * Once an index file is no longer listed in the most recent snapshot, Indexer
+ * attempts to delete it as part of a post-Commit() cleanup routine. It is
+ * possible that at the moment an Indexer is deleting files which it believes
+ * no longer needed, a Searcher referencing an earlier snapshot is in fact
+ * using them. The more often that an index is either updated or searched,
+ * the more likely it is that this conflict will arise from time to time.
+ *
+ * Ordinarily, the deletion attempts are not a problem. On a typical unix
+ * volume, the files will be deleted in name only: any process which holds an
+ * open filehandle against a given file will continue to have access, and the
+ * file won't actually get vaporized until the last filehandle is cleared.
+ * Thanks to "delete on last close semantics", an Indexer can't truly delete
+ * the file out from underneath an active Searcher. On Windows, where file
+ * deletion fails whenever any process holds an open handle, the situation is
+ * different but still workable: Indexer just keeps retrying after each commit
+ * until deletion finally succeeds.
+ *
+ * On NFS, however, the system breaks, because NFS allows files to be deleted
+ * out from underneath active processes. Should this happen, the unlucky read
+ * process will crash with a "Stale NFS filehandle" exception.
+ *
+ * Under normal circumstances, it is neither necessary nor desirable for
+ * IndexReaders to secure read locks against an index, but for NFS we have to
+ * make an exception. LockFactory's Make_Shared_Lock() method exists for this
+ * reason; supplying an IndexManager instance to IndexReader's constructor
+ * activates an internal locking mechanism using Make_Shared_Lock() which
+ * prevents concurrent indexing processes from deleting files that are needed
+ * by active readers.
+ *
+ * Since shared locks are implemented using lockfiles located in the index
+ * directory (as are exclusive locks), reader applications must have write
+ * access for read locking to work. Stale lock files from crashed processes
+ * are ordinarily cleared away the next time the same machine -- as identified
+ * by the <code>host</code> parameter -- opens another IndexReader. (The
+ * classic technique of timing out lock files is not feasible because search
+ * processes may lie dormant indefinitely.) However, please be aware that if
+ * the last thing a given machine does is crash, lock files belonging to it
+ * may persist, preventing deletion of obsolete index data.
+ */
+
+inert class Lucy::Docs::FileLocking { }
+
+
diff --git a/core/Lucy/Document/Doc.c b/core/Lucy/Document/Doc.c
new file mode 100644
index 0000000..9c6e6e5
--- /dev/null
+++ b/core/Lucy/Document/Doc.c
@@ -0,0 +1,43 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DOC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Document/Doc.h"
+
+Doc*
+Doc_new(void *fields, int32_t doc_id) {
+ Doc *self = (Doc*)VTable_Make_Obj(DOC);
+ return Doc_init(self, fields, doc_id);
+}
+
+void
+Doc_set_doc_id(Doc *self, int32_t doc_id) {
+ self->doc_id = doc_id;
+}
+
+int32_t
+Doc_get_doc_id(Doc *self) {
+ return self->doc_id;
+}
+
+void*
+Doc_get_fields(Doc *self) {
+ return self->fields;
+}
+
+
diff --git a/core/Lucy/Document/Doc.cfh b/core/Lucy/Document/Doc.cfh
new file mode 100644
index 0000000..51b9763
--- /dev/null
+++ b/core/Lucy/Document/Doc.cfh
@@ -0,0 +1,104 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** A document.
+ *
+ * A Doc object is akin to a row in a database, in that it is made up of one
+ * or more fields, each of which has a value.
+ */
+
+public class Lucy::Document::Doc inherits Lucy::Object::Obj
+ : dumpable {
+
+ void *fields;
+ int32_t doc_id;
+
+ inert incremented Doc*
+ new(void *fields = NULL, int32_t doc_id = 0);
+
+ /**
+ * @param fields Field-value pairs.
+ * @param doc_id Internal Lucy document id. Default of 0 (an
+ * invalid doc id).
+ */
+ public inert Doc*
+ init(Doc *self, void *fields = NULL, int32_t doc_id = 0);
+
+ /** Set internal Lucy document id.
+ */
+ public void
+ Set_Doc_ID(Doc *self, int32_t doc_id);
+
+ /** Retrieve internal Lucy document id.
+ */
+ public int32_t
+ Get_Doc_ID(Doc *self);
+
+ /** Store a field value in the Doc.
+ */
+ void
+ Store(Doc *self, const CharBuf *field, Obj *value);
+
+ /** Set the doc's field's attribute.
+ */
+ void
+ Set_Fields(Doc *self, void *fields);
+
+ /** Return the Doc's backing fields hash.
+ */
+ public nullable void*
+ Get_Fields(Doc *self);
+
+ /** Return the number of fields in the Doc.
+ */
+ public uint32_t
+ Get_Size(Doc *self);
+
+ /** Retrieve the field's value, or NULL if the field is not present. If
+ * the field is a text type, assign it to <code>target</code>. Otherwise,
+ * return the interior object. Callers must check to verify the kind of
+ * object returned.
+ */
+ nullable Obj*
+ Extract(Doc *self, CharBuf *field, ViewCharBuf *target);
+
+ /* Unimplemented methods.
+ */
+ public bool_t
+ Equals(Doc *self, Obj *other);
+
+ public void
+ Serialize(Doc *self, OutStream *outstream);
+
+ public incremented Doc*
+ Deserialize(Doc *self, InStream *instream);
+
+ public incremented Hash*
+ Dump(Doc *self);
+
+ public incremented Doc*
+ Load(Doc *self, Obj *dump);
+
+ void*
+ To_Host(Doc *self);
+
+ public void
+ Destroy(Doc *self);
+}
+
+
diff --git a/core/Lucy/Document/HitDoc.c b/core/Lucy/Document/HitDoc.c
new file mode 100644
index 0000000..6b4e05a
--- /dev/null
+++ b/core/Lucy/Document/HitDoc.c
@@ -0,0 +1,91 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HITDOC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+HitDoc*
+HitDoc_new(void *fields, int32_t doc_id, float score) {
+ HitDoc *self = (HitDoc*)VTable_Make_Obj(HITDOC);
+ return HitDoc_init(self, fields, doc_id, score);
+}
+
+HitDoc*
+HitDoc_init(HitDoc *self, void *fields, int32_t doc_id, float score) {
+ Doc_init((Doc*)self, fields, doc_id);
+ self->score = score;
+ return self;
+}
+
+void
+HitDoc_set_score(HitDoc *self, float score) {
+ self->score = score;
+}
+
+float
+HitDoc_get_score(HitDoc *self) {
+ return self->score;
+}
+
+void
+HitDoc_serialize(HitDoc *self, OutStream *outstream) {
+ Doc_serialize((Doc*)self, outstream);
+ OutStream_Write_F32(outstream, self->score);
+}
+
+HitDoc*
+HitDoc_deserialize(HitDoc *self, InStream *instream) {
+ self = self ? self : (HitDoc*)VTable_Make_Obj(HITDOC);
+ Doc_deserialize((Doc*)self, instream);
+ self->score = InStream_Read_F32(instream);
+ return self;
+}
+
+Hash*
+HitDoc_dump(HitDoc *self) {
+ HitDoc_dump_t super_dump
+ = (HitDoc_dump_t)SUPER_METHOD(HITDOC, HitDoc, Dump);
+ Hash *dump = super_dump(self);
+ Hash_Store_Str(dump, "score", 5, (Obj*)CB_newf("%f64", self->score));
+ return dump;
+}
+
+HitDoc*
+HitDoc_load(HitDoc *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ HitDoc_load_t super_load
+ = (HitDoc_load_t)SUPER_METHOD(HITDOC, HitDoc, Load);
+ HitDoc *loaded = super_load(self, dump);
+ Obj *score = CERTIFY(Hash_Fetch_Str(source, "score", 5), OBJ);
+ loaded->score = (float)Obj_To_F64(score);
+ return loaded;
+}
+
+bool_t
+HitDoc_equals(HitDoc *self, Obj *other) {
+ HitDoc *twin = (HitDoc*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, HITDOC)) { return false; }
+ if (!Doc_equals((Doc*)self, other)) { return false; }
+ if (self->score != twin->score) { return false; }
+ return true;
+}
+
+
diff --git a/core/Lucy/Document/HitDoc.cfh b/core/Lucy/Document/HitDoc.cfh
new file mode 100644
index 0000000..cbc5b06
--- /dev/null
+++ b/core/Lucy/Document/HitDoc.cfh
@@ -0,0 +1,69 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * A document read from an index.
+ *
+ * HitDoc is the search-time relative of the index-time class Doc; it is
+ * augmented by a numeric score attribute that Doc doesn't have.
+ */
+
+class Lucy::Document::HitDoc inherits Lucy::Document::Doc {
+
+ float score;
+
+ inert incremented HitDoc*
+ new(void *fields = NULL, int32_t doc_id = 0, float score = 0.0);
+
+ /** Constructor.
+ *
+ * @param fields A hash of field name / field value pairs.
+ * @param doc_id Internal document id.
+ * @param score Number indicating how well the doc scored against a query.
+ */
+ inert HitDoc*
+ init(HitDoc *self, void *fields = NULL, int32_t doc_id = 0,
+ float score = 0.0);
+
+ /** Set score attribute.
+ */
+ public void
+ Set_Score(HitDoc *self, float score);
+
+ /** Get score attribute.
+ */
+ public float
+ Get_Score(HitDoc *self);
+
+ public bool_t
+ Equals(HitDoc *self, Obj *other);
+
+ public incremented Hash*
+ Dump(HitDoc *self);
+
+ public incremented HitDoc*
+ Load(HitDoc *self, Obj *dump);
+
+ public void
+ Serialize(HitDoc *self, OutStream *outstream);
+
+ public incremented HitDoc*
+ Deserialize(HitDoc *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Highlight/HeatMap.c b/core/Lucy/Highlight/HeatMap.c
new file mode 100644
index 0000000..0472022
--- /dev/null
+++ b/core/Lucy/Highlight/HeatMap.c
@@ -0,0 +1,210 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HEATMAP
+#define C_LUCY_SPAN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Highlight/HeatMap.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Util/SortUtils.h"
+
+HeatMap*
+HeatMap_new(VArray *spans, uint32_t window) {
+ HeatMap *self = (HeatMap*)VTable_Make_Obj(HEATMAP);
+ return HeatMap_init(self, spans, window);
+}
+
+HeatMap*
+HeatMap_init(HeatMap *self, VArray *spans, uint32_t window) {
+ VArray *spans_copy = VA_Shallow_Copy(spans);
+ VArray *spans_plus_boosts;
+
+ self->spans = NULL;
+ self->window = window;
+
+ VA_Sort(spans_copy, NULL, NULL);
+ spans_plus_boosts = HeatMap_Generate_Proximity_Boosts(self, spans_copy);
+ VA_Push_VArray(spans_plus_boosts, spans_copy);
+ VA_Sort(spans_plus_boosts, NULL, NULL);
+ self->spans = HeatMap_Flatten_Spans(self, spans_plus_boosts);
+
+ DECREF(spans_plus_boosts);
+ DECREF(spans_copy);
+
+ return self;
+}
+
+void
+HeatMap_destroy(HeatMap *self) {
+ DECREF(self->spans);
+ SUPER_DESTROY(self, HEATMAP);
+}
+
+static int
+S_compare_i32(void *context, const void *va, const void *vb) {
+ UNUSED_VAR(context);
+ return *(int32_t*)va - *(int32_t*)vb;
+}
+
+// Create all the spans needed by HeatMap_Flatten_Spans, based on the source
+// offsets and lengths... but leave the scores at 0.
+static VArray*
+S_flattened_but_empty_spans(VArray *spans) {
+ const uint32_t num_spans = VA_Get_Size(spans);
+ int32_t *bounds = (int32_t*)MALLOCATE((num_spans * 2) * sizeof(int32_t));
+
+ // Assemble a list of all unique start/end boundaries.
+ for (uint32_t i = 0; i < num_spans; i++) {
+ Span *span = (Span*)VA_Fetch(spans, i);
+ bounds[i] = span->offset;
+ bounds[i + num_spans] = span->offset + span->length;
+ }
+ Sort_quicksort(bounds, num_spans * 2, sizeof(uint32_t),
+ S_compare_i32, NULL);
+ uint32_t num_bounds = 0;
+ int32_t last = I32_MAX;
+ for (uint32_t i = 0; i < num_spans * 2; i++) {
+ if (bounds[i] != last) {
+ bounds[num_bounds++] = bounds[i];
+ last = bounds[i];
+ }
+ }
+
+ // Create one Span for each zone between two bounds.
+ VArray *flattened = VA_new(num_bounds - 1);
+ for (uint32_t i = 0; i < num_bounds - 1; i++) {
+ int32_t start = bounds[i];
+ int32_t length = bounds[i + 1] - start;
+ VA_Push(flattened, (Obj*)Span_new(start, length, 0.0f));
+ }
+
+ FREEMEM(bounds);
+ return flattened;
+}
+
+VArray*
+HeatMap_flatten_spans(HeatMap *self, VArray *spans) {
+ const uint32_t num_spans = VA_Get_Size(spans);
+ UNUSED_VAR(self);
+
+ if (!num_spans) {
+ return VA_new(0);
+ }
+ else {
+ VArray *flattened = S_flattened_but_empty_spans(spans);
+ const uint32_t num_raw_flattened = VA_Get_Size(flattened);
+
+ // Iterate over each of the source spans, contributing their scores to
+ // any destination span that falls within range.
+ uint32_t dest_tick = 0;
+ for (uint32_t i = 0; i < num_spans; i++) {
+ Span *source_span = (Span*)VA_Fetch(spans, i);
+ int32_t source_span_end
+ = source_span->offset + source_span->length;
+
+ // Get the location of the flattened span that shares the source
+ // span's offset.
+ for (; dest_tick < num_raw_flattened; dest_tick++) {
+ Span *dest_span = (Span*)VA_Fetch(flattened, dest_tick);
+ if (dest_span->offset == source_span->offset) {
+ break;
+ }
+ }
+
+ // Fill in scores.
+ for (uint32_t j = dest_tick; j < num_raw_flattened; j++) {
+ Span *dest_span = (Span*)VA_Fetch(flattened, j);
+ if (dest_span->offset == source_span_end) {
+ break;
+ }
+ else {
+ dest_span->weight += source_span->weight;
+ }
+ }
+ }
+
+ // Leave holes instead of spans that don't have any score.
+ dest_tick = 0;
+ for (uint32_t i = 0; i < num_raw_flattened; i++) {
+ Span *span = (Span*)VA_Fetch(flattened, i);
+ if (span->weight) {
+ VA_Store(flattened, dest_tick++, INCREF(span));
+ }
+ }
+ VA_Excise(flattened, dest_tick, num_raw_flattened - dest_tick);
+
+ return flattened;
+ }
+}
+
+float
+HeatMap_calc_proximity_boost(HeatMap *self, Span *span1, Span *span2) {
+ int32_t comparison = Span_Compare_To(span1, (Obj*)span2);
+ Span *lower = comparison <= 0 ? span1 : span2;
+ Span *upper = comparison >= 0 ? span1 : span2;
+ int32_t lower_end_offset = lower->offset + lower->length;
+ int32_t distance = upper->offset - lower_end_offset;
+
+ // If spans overlap, set distance to 0.
+ if (distance < 0) { distance = 0; }
+
+ if (distance > (int32_t)self->window) {
+ return 0.0f;
+ }
+ else {
+ float factor = (self->window - distance) / (float)self->window;
+ // Damp boost with greater distance.
+ factor *= factor;
+ return factor * (lower->weight + upper->weight);
+ }
+}
+
+VArray*
+HeatMap_generate_proximity_boosts(HeatMap *self, VArray *spans) {
+ VArray *boosts = VA_new(0);
+ const uint32_t num_spans = VA_Get_Size(spans);
+
+ if (num_spans > 1) {
+ for (uint32_t i = 0, max = num_spans - 1; i < max; i++) {
+ Span *span1 = (Span*)VA_Fetch(spans, i);
+
+ for (uint32_t j = i + 1; j <= max; j++) {
+ Span *span2 = (Span*)VA_Fetch(spans, j);
+ float prox_score
+ = HeatMap_Calc_Proximity_Boost(self, span1, span2);
+ if (prox_score == 0) {
+ break;
+ }
+ else {
+ int32_t length = (span2->offset - span1->offset)
+ + span2->length;
+ VA_Push(boosts,
+ (Obj*)Span_new(span1->offset, length, prox_score));
+ }
+ }
+ }
+ }
+
+ return boosts;
+}
+
+VArray*
+HeatMap_get_spans(HeatMap *self) {
+ return self->spans;
+}
+
+
diff --git a/core/Lucy/Highlight/HeatMap.cfh b/core/Lucy/Highlight/HeatMap.cfh
new file mode 100644
index 0000000..1dcb726
--- /dev/null
+++ b/core/Lucy/Highlight/HeatMap.cfh
@@ -0,0 +1,82 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Density of relevant data in a string.
+ *
+ * A HeatMap stores a number for each location in a string, indicating the
+ * "heat" (density) of relevant data in areas which match a search query.
+ */
+
+class Lucy::Highlight::HeatMap inherits Lucy::Object::Obj {
+
+ VArray *spans;
+ uint32_t window;
+
+ inert incremented HeatMap*
+ new(VArray *spans, uint32_t window = 133);
+
+ /**
+ * @param spans An array of Spans, which need not be sorted and will not
+ * be modified.
+ * @param window The greatest distance between which heat points may
+ * reinforce each other.
+ */
+ public inert HeatMap*
+ init(HeatMap *self, VArray *spans, uint32_t window = 133);
+
+ /** Reduce/slice overlapping spans. Say we have two spans:
+ *
+ * Span 1: positions 11-20, score .3
+ * Span 2: positions 16-30, score .5
+ *
+ * After merging, there will be three.
+ *
+ * Span 1: positions 11-16, score .3
+ * Span 2: positions 16-20, score .8
+ * Span 3: positions 20-30, score .5
+ *
+ * @param spans An array of Spans, which must be sorted by offset then
+ * length.
+ */
+ incremented VArray*
+ Flatten_Spans(HeatMap *self, VArray *spans);
+
+ /** If the two spans overlap or abut, return a bonus equal to their summed
+ * scores; as they move further apart, tail the bonus down until it hits 0
+ * at the edge of the <code>window</code>.
+ */
+ float
+ Calc_Proximity_Boost(HeatMap *self, Span *span1, Span *span2);
+
+ /** Iterate through a sorted array of spans, generating a new span for
+ * each pair that yields a non-zero proximity boost.
+ *
+ * @param spans An array of Spans, which must be sorted by offset then
+ * length.
+ */
+ incremented VArray*
+ Generate_Proximity_Boosts(HeatMap *self, VArray *spans);
+
+ VArray*
+ Get_Spans(HeatMap *self);
+
+ public void
+ Destroy(HeatMap *self);
+}
+
+
diff --git a/core/Lucy/Highlight/Highlighter.c b/core/Lucy/Highlight/Highlighter.c
new file mode 100644
index 0000000..d66a3d6
--- /dev/null
+++ b/core/Lucy/Highlight/Highlighter.c
@@ -0,0 +1,672 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HIGHLIGHTER
+#define C_LUCY_SPAN
+#include <ctype.h>
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Highlight/Highlighter.h"
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Highlight/HeatMap.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Index/DocVector.h"
+
+const uint32_t ELLIPSIS_CODE_POINT = 0x2026;
+
+/* If Highlighter_Encode has been overridden, return its output. If not,
+ * increment the refcount of the supplied encode_buf and call encode_entities.
+ * Either way, the caller takes responsibility for one refcount.
+ *
+ * The point of this routine is to minimize CharBuf object creation when
+ * possible.
+ */
+static CharBuf*
+S_do_encode(Highlighter *self, CharBuf *text, CharBuf **encode_buf);
+
+// Place HTML entity encoded version of [text] into [encoded].
+static CharBuf*
+S_encode_entities(CharBuf *text, CharBuf *encoded);
+
+Highlighter*
+Highlighter_new(Searcher *searcher, Obj *query, const CharBuf *field,
+ uint32_t excerpt_length) {
+ Highlighter *self = (Highlighter*)VTable_Make_Obj(HIGHLIGHTER);
+ return Highlighter_init(self, searcher, query, field, excerpt_length);
+}
+
+Highlighter*
+Highlighter_init(Highlighter *self, Searcher *searcher, Obj *query,
+ const CharBuf *field, uint32_t excerpt_length) {
+ self->query = Searcher_Glean_Query(searcher, query);
+ self->searcher = (Searcher*)INCREF(searcher);
+ self->field = CB_Clone(field);
+ self->compiler = Query_Make_Compiler(self->query, searcher,
+ Query_Get_Boost(self->query));
+ self->excerpt_length = excerpt_length;
+ self->slop = excerpt_length / 3;
+ self->window_width = excerpt_length + (self->slop * 2);
+ self->pre_tag = CB_new_from_trusted_utf8("<strong>", 8);
+ self->post_tag = CB_new_from_trusted_utf8("</strong>", 9);
+ return self;
+}
+
+void
+Highlighter_destroy(Highlighter *self) {
+ DECREF(self->searcher);
+ DECREF(self->query);
+ DECREF(self->compiler);
+ DECREF(self->field);
+ DECREF(self->pre_tag);
+ DECREF(self->post_tag);
+ SUPER_DESTROY(self, HIGHLIGHTER);
+}
+
+CharBuf*
+Highlighter_highlight(Highlighter *self, const CharBuf *text) {
+ size_t size = CB_Get_Size(text)
+ + CB_Get_Size(self->pre_tag)
+ + CB_Get_Size(self->post_tag);
+ CharBuf *retval = CB_new(size);
+ CB_Cat(retval, self->pre_tag);
+ CB_Cat(retval, text);
+ CB_Cat(retval, self->post_tag);
+ return retval;
+}
+
+void
+Highlighter_set_pre_tag(Highlighter *self, const CharBuf *pre_tag) {
+ CB_Mimic(self->pre_tag, (Obj*)pre_tag);
+}
+
+void
+Highlighter_set_post_tag(Highlighter *self, const CharBuf *post_tag) {
+ CB_Mimic(self->post_tag, (Obj*)post_tag);
+}
+
+CharBuf*
+Highlighter_get_pre_tag(Highlighter *self) {
+ return self->pre_tag;
+}
+
+CharBuf*
+Highlighter_get_post_tag(Highlighter *self) {
+ return self->post_tag;
+}
+
+CharBuf*
+Highlighter_get_field(Highlighter *self) {
+ return self->field;
+}
+
+Query*
+Highlighter_get_query(Highlighter *self) {
+ return self->query;
+}
+
+Searcher*
+Highlighter_get_searcher(Highlighter *self) {
+ return self->searcher;
+}
+
+Compiler*
+Highlighter_get_compiler(Highlighter *self) {
+ return self->compiler;
+}
+
+uint32_t
+Highlighter_get_excerpt_length(Highlighter *self) {
+ return self->excerpt_length;
+}
+
+CharBuf*
+Highlighter_create_excerpt(Highlighter *self, HitDoc *hit_doc) {
+ ZombieCharBuf *field_val
+ = (ZombieCharBuf*)HitDoc_Extract(hit_doc, self->field,
+ (ViewCharBuf*)ZCB_BLANK());
+
+ if (!field_val || !Obj_Is_A((Obj*)field_val, CHARBUF)) {
+ return NULL;
+ }
+ else if (!ZCB_Get_Size(field_val)) {
+ // Empty string yields empty string.
+ return CB_new(0);
+ }
+ else {
+ ZombieCharBuf *fragment = ZCB_WRAP((CharBuf*)field_val);
+ CharBuf *raw_excerpt = CB_new(self->excerpt_length + 10);
+ CharBuf *highlighted = CB_new((self->excerpt_length * 3) / 2);
+ DocVector *doc_vec
+ = Searcher_Fetch_Doc_Vec(self->searcher,
+ HitDoc_Get_Doc_ID(hit_doc));
+ VArray *maybe_spans
+ = Compiler_Highlight_Spans(self->compiler, self->searcher,
+ doc_vec, self->field);
+ VArray *score_spans = maybe_spans ? maybe_spans : VA_new(0);
+ HeatMap *heat_map
+ = HeatMap_new(score_spans, (self->excerpt_length * 2) / 3);
+ int32_t top
+ = Highlighter_Find_Best_Fragment(self, (CharBuf*)field_val,
+ (ViewCharBuf*)fragment, heat_map);
+ VArray *sentences
+ = Highlighter_Find_Sentences(self, (CharBuf*)field_val, 0,
+ top + self->window_width);
+
+ top = Highlighter_Raw_Excerpt(self, (CharBuf*)field_val,
+ (CharBuf*)fragment, raw_excerpt, top,
+ heat_map, sentences);
+ VA_Sort(score_spans, NULL, NULL);
+ Highlighter_highlight_excerpt(self, score_spans, raw_excerpt,
+ highlighted, top);
+
+ DECREF(sentences);
+ DECREF(heat_map);
+ DECREF(score_spans);
+ DECREF(doc_vec);
+ DECREF(raw_excerpt);
+
+ return highlighted;
+ }
+}
+
+static int32_t
+S_hottest(HeatMap *heat_map) {
+ float max_score = 0.0f;
+ int32_t retval = 0;
+ VArray *spans = HeatMap_Get_Spans(heat_map);
+ for (uint32_t i = VA_Get_Size(spans); i--;) {
+ Span *span = (Span*)VA_Fetch(spans, i);
+ if (span->weight >= max_score) {
+ retval = span->offset;
+ max_score = span->weight;
+ }
+ }
+ return retval;
+}
+
+int32_t
+Highlighter_find_best_fragment(Highlighter *self, const CharBuf *field_val,
+ ViewCharBuf *fragment, HeatMap *heat_map) {
+ // Window is 1.66 * excerpt_length, with the loc in the middle.
+ int32_t best_location = S_hottest(heat_map);
+
+ if (best_location < (int32_t)self->slop) {
+ // If the beginning of the string falls within the window centered
+ // around the hottest point in the field, start the fragment at the
+ // beginning.
+ ViewCB_Assign(fragment, (CharBuf*)field_val);
+ int32_t top = ViewCB_Trim_Top(fragment);
+ ViewCB_Truncate(fragment, self->window_width);
+ return top;
+ }
+ else {
+ int32_t top = best_location - self->slop;
+ ViewCB_Assign(fragment, (CharBuf*)field_val);
+ ViewCB_Nip(fragment, top);
+ top += ViewCB_Trim_Top(fragment);
+ int32_t chars_left = ViewCB_Truncate(fragment, self->excerpt_length);
+ int32_t overrun = self->excerpt_length - chars_left;
+
+ if (!overrun) {
+ // We've found an acceptable window.
+ ViewCB_Assign(fragment, (CharBuf*)field_val);
+ ViewCB_Nip(fragment, top);
+ top += ViewCB_Trim_Top(fragment);
+ ViewCB_Truncate(fragment, self->window_width);
+ return top;
+ }
+ else if (overrun > top) {
+ // The field is very short, so make the whole field the
+ // "fragment".
+ ViewCB_Assign(fragment, (CharBuf*)field_val);
+ return ViewCB_Trim_Top(fragment);
+ }
+ else {
+ // The fragment is too close to the end, so slide it back.
+ top -= overrun;
+ ViewCB_Assign(fragment, (CharBuf*)field_val);
+ ViewCB_Nip(fragment, top);
+ top += ViewCB_Trim_Top(fragment);
+ ViewCB_Truncate(fragment, self->excerpt_length);
+ return top;
+ }
+ }
+}
+
+// Return true if the window represented by "offset" and "length" overlaps a
+// score span, or if there are no score spans so that no excerpt is measurably
+// superior.
+static bool_t
+S_has_heat(HeatMap *heat_map, int32_t offset, int32_t length) {
+ VArray *spans = HeatMap_Get_Spans(heat_map);
+ uint32_t num_spans = VA_Get_Size(spans);
+ int32_t end = offset + length;
+
+ if (length == 0) { return false; }
+ if (num_spans == 0) { return true; }
+
+ for (uint32_t i = 0; i < num_spans; i++) {
+ Span *span = (Span*)VA_Fetch(spans, i);
+ int32_t span_start = span->offset;
+ int32_t span_end = span_start + span->length;
+ if (offset >= span_start && offset < span_end) { return true; }
+ if (end > span_start && end <= span_end) { return true; }
+ if (offset <= span_start && end >= span_end) { return true; }
+ if (span_start > end) { break; }
+ }
+
+ return false;
+}
+
+int32_t
+Highlighter_raw_excerpt(Highlighter *self, const CharBuf *field_val,
+ const CharBuf *fragment, CharBuf *raw_excerpt,
+ int32_t top, HeatMap *heat_map, VArray *sentences) {
+ bool_t found_starting_edge = false;
+ bool_t found_ending_edge = false;
+ int32_t start = top;
+ int32_t end = 0;
+ double field_len = CB_Length(field_val);
+ uint32_t min_len = field_len < self->excerpt_length * 0.6666
+ ? (uint32_t)field_len
+ : (uint32_t)(self->excerpt_length * 0.6666);
+
+ // Try to find a starting sentence boundary.
+ const uint32_t num_sentences = VA_Get_Size(sentences);
+ if (num_sentences) {
+ for (uint32_t i = 0; i < num_sentences; i++) {
+ Span *sentence = (Span*)VA_Fetch(sentences, i);
+ int32_t candidate = sentence->offset;
+
+ if (candidate > top + (int32_t)self->window_width) {
+ break;
+ }
+ else if (candidate >= top) {
+ // Try to start on the first sentence boundary, but only if
+ // there's enough relevant material left after it in the
+ // fragment.
+ ZombieCharBuf *temp = ZCB_WRAP(fragment);
+ ZCB_Nip(temp, candidate - top);
+ uint32_t chars_left = ZCB_Truncate(temp, self->excerpt_length);
+ if (chars_left >= min_len
+ && S_has_heat(heat_map, candidate, chars_left)
+ ) {
+ start = candidate;
+ found_starting_edge = true;
+ break;
+ }
+ }
+ }
+ }
+
+ // Try to end on a sentence boundary (but don't try very hard).
+ if (num_sentences) {
+ ZombieCharBuf *start_trimmed = ZCB_WRAP(fragment);
+ ZCB_Nip(start_trimmed, start - top);
+
+ for (uint32_t i = num_sentences; i--;) {
+ Span *sentence = (Span*)VA_Fetch(sentences, i);
+ int32_t last_edge = sentence->offset + sentence->length;
+
+ if (last_edge <= start) {
+ break;
+ }
+ else if (last_edge - start > (int32_t)self->excerpt_length) {
+ continue;
+ }
+ else {
+ uint32_t chars_left = last_edge - start;
+ if (chars_left > min_len
+ && S_has_heat(heat_map, start, chars_left)
+ ) {
+ found_ending_edge = true;
+ end = last_edge;
+ break;
+ }
+ else {
+ ZombieCharBuf *temp = ZCB_WRAP((CharBuf*)start_trimmed);
+ ZCB_Nip(temp, chars_left);
+ ZCB_Trim_Tail(temp);
+ if (ZCB_Get_Size(temp) == 0) {
+ // Short, but ending on a boundary already.
+ found_ending_edge = true;
+ end = last_edge;
+ break;
+ }
+ }
+ }
+ }
+ }
+ int32_t this_excerpt_len = found_ending_edge
+ ? end - start
+ : (int32_t)self->excerpt_length;
+ if (!this_excerpt_len) { return start; }
+
+ if (found_starting_edge) {
+ ZombieCharBuf *temp = ZCB_WRAP((CharBuf*)field_val);
+ ZCB_Nip(temp, start);
+ ZCB_Truncate(temp, this_excerpt_len);
+ CB_Mimic(raw_excerpt, (Obj*)temp);
+ }
+ // If not starting on a sentence boundary, prepend an ellipsis.
+ else {
+ ZombieCharBuf *temp = ZCB_WRAP((CharBuf*)field_val);
+ const size_t ELLIPSIS_LEN = 2; // Unicode ellipsis plus a space.
+
+ // If the excerpt is already shorter than the spec'd length, we might
+ // not need to make room.
+ this_excerpt_len += ELLIPSIS_LEN;
+
+ // Move the start back one in case the character right before the
+ // excerpt starts is whitespace.
+ if (start) {
+ this_excerpt_len += 1;
+ start -= 1;
+ ZCB_Nip(temp, start);
+ }
+
+ do {
+ uint32_t code_point = ZCB_Nip_One(temp);
+ start++;
+ this_excerpt_len--;
+
+ if (StrHelp_is_whitespace(code_point)) {
+ if (!found_ending_edge) {
+ // If we still need room, we'll lop it off the end since
+ // we don't know a solid end point yet.
+ break;
+ }
+ else if (this_excerpt_len <= (int32_t)self->excerpt_length) {
+ break;
+ }
+ }
+ } while (ZCB_Get_Size(temp));
+
+ ZCB_Truncate(temp, self->excerpt_length - ELLIPSIS_LEN);
+ CB_Cat_Char(raw_excerpt, ELLIPSIS_CODE_POINT);
+ CB_Cat_Char(raw_excerpt, ' ');
+ CB_Cat(raw_excerpt, (CharBuf*)temp);
+ start -= ELLIPSIS_LEN;
+ }
+
+ // If excerpt doesn't end on a sentence boundary, tack on an ellipsis.
+ if (found_ending_edge) {
+ CB_Truncate(raw_excerpt, end - start);
+ }
+ else {
+ do {
+ uint32_t code_point = CB_Code_Point_From(raw_excerpt, 1);
+ CB_Chop(raw_excerpt, 1);
+ if (StrHelp_is_whitespace(code_point)) {
+ CB_Trim_Tail(raw_excerpt);
+
+ // Strip punctuation that collides with an ellipsis.
+ code_point = CB_Code_Point_From(raw_excerpt, 1);
+ while (code_point == '.'
+ || code_point == ','
+ || code_point == ';'
+ || code_point == ':'
+ || code_point == ':'
+ || code_point == '?'
+ || code_point == '!'
+ ) {
+ CB_Chop(raw_excerpt, 1);
+ code_point = CB_Code_Point_From(raw_excerpt, 1);
+ }
+
+ break;
+ }
+ } while (CB_Get_Size(raw_excerpt));
+ CB_Cat_Char(raw_excerpt, ELLIPSIS_CODE_POINT);
+ }
+
+ return start;
+}
+
+void
+Highlighter_highlight_excerpt(Highlighter *self, VArray *spans,
+ CharBuf *raw_excerpt, CharBuf *highlighted,
+ int32_t top) {
+ int32_t last_end = 0;
+ ZombieCharBuf *temp = ZCB_WRAP(raw_excerpt);
+ CharBuf *encode_buf = NULL;
+ int32_t raw_excerpt_end = top + CB_Length(raw_excerpt);
+
+ for (uint32_t i = 0, max = VA_Get_Size(spans); i < max; i++) {
+ Span *span = (Span*)VA_Fetch(spans, i);
+ if (span->offset < top) {
+ continue;
+ }
+ else if (span->offset >= raw_excerpt_end) {
+ break;
+ }
+ else {
+ int32_t relative_start = span->offset - top;
+ int32_t relative_end = relative_start + span->length;
+
+ if (relative_start > last_end) {
+ CharBuf *encoded;
+ int32_t non_highlighted_len = relative_start - last_end;
+ ZombieCharBuf *to_cat = ZCB_WRAP((CharBuf*)temp);
+ ZCB_Truncate(to_cat, non_highlighted_len);
+ encoded = S_do_encode(self, (CharBuf*)to_cat, &encode_buf);
+ CB_Cat(highlighted, (CharBuf*)encoded);
+ ZCB_Nip(temp, non_highlighted_len);
+ DECREF(encoded);
+ }
+ if (relative_end > relative_start) {
+ CharBuf *encoded;
+ CharBuf *hl_frag;
+ int32_t highlighted_len = relative_end - relative_start;
+ ZombieCharBuf *to_cat = ZCB_WRAP((CharBuf*)temp);
+ ZCB_Truncate(to_cat, highlighted_len);
+ encoded = S_do_encode(self, (CharBuf*)to_cat, &encode_buf);
+ hl_frag = Highlighter_Highlight(self, encoded);
+ CB_Cat(highlighted, hl_frag);
+ ZCB_Nip(temp, highlighted_len);
+ DECREF(hl_frag);
+ DECREF(encoded);
+ }
+ last_end = relative_end;
+ }
+ }
+
+ // Last text, beyond last highlight span.
+ if (ZCB_Get_Size(temp)) {
+ CharBuf *encoded = S_do_encode(self, (CharBuf*)temp, &encode_buf);
+ CB_Cat(highlighted, encoded);
+ DECREF(encoded);
+ }
+ CB_Trim_Tail(highlighted);
+
+ DECREF(encode_buf);
+}
+
+static Span*
+S_start_sentence(int32_t pos) {
+ return Span_new(pos, 0, 0.0);
+}
+
+static void
+S_close_sentence(VArray *sentences, Span **sentence_ptr,
+ int32_t sentence_end) {
+ Span *sentence = *sentence_ptr;
+ int32_t length = sentence_end - Span_Get_Offset(sentence);
+ const int32_t MIN_SENTENCE_LENGTH = 3; // e.g. "OK.", but not "2."
+ if (length >= MIN_SENTENCE_LENGTH) {
+ Span_Set_Length(sentence, length);
+ VA_Push(sentences, (Obj*)sentence);
+ *sentence_ptr = NULL;
+ }
+}
+
+VArray*
+Highlighter_find_sentences(Highlighter *self, CharBuf *text, int32_t offset,
+ int32_t length) {
+ /* When [sentence] is NULL, that means a sentence start has not yet been
+ * found. When it is a Span object, we have a start, but we haven't found
+ * an end. Once we find the end, we add the sentence to the [sentences]
+ * array and set [sentence] back to NULL to indicate that we're looking
+ * for a start once more.
+ */
+ Span *sentence = NULL;
+ VArray *sentences = VA_new(10);
+ int32_t stop = length == 0
+ ? I32_MAX
+ : offset + length;
+ ZombieCharBuf *fragment = ZCB_WRAP(text);
+ int32_t pos = ZCB_Trim_Top(fragment);
+ UNUSED_VAR(self);
+
+ /* Our first task will be to find a sentence that either starts at the top
+ * of the fragment, or overlaps its start. Starting at the top of the
+ * field is a special case: we define the first non-whitespace character
+ * to begin a sentence, rather than look for the first character following
+ * a period and whitespace. Everywhere else, we have to define sentence
+ * starts based on a sentence end that has just passed by.
+ */
+ if (offset <= pos) {
+ // Assume that first non-whitespace character begins a sentence.
+ if (pos < stop && ZCB_Get_Size(fragment) > 0) {
+ sentence = S_start_sentence(pos);
+ }
+ }
+ else {
+ ZCB_Nip(fragment, offset - pos);
+ pos = offset;
+ }
+
+ while (1) {
+ uint32_t code_point = ZCB_Code_Point_At(fragment, 0);
+ if (!code_point) {
+ // End of fragment. If we have a sentence open, close it,
+ // then bail.
+ if (sentence) { S_close_sentence(sentences, &sentence, pos); }
+ break;
+ }
+ else if (code_point == '.') {
+ uint32_t whitespace_count;
+ pos += ZCB_Nip(fragment, 1); // advance past "."
+
+ if (pos == stop && ZCB_Get_Size(fragment) == 0) {
+ // Period ending the field string.
+ if (sentence) { S_close_sentence(sentences, &sentence, pos); }
+ break;
+ }
+ else if (0 != (whitespace_count = ZCB_Trim_Top(fragment))) {
+ // We've found a period followed by whitespace. Close out the
+ // existing sentence, if there is one. */
+ if (sentence) { S_close_sentence(sentences, &sentence, pos); }
+
+ // Advance past whitespace.
+ pos += whitespace_count;
+ if (pos < stop && ZCB_Get_Size(fragment) > 0) {
+ // Not at the end of the string? Then we've found a
+ // sentence start.
+ sentence = S_start_sentence(pos);
+ }
+ }
+
+ // We may not have reached the end of the field yet, but it's
+ // entirely possible that our last sentence overlapped the end of
+ // the fragment -- in which case, it's time to bail.
+ if (pos >= stop) { break; }
+ }
+ else {
+ ZCB_Nip(fragment, 1);
+ pos++;
+ }
+ }
+
+ return sentences;
+}
+
+CharBuf*
+Highlighter_encode(Highlighter *self, CharBuf *text) {
+ CharBuf *encoded = CB_new(0);
+ UNUSED_VAR(self);
+ return S_encode_entities(text, encoded);
+}
+
+static CharBuf*
+S_do_encode(Highlighter *self, CharBuf *text, CharBuf **encode_buf) {
+ if (OVERRIDDEN(self, Highlighter, Encode, encode)) {
+ return Highlighter_Encode(self, text);
+ }
+ else {
+ if (*encode_buf == NULL) { *encode_buf = CB_new(0); }
+ (void)S_encode_entities(text, *encode_buf);
+ return (CharBuf*)INCREF(*encode_buf);
+ }
+}
+
+static CharBuf*
+S_encode_entities(CharBuf *text, CharBuf *encoded) {
+ ZombieCharBuf *temp = ZCB_WRAP(text);
+ size_t space = 0;
+ const int MAX_ENTITY_BYTES = 9; // &#dddddd;
+
+ // Scan first so that we only allocate once.
+ uint32_t code_point;
+ while (0 != (code_point = ZCB_Nip_One(temp))) {
+ if (code_point > 127
+ || (!isgraph(code_point) && !isspace(code_point))
+ || code_point == '<'
+ || code_point == '>'
+ || code_point == '&'
+ || code_point == '"'
+ ) {
+ space += MAX_ENTITY_BYTES;
+ }
+ else {
+ space += 1;
+ }
+ }
+
+ CB_Grow(encoded, space);
+ CB_Set_Size(encoded, 0);
+ ZCB_Assign(temp, text);
+ while (0 != (code_point = ZCB_Nip_One(temp))) {
+ if (code_point > 127
+ || (!isgraph(code_point) && !isspace(code_point))
+ ) {
+ CB_catf(encoded, "&#%u32;", code_point);
+ }
+ else if (code_point == '<') {
+ CB_Cat_Trusted_Str(encoded, "<", 4);
+ }
+ else if (code_point == '>') {
+ CB_Cat_Trusted_Str(encoded, ">", 4);
+ }
+ else if (code_point == '&') {
+ CB_Cat_Trusted_Str(encoded, "&", 5);
+ }
+ else if (code_point == '"') {
+ CB_Cat_Trusted_Str(encoded, """, 6);
+ }
+ else {
+ CB_Cat_Char(encoded, code_point);
+ }
+ }
+
+ return encoded;
+}
+
+
+
diff --git a/core/Lucy/Highlight/Highlighter.cfh b/core/Lucy/Highlight/Highlighter.cfh
new file mode 100644
index 0000000..a88b899
--- /dev/null
+++ b/core/Lucy/Highlight/Highlighter.cfh
@@ -0,0 +1,174 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Create and highlight excerpts.
+ *
+ * The Highlighter can be used to select relevant snippets from a document,
+ * and to surround search terms with highlighting tags. It handles both stems
+ * and phrases correctly and efficiently, using special-purpose data generated
+ * at index-time.
+*/
+class Lucy::Highlight::Highlighter inherits Lucy::Object::Obj {
+
+ Searcher *searcher;
+ Query *query;
+ CharBuf *field;
+ uint32_t excerpt_length;
+ uint32_t window_width;
+ uint32_t slop;
+ CharBuf *pre_tag;
+ CharBuf *post_tag;
+ Compiler *compiler;
+
+ inert incremented Highlighter*
+ new(Searcher *searcher, Obj *query, const CharBuf *field,
+ uint32_t excerpt_length = 200);
+
+ /**
+ * @param searcher An object which inherits from
+ * L<Searcher|Lucy::Search::Searcher>, such as an
+ * L<IndexSearcher|Lucy::Search::IndexSearcher>.
+ * @param query Query object or a query string.
+ * @param field The name of the field from which to draw the excerpt. The
+ * field must marked as be C<highlightable> (see
+ * L<FieldType|Lucy::Plan::FieldType>).
+ * @param excerpt_length Maximum length of the excerpt, in characters.
+ */
+ public inert Highlighter*
+ init(Highlighter *self, Searcher *searcher, Obj *query,
+ const CharBuf *field, uint32_t excerpt_length = 200);
+
+ /** Take a HitDoc object and return a highlighted excerpt as a string if
+ * the HitDoc has a value for the specified <code>field</code>.
+ */
+ public incremented CharBuf*
+ Create_Excerpt(Highlighter *self, HitDoc *hit_doc);
+
+ /** Encode text with HTML entities. This method is called internally by
+ * Create_Excerpt() for each text fragment when assembling an excerpt. A
+ * subclass can override this if the text should be encoded differently or
+ * not at all.
+ */
+ public incremented CharBuf*
+ Encode(Highlighter *self, CharBuf *text);
+
+ /** Find sentence boundaries within the specified range, returning them as
+ * an array of Spans. The "offset" of each Span indicates the start of
+ * the sentence, and is measured from 0, not from <code>offset</code>.
+ * The Span's "length" member indicates the sentence length in code
+ * points.
+ *
+ * @param text The string to scan.
+ * @param offset The place to start looking for offsets, measured in
+ * Unicode code points from the top of <code>text</code>.
+ * @param length The number of code points from <code>offset</code> to
+ * scan. The default value of 0 is a sentinel which indicates to scan
+ * until the end of the string.
+ */
+ incremented VArray*
+ Find_Sentences(Highlighter *self, CharBuf *text, int32_t offset = 0,
+ int32_t length = 0);
+
+ /** Highlight a small section of text. By default, prepends pre-tag and
+ * appends post-tag. This method is called internally by Create_Excerpt()
+ * when assembling an excerpt.
+ */
+ public incremented CharBuf*
+ Highlight(Highlighter *self, const CharBuf *text);
+
+ /** Setter. The default value is "<strong>".
+ */
+ public void
+ Set_Pre_Tag(Highlighter *self, const CharBuf *pre_tag);
+
+ /** Setter. The default value is "</strong>".
+ */
+ public void
+ Set_Post_Tag(Highlighter *self, const CharBuf *post_tag);
+
+ /** Accessor.
+ */
+ public CharBuf*
+ Get_Pre_Tag(Highlighter *self);
+
+ /** Accessor.
+ */
+ public CharBuf*
+ Get_Post_Tag(Highlighter *self);
+
+ /** Accessor.
+ */
+ public CharBuf*
+ Get_Field(Highlighter *self);
+
+ /** Accessor.
+ */
+ public uint32_t
+ Get_Excerpt_Length(Highlighter *self);
+
+ /** Accessor.
+ */
+ public Searcher*
+ Get_Searcher(Highlighter *self);
+
+ /** Accessor.
+ */
+ public Query*
+ Get_Query(Highlighter *self);
+
+ /** Accessor for the Lucy::Search::Compiler object derived from
+ * <code>query</code> and <code>searcher</code>.
+ */
+ public Compiler*
+ Get_Compiler(Highlighter *self);
+
+ /** Decide based on heat map the best fragment of field to concentrate on.
+ * Place the result into <code>fragment<code> and return its offset in
+ * code points from the top of the field.
+ *
+ * (Helper function for Create_Excerpt only exposed for testing purposes.)
+ */
+ int32_t
+ Find_Best_Fragment(Highlighter *self, const CharBuf *field_val,
+ ViewCharBuf *fragment, HeatMap *heat_map);
+
+ /** Take the fragment and determine the best edges for it based on
+ * sentence boundaries when possible. Add ellipses when boundaries cannot
+ * be found.
+ *
+ * (Helper function for Create_Excerpt only exposed for testing purposes.)
+ */
+ int32_t
+ Raw_Excerpt(Highlighter *self, const CharBuf *field_val,
+ const CharBuf *fragment, CharBuf *raw_excerpt, int32_t top,
+ HeatMap *heat_map, VArray *sentences);
+
+ /** Take the text in raw_excerpt, add highlight tags, encode, and place
+ * the result into <code>highlighted</code>.
+ *
+ * (Helper function for Create_Excerpt only exposed for testing purposes.)
+ */
+ void
+ Highlight_Excerpt(Highlighter *self, VArray *spans, CharBuf *raw_excerpt,
+ CharBuf *highlighted, int32_t top);
+
+ public void
+ Destroy(Highlighter *self);
+}
+
+
diff --git a/core/Lucy/Index/BackgroundMerger.c b/core/Lucy/Index/BackgroundMerger.c
new file mode 100644
index 0000000..b4c440d
--- /dev/null
+++ b/core/Lucy/Index/BackgroundMerger.c
@@ -0,0 +1,574 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BACKGROUNDMERGER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/BackgroundMerger.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/FilePurger.h"
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SegWriter.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/Json.h"
+
+// Verify a Folder or derive an FSFolder from a CharBuf path.
+static Folder*
+S_init_folder(Obj *index);
+
+// Grab the write lock and store it in self.
+static void
+S_obtain_write_lock(BackgroundMerger *self);
+
+// Grab the merge lock and store it in self.
+static void
+S_obtain_merge_lock(BackgroundMerger *self);
+
+// Release the write lock - if it's there.
+static void
+S_release_write_lock(BackgroundMerger *self);
+
+// Release the merge lock - if it's there.
+static void
+S_release_merge_lock(BackgroundMerger *self);
+
+BackgroundMerger*
+BGMerger_new(Obj *index, IndexManager *manager) {
+ BackgroundMerger *self
+ = (BackgroundMerger*)VTable_Make_Obj(BACKGROUNDMERGER);
+ return BGMerger_init(self, index, manager);
+}
+
+BackgroundMerger*
+BGMerger_init(BackgroundMerger *self, Obj *index, IndexManager *manager) {
+ Folder *folder = S_init_folder(index);
+
+ // Init.
+ self->optimize = false;
+ self->prepared = false;
+ self->needs_commit = false;
+ self->snapfile = NULL;
+ self->doc_maps = Hash_new(0);
+
+ // Assign.
+ self->folder = folder;
+ if (manager) {
+ self->manager = (IndexManager*)INCREF(manager);
+ }
+ else {
+ self->manager = IxManager_new(NULL, NULL);
+ IxManager_Set_Write_Lock_Timeout(self->manager, 10000);
+ }
+ IxManager_Set_Folder(self->manager, folder);
+
+ // Obtain write lock (which we'll only hold briefly), then merge lock.
+ S_obtain_write_lock(self);
+ if (!self->write_lock) {
+ DECREF(self);
+ RETHROW(INCREF(Err_get_error()));
+ }
+ S_obtain_merge_lock(self);
+ if (!self->merge_lock) {
+ DECREF(self);
+ RETHROW(INCREF(Err_get_error()));
+ }
+
+ // Find the latest snapshot. If there's no index content, bail early.
+ self->snapshot = Snapshot_Read_File(Snapshot_new(), folder, NULL);
+ if (!Snapshot_Get_Path(self->snapshot)) {
+ S_release_write_lock(self);
+ S_release_merge_lock(self);
+ return self;
+ }
+
+ // Create FilePurger. Zap detritus from previous sessions.
+ self->file_purger = FilePurger_new(folder, self->snapshot, self->manager);
+ FilePurger_Purge(self->file_purger);
+
+ // Open a PolyReader, passing in the IndexManager so we get a read lock on
+ // the Snapshot's files -- so that Indexers don't zap our files while
+ // we're operating in the background.
+ self->polyreader = PolyReader_open((Obj*)folder, NULL, self->manager);
+
+ // Clone the PolyReader's schema.
+ {
+ Hash *dump = Schema_Dump(PolyReader_Get_Schema(self->polyreader));
+ self->schema = (Schema*)CERTIFY(VTable_Load_Obj(SCHEMA, (Obj*)dump),
+ SCHEMA);
+ DECREF(dump);
+ }
+
+ // Create new Segment.
+ {
+ int64_t new_seg_num
+ = IxManager_Highest_Seg_Num(self->manager, self->snapshot) + 1;
+ VArray *fields = Schema_All_Fields(self->schema);
+ uint32_t i, max;
+ self->segment = Seg_new(new_seg_num);
+ for (i = 0, max = VA_Get_Size(fields); i < max; i++) {
+ Seg_Add_Field(self->segment, (CharBuf*)VA_Fetch(fields, i));
+ }
+ DECREF(fields);
+ }
+
+ // Our "cutoff" is the segment this BackgroundMerger will write. Now that
+ // we've determined the cutoff, write the merge data file.
+ self->cutoff = Seg_Get_Number(self->segment);
+ IxManager_Write_Merge_Data(self->manager, self->cutoff);
+
+ /* Create the SegWriter but hold off on preparing the new segment
+ * directory -- because if we don't need to merge any segments we don't
+ * need it. (We've reserved the dir by plopping down the merge.json
+ * file.) */
+ self->seg_writer = SegWriter_new(self->schema, self->snapshot,
+ self->segment, self->polyreader);
+
+ // Grab a local ref to the DeletionsWriter.
+ self->del_writer
+ = (DeletionsWriter*)INCREF(SegWriter_Get_Del_Writer(self->seg_writer));
+
+ // Release the write lock. Now new Indexers can start while we work in
+ // the background.
+ S_release_write_lock(self);
+
+ return self;
+}
+
+void
+BGMerger_destroy(BackgroundMerger *self) {
+ S_release_merge_lock(self);
+ S_release_write_lock(self);
+ DECREF(self->schema);
+ DECREF(self->folder);
+ DECREF(self->segment);
+ DECREF(self->manager);
+ DECREF(self->polyreader);
+ DECREF(self->del_writer);
+ DECREF(self->snapshot);
+ DECREF(self->seg_writer);
+ DECREF(self->file_purger);
+ DECREF(self->write_lock);
+ DECREF(self->snapfile);
+ DECREF(self->doc_maps);
+ SUPER_DESTROY(self, BACKGROUNDMERGER);
+}
+
+static Folder*
+S_init_folder(Obj *index) {
+ Folder *folder = NULL;
+
+ // Validate or acquire a Folder.
+ if (Obj_Is_A(index, FOLDER)) {
+ folder = (Folder*)INCREF(index);
+ }
+ else if (Obj_Is_A(index, CHARBUF)) {
+ folder = (Folder*)FSFolder_new((CharBuf*)index);
+ }
+ else {
+ THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
+ }
+
+ // Validate index directory.
+ if (!Folder_Check(folder)) {
+ THROW(ERR, "Folder '%o' failed check", Folder_Get_Path(folder));
+ }
+
+ return folder;
+}
+
+void
+BGMerger_optimize(BackgroundMerger *self) {
+ self->optimize = true;
+}
+
+static uint32_t
+S_maybe_merge(BackgroundMerger *self) {
+ VArray *to_merge = IxManager_Recycle(self->manager, self->polyreader,
+ self->del_writer, 0, self->optimize);
+ int32_t num_to_merge = VA_Get_Size(to_merge);
+ uint32_t i, max;
+
+ // There's no point in merging one segment if it has no deletions, because
+ // we'd just be rewriting it. */
+ if (num_to_merge == 1) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(to_merge, 0);
+ if (!SegReader_Del_Count(seg_reader)) {
+ DECREF(to_merge);
+ return 0;
+ }
+ }
+ else if (num_to_merge == 0) {
+ DECREF(to_merge);
+ return 0;
+ }
+
+ // Now that we're sure we're writing a new segment, prep the seg dir.
+ SegWriter_Prep_Seg_Dir(self->seg_writer);
+
+ // Consolidate segments.
+ for (i = 0, max = num_to_merge; i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(to_merge, i);
+ CharBuf *seg_name = SegReader_Get_Seg_Name(seg_reader);
+ int64_t doc_count = Seg_Get_Count(self->segment);
+ Matcher *deletions
+ = DelWriter_Seg_Deletions(self->del_writer, seg_reader);
+ I32Array *doc_map = DelWriter_Generate_Doc_Map(
+ self->del_writer, deletions,
+ SegReader_Doc_Max(seg_reader),
+ (int32_t)doc_count);
+
+ Hash_Store(self->doc_maps, (Obj*)seg_name, (Obj*)doc_map);
+ SegWriter_Merge_Segment(self->seg_writer, seg_reader, doc_map);
+ DECREF(deletions);
+ }
+
+ DECREF(to_merge);
+ return num_to_merge;
+}
+
+static bool_t
+S_merge_updated_deletions(BackgroundMerger *self) {
+ Hash *updated_deletions = NULL;
+
+ {
+ PolyReader *new_polyreader
+ = PolyReader_open((Obj*)self->folder, NULL, NULL);
+ VArray *new_seg_readers
+ = PolyReader_Get_Seg_Readers(new_polyreader);
+ VArray *old_seg_readers
+ = PolyReader_Get_Seg_Readers(self->polyreader);
+ Hash *new_segs = Hash_new(VA_Get_Size(new_seg_readers));
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(new_seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(new_seg_readers, i);
+ CharBuf *seg_name = SegReader_Get_Seg_Name(seg_reader);
+ Hash_Store(new_segs, (Obj*)seg_name, INCREF(seg_reader));
+ }
+
+ for (i = 0, max = VA_Get_Size(old_seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(old_seg_readers, i);
+ CharBuf *seg_name = SegReader_Get_Seg_Name(seg_reader);
+
+ // If this segment was merged away...
+ if (Hash_Fetch(self->doc_maps, (Obj*)seg_name)) {
+ SegReader *new_seg_reader
+ = (SegReader*)CERTIFY(
+ Hash_Fetch(new_segs, (Obj*)seg_name),
+ SEGREADER);
+ int32_t old_del_count = SegReader_Del_Count(seg_reader);
+ int32_t new_del_count = SegReader_Del_Count(new_seg_reader);
+ // ... were any new deletions applied against it?
+ if (old_del_count != new_del_count) {
+ DeletionsReader *del_reader
+ = (DeletionsReader*)SegReader_Obtain(
+ new_seg_reader,
+ VTable_Get_Name(DELETIONSREADER));
+ if (!updated_deletions) {
+ updated_deletions = Hash_new(max);
+ }
+ Hash_Store(updated_deletions, (Obj*)seg_name,
+ (Obj*)DelReader_Iterator(del_reader));
+ }
+ }
+ }
+
+ DECREF(new_polyreader);
+ DECREF(new_segs);
+ }
+
+ if (!updated_deletions) {
+ return false;
+ }
+ else {
+ PolyReader *merge_polyreader
+ = PolyReader_open((Obj*)self->folder, self->snapshot, NULL);
+ VArray *merge_seg_readers
+ = PolyReader_Get_Seg_Readers(merge_polyreader);
+ Snapshot *latest_snapshot
+ = Snapshot_Read_File(Snapshot_new(), self->folder, NULL);
+ int64_t new_seg_num
+ = IxManager_Highest_Seg_Num(self->manager, latest_snapshot) + 1;
+ Segment *new_segment = Seg_new(new_seg_num);
+ SegWriter *seg_writer = SegWriter_new(self->schema, self->snapshot,
+ new_segment, merge_polyreader);
+ DeletionsWriter *del_writer = SegWriter_Get_Del_Writer(seg_writer);
+ int64_t merge_seg_num = Seg_Get_Number(self->segment);
+ uint32_t seg_tick = I32_MAX;
+ int32_t offset = I32_MAX;
+ CharBuf *seg_name = NULL;
+ Matcher *deletions = NULL;
+ uint32_t i, max;
+
+ SegWriter_Prep_Seg_Dir(seg_writer);
+
+ for (i = 0, max = VA_Get_Size(merge_seg_readers); i < max; i++) {
+ SegReader *seg_reader
+ = (SegReader*)VA_Fetch(merge_seg_readers, i);
+ if (SegReader_Get_Seg_Num(seg_reader) == merge_seg_num) {
+ I32Array *offsets = PolyReader_Offsets(merge_polyreader);
+ seg_tick = i;
+ offset = I32Arr_Get(offsets, seg_tick);
+ DECREF(offsets);
+ }
+ }
+ if (offset == I32_MAX) { THROW(ERR, "Failed sanity check"); }
+
+ Hash_Iterate(updated_deletions);
+ while (Hash_Next(updated_deletions,
+ (Obj**)&seg_name, (Obj**)&deletions)
+ ) {
+ I32Array *doc_map
+ = (I32Array*)CERTIFY(
+ Hash_Fetch(self->doc_maps, (Obj*)seg_name),
+ I32ARRAY);
+ int32_t del;
+ while (0 != (del = Matcher_Next(deletions))) {
+ // Find the slot where the deleted doc resides in the
+ // rewritten segment. If the doc was already deleted when we
+ // were merging, do nothing.
+ int32_t remapped = I32Arr_Get(doc_map, del);
+ if (remapped) {
+ // It's a new deletion, so carry it forward and zap it in
+ // the rewritten segment.
+ DelWriter_Delete_By_Doc_ID(del_writer, remapped + offset);
+ }
+ }
+ }
+
+ // Finish the segment and clean up.
+ DelWriter_Finish(del_writer);
+ SegWriter_Finish(seg_writer);
+ DECREF(seg_writer);
+ DECREF(new_segment);
+ DECREF(latest_snapshot);
+ DECREF(merge_polyreader);
+ DECREF(updated_deletions);
+ }
+
+ return true;
+}
+
+void
+BGMerger_prepare_commit(BackgroundMerger *self) {
+ VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader);
+ uint32_t num_seg_readers = VA_Get_Size(seg_readers);
+ uint32_t segs_merged = 0;
+
+ if (self->prepared) {
+ THROW(ERR, "Can't call Prepare_Commit() more than once");
+ }
+
+ // Maybe merge existing index data.
+ if (num_seg_readers) {
+ segs_merged = S_maybe_merge(self);
+ }
+
+ if (!segs_merged) {
+ // Nothing merged. Leave self->needs_commit false and bail out.
+ self->prepared = true;
+ return;
+ }
+ // Finish the segment and write a new snapshot file.
+ else {
+ Folder *folder = self->folder;
+ Snapshot *snapshot = self->snapshot;
+
+ // Write out new deletions.
+ if (DelWriter_Updated(self->del_writer)) {
+ // Only write out if they haven't all been applied.
+ if (segs_merged != num_seg_readers) {
+ DelWriter_Finish(self->del_writer);
+ }
+ }
+
+ // Finish the segment.
+ SegWriter_Finish(self->seg_writer);
+
+ // Grab the write lock.
+ S_obtain_write_lock(self);
+ if (!self->write_lock) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+
+ // Write temporary snapshot file.
+ DECREF(self->snapfile);
+ self->snapfile = IxManager_Make_Snapshot_Filename(self->manager);
+ CB_Cat_Trusted_Str(self->snapfile, ".temp", 5);
+ Folder_Delete(folder, self->snapfile);
+ Snapshot_Write_File(snapshot, folder, self->snapfile);
+
+ // Determine whether the index has been updated while this background
+ // merge process was running.
+ {
+ CharBuf *start_snapfile
+ = Snapshot_Get_Path(PolyReader_Get_Snapshot(self->polyreader));
+ Snapshot *latest_snapshot
+ = Snapshot_Read_File(Snapshot_new(), self->folder, NULL);
+ CharBuf *latest_snapfile = Snapshot_Get_Path(latest_snapshot);
+ bool_t index_updated
+ = !CB_Equals(start_snapfile, (Obj*)latest_snapfile);
+
+ if (index_updated) {
+ /* See if new deletions have been applied since this
+ * background merge process started against any of the
+ * segments we just merged away. If that's true, we need to
+ * write another segment which applies the deletions against
+ * the new composite segment.
+ */
+ S_merge_updated_deletions(self);
+
+ // Add the fresh content to our snapshot. (It's important to
+ // run this AFTER S_merge_updated_deletions, because otherwise
+ // we couldn't tell whether the deletion counts changed.)
+ {
+ VArray *files = Snapshot_List(latest_snapshot);
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *file = (CharBuf*)VA_Fetch(files, i);
+ if (CB_Starts_With_Str(file, "seg_", 4)) {
+ int64_t gen = (int64_t)IxFileNames_extract_gen(file);
+ if (gen > self->cutoff) {
+ Snapshot_Add_Entry(self->snapshot, file);
+ }
+ }
+ }
+ DECREF(files);
+ }
+
+ // Since the snapshot content has changed, we need to rewrite
+ // it.
+ Folder_Delete(folder, self->snapfile);
+ Snapshot_Write_File(snapshot, folder, self->snapfile);
+ }
+
+ DECREF(latest_snapshot);
+ }
+
+ self->needs_commit = true;
+ }
+
+ // Close reader, so that we can delete its files if appropriate.
+ PolyReader_Close(self->polyreader);
+
+ self->prepared = true;
+}
+
+void
+BGMerger_commit(BackgroundMerger *self) {
+ // Safety check.
+ if (!self->merge_lock) {
+ THROW(ERR, "Can't call commit() more than once");
+ }
+
+ if (!self->prepared) {
+ BGMerger_Prepare_Commit(self);
+ }
+
+ if (self->needs_commit) {
+ bool_t success = false;
+ CharBuf *temp_snapfile = CB_Clone(self->snapfile);
+
+ // Rename temp snapshot file.
+ CB_Chop(self->snapfile, sizeof(".temp") - 1);
+ success = Folder_Hard_Link(self->folder, temp_snapfile,
+ self->snapfile);
+ Snapshot_Set_Path(self->snapshot, self->snapfile);
+ if (!success) {
+ CharBuf *mess = CB_newf("Can't create hard link from %o to %o",
+ temp_snapfile, self->snapfile);
+ DECREF(temp_snapfile);
+ Err_throw_mess(ERR, mess);
+ }
+ if (!Folder_Delete(self->folder, temp_snapfile)) {
+ CharBuf *mess = CB_newf("Can't delete %o", temp_snapfile);
+ DECREF(temp_snapfile);
+ Err_throw_mess(ERR, mess);
+ }
+ DECREF(temp_snapfile);
+ }
+
+ // Release the merge lock and remove the merge data file.
+ S_release_merge_lock(self);
+ IxManager_Remove_Merge_Data(self->manager);
+
+ if (self->needs_commit) {
+ // Purge obsolete files.
+ FilePurger_Purge(self->file_purger);
+ }
+
+ // Release the write lock.
+ S_release_write_lock(self);
+}
+
+static void
+S_obtain_write_lock(BackgroundMerger *self) {
+ Lock *write_lock = IxManager_Make_Write_Lock(self->manager);
+ Lock_Clear_Stale(write_lock);
+ if (Lock_Obtain(write_lock)) {
+ // Only assign if successful, otherwise DESTROY unlocks -- bad!
+ self->write_lock = write_lock;
+ }
+ else {
+ DECREF(write_lock);
+ }
+}
+
+static void
+S_obtain_merge_lock(BackgroundMerger *self) {
+ Lock *merge_lock = IxManager_Make_Merge_Lock(self->manager);
+ Lock_Clear_Stale(merge_lock);
+ if (Lock_Obtain(merge_lock)) {
+ // Only assign if successful, same rationale as above.
+ self->merge_lock = merge_lock;
+ }
+ else {
+ // We can't get the merge lock, so it seems there must be another
+ // BackgroundMerger running.
+ DECREF(merge_lock);
+ }
+}
+
+static void
+S_release_write_lock(BackgroundMerger *self) {
+ if (self->write_lock) {
+ Lock_Release(self->write_lock);
+ DECREF(self->write_lock);
+ self->write_lock = NULL;
+ }
+}
+
+static void
+S_release_merge_lock(BackgroundMerger *self) {
+ if (self->merge_lock) {
+ Lock_Release(self->merge_lock);
+ DECREF(self->merge_lock);
+ self->merge_lock = NULL;
+ }
+}
+
+
diff --git a/core/Lucy/Index/BackgroundMerger.cfh b/core/Lucy/Index/BackgroundMerger.cfh
new file mode 100644
index 0000000..812050d
--- /dev/null
+++ b/core/Lucy/Index/BackgroundMerger.cfh
@@ -0,0 +1,90 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Consolidate index segments in the background.
+ *
+ * Adding documents to an index is usually fast, but every once in a while the
+ * index must be compacted and an update takes substantially longer to
+ * complete. See L<Lucy::Docs::Cookbook::FastUpdates> for how to use
+ * this class to control worst-case index update performance.
+ *
+ * As with L<Indexer|Lucy::Index::Indexer>, see
+ * L<Lucy::Docs::FileLocking> if your index is on a shared volume.
+ */
+class Lucy::Index::BackgroundMerger cnick BGMerger
+ inherits Lucy::Object::Obj {
+
+ Schema *schema;
+ Folder *folder;
+ Segment *segment;
+ IndexManager *manager;
+ PolyReader *polyreader;
+ Snapshot *snapshot;
+ SegWriter *seg_writer;
+ DeletionsWriter *del_writer;
+ FilePurger *file_purger;
+ Lock *write_lock;
+ Lock *merge_lock;
+ CharBuf *snapfile;
+ Hash *doc_maps;
+ int64_t cutoff;
+ bool_t optimize;
+ bool_t needs_commit;
+ bool_t prepared;
+
+ public inert incremented BackgroundMerger*
+ new(Obj *index, IndexManager *manager = NULL);
+
+ /** Open a new BackgroundMerger.
+ *
+ * @param index Either a string filepath or a Folder.
+ * @param manager An IndexManager. If not supplied, an IndexManager with
+ * a 10-second write lock timeout will be created.
+ */
+ public inert BackgroundMerger*
+ init(BackgroundMerger *self, Obj *index, IndexManager *manager = NULL);
+
+ /** Optimize the index for search-time performance. This may take a
+ * while, as it can involve rewriting large amounts of data.
+ */
+ public void
+ Optimize(BackgroundMerger *self);
+
+ /** Commit any changes made to the index. Until this is called, none of
+ * the changes made during an indexing session are permanent.
+ *
+ * Calls Prepare_Commit() implicitly if it has not already been called.
+ */
+ public void
+ Commit(BackgroundMerger *self);
+
+ /** Perform the expensive setup for Commit() in advance, so that Commit()
+ * completes quickly.
+ *
+ * Towards the end of Prepare_Commit(), the BackgroundMerger attempts to
+ * re-acquire the write lock, which is then held until Commit() finishes
+ * and releases it.
+ */
+ public void
+ Prepare_Commit(BackgroundMerger *self);
+
+ public void
+ Destroy(BackgroundMerger *self);
+}
+
+
diff --git a/core/Lucy/Index/BitVecDelDocs.c b/core/Lucy/Index/BitVecDelDocs.c
new file mode 100644
index 0000000..690e6d5
--- /dev/null
+++ b/core/Lucy/Index/BitVecDelDocs.c
@@ -0,0 +1,60 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BITVECDELDOCS
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/BitVecDelDocs.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+
+BitVecDelDocs*
+BitVecDelDocs_new(Folder *folder, const CharBuf *filename) {
+ BitVecDelDocs *self = (BitVecDelDocs*)VTable_Make_Obj(BITVECDELDOCS);
+ return BitVecDelDocs_init(self, folder, filename);
+}
+
+BitVecDelDocs*
+BitVecDelDocs_init(BitVecDelDocs *self, Folder *folder,
+ const CharBuf *filename) {
+ int32_t len;
+
+ BitVec_init((BitVector*)self, 0);
+ self->filename = CB_Clone(filename);
+ self->instream = Folder_Open_In(folder, filename);
+ if (!self->instream) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(self);
+ RETHROW(error);
+ }
+ len = (int32_t)InStream_Length(self->instream);
+ self->bits = (uint8_t*)InStream_Buf(self->instream, len);
+ self->cap = (uint32_t)(len * 8);
+ return self;
+}
+
+void
+BitVecDelDocs_destroy(BitVecDelDocs *self) {
+ DECREF(self->filename);
+ if (self->instream) {
+ InStream_Close(self->instream);
+ DECREF(self->instream);
+ }
+ self->bits = NULL;
+ SUPER_DESTROY(self, BITVECDELDOCS);
+}
+
+
diff --git a/core/Lucy/Index/BitVecDelDocs.cfh b/core/Lucy/Index/BitVecDelDocs.cfh
new file mode 100644
index 0000000..87f31ae
--- /dev/null
+++ b/core/Lucy/Index/BitVecDelDocs.cfh
@@ -0,0 +1,34 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::BitVecDelDocs inherits Lucy::Object::BitVector {
+
+ InStream *instream;
+ CharBuf *filename;
+
+ inert incremented BitVecDelDocs*
+ new(Folder *folder, const CharBuf *filename);
+
+ inert BitVecDelDocs*
+ init(BitVecDelDocs *self, Folder *folder, const CharBuf *filename);
+
+ public void
+ Destroy(BitVecDelDocs *self);
+}
+
+
diff --git a/core/Lucy/Index/DataReader.c b/core/Lucy/Index/DataReader.c
new file mode 100644
index 0000000..f8df36f
--- /dev/null
+++ b/core/Lucy/Index/DataReader.c
@@ -0,0 +1,95 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DATAREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/DataReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+
+DataReader*
+DataReader_init(DataReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ self->schema = (Schema*)INCREF(schema);
+ self->folder = (Folder*)INCREF(folder);
+ self->snapshot = (Snapshot*)INCREF(snapshot);
+ self->segments = (VArray*)INCREF(segments);
+ self->seg_tick = seg_tick;
+ if (seg_tick != -1) {
+ if (!segments) {
+ THROW(ERR, "No segments array provided, but seg_tick is %i32",
+ seg_tick);
+ }
+ else {
+ Segment *segment = (Segment*)VA_Fetch(segments, seg_tick);
+ if (!segment) {
+ THROW(ERR, "No segment at seg_tick %i32", seg_tick);
+ }
+ self->segment = (Segment*)INCREF(segment);
+ }
+ }
+ else {
+ self->segment = NULL;
+ }
+
+ ABSTRACT_CLASS_CHECK(self, DATAREADER);
+ return self;
+}
+
+void
+DataReader_destroy(DataReader *self) {
+ DECREF(self->schema);
+ DECREF(self->folder);
+ DECREF(self->snapshot);
+ DECREF(self->segments);
+ DECREF(self->segment);
+ SUPER_DESTROY(self, DATAREADER);
+}
+
+Schema*
+DataReader_get_schema(DataReader *self) {
+ return self->schema;
+}
+
+Folder*
+DataReader_get_folder(DataReader *self) {
+ return self->folder;
+}
+
+Snapshot*
+DataReader_get_snapshot(DataReader *self) {
+ return self->snapshot;
+}
+
+VArray*
+DataReader_get_segments(DataReader *self) {
+ return self->segments;
+}
+
+int32_t
+DataReader_get_seg_tick(DataReader *self) {
+ return self->seg_tick;
+}
+
+Segment*
+DataReader_get_segment(DataReader *self) {
+ return self->segment;
+}
+
+
diff --git a/core/Lucy/Index/DataReader.cfh b/core/Lucy/Index/DataReader.cfh
new file mode 100644
index 0000000..818a059
--- /dev/null
+++ b/core/Lucy/Index/DataReader.cfh
@@ -0,0 +1,101 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Abstract base class for reading index data.
+ *
+ * DataReader is the companion class to
+ * L<DataWriter|Lucy::Index::DataWriter>. Every index component will
+ * implement one of each.
+ */
+
+class Lucy::Index::DataReader inherits Lucy::Object::Obj {
+
+ Schema *schema;
+ Folder *folder;
+ Snapshot *snapshot;
+ VArray *segments;
+ Segment *segment;
+ int32_t seg_tick;
+
+ /**
+ * @param schema A Schema.
+ * @param folder A Folder.
+ * @param snapshot A Snapshot.
+ * @param segments An array of Segments.
+ * @param seg_tick The array index of the Segment object within the
+ * <code>segments</code> array that this particular DataReader is assigned
+ * to, if any. A value of -1 indicates that no Segment should be
+ * assigned.
+ */
+ public inert DataReader*
+ init(DataReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Create a reader which aggregates the output of several lower level
+ * readers. Return NULL if such a reader is not valid.
+ *
+ * @param readers An array of DataReaders.
+ * @param offsets Doc id start offsets for each reader.
+ */
+ public abstract incremented nullable DataReader*
+ Aggregator(DataReader *self, VArray *readers, I32Array *offsets);
+
+ /** Accessor for "schema" member var.
+ */
+ public nullable Schema*
+ Get_Schema(DataReader *self);
+
+ /** Accessor for "folder" member var.
+ */
+ public nullable Folder*
+ Get_Folder(DataReader *self);
+
+ /** Accessor for "snapshot" member var.
+ */
+ public nullable Snapshot*
+ Get_Snapshot(DataReader *self);
+
+ /** Accessor for "segments" member var.
+ */
+ public nullable VArray*
+ Get_Segments(DataReader *self);
+
+ /** Accessor for "segment" member var.
+ */
+ public nullable Segment*
+ Get_Segment(DataReader *self);
+
+ /** Accessor for "seg_tick" member var.
+ */
+ public int32_t
+ Get_Seg_Tick(DataReader *self);
+
+ /** Release external resources, e.g. streams. Implementations must be
+ * safe for multiple calls. Once called, no other operations may be
+ * performed upon either the reader or any component subreaders other than
+ * object destruction.
+ */
+ public abstract void
+ Close(DataReader *self);
+
+ public void
+ Destroy(DataReader *self);
+}
+
+
diff --git a/core/Lucy/Index/DataWriter.c b/core/Lucy/Index/DataWriter.c
new file mode 100644
index 0000000..bddc287
--- /dev/null
+++ b/core/Lucy/Index/DataWriter.c
@@ -0,0 +1,97 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DATAWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/DataWriter.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Store/Folder.h"
+
+DataWriter*
+DataWriter_init(DataWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ self->snapshot = (Snapshot*)INCREF(snapshot);
+ self->segment = (Segment*)INCREF(segment);
+ self->polyreader = (PolyReader*)INCREF(polyreader);
+ self->schema = (Schema*)INCREF(schema);
+ self->folder = (Folder*)INCREF(PolyReader_Get_Folder(polyreader));
+ ABSTRACT_CLASS_CHECK(self, DATAWRITER);
+ return self;
+}
+
+void
+DataWriter_destroy(DataWriter *self) {
+ DECREF(self->snapshot);
+ DECREF(self->segment);
+ DECREF(self->polyreader);
+ DECREF(self->schema);
+ DECREF(self->folder);
+ SUPER_DESTROY(self, DATAWRITER);
+}
+
+Snapshot*
+DataWriter_get_snapshot(DataWriter *self) {
+ return self->snapshot;
+}
+
+Segment*
+DataWriter_get_segment(DataWriter *self) {
+ return self->segment;
+}
+
+PolyReader*
+DataWriter_get_polyreader(DataWriter *self) {
+ return self->polyreader;
+}
+
+Schema*
+DataWriter_get_schema(DataWriter *self) {
+ return self->schema;
+}
+
+Folder*
+DataWriter_get_folder(DataWriter *self) {
+ return self->folder;
+}
+
+void
+DataWriter_delete_segment(DataWriter *self, SegReader *reader) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(reader);
+}
+
+void
+DataWriter_merge_segment(DataWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ DataWriter_Add_Segment(self, reader, doc_map);
+ DataWriter_Delete_Segment(self, reader);
+}
+
+Hash*
+DataWriter_metadata(DataWriter *self) {
+ Hash *metadata = Hash_new(0);
+ Hash_Store_Str(metadata, "format", 6,
+ (Obj*)CB_newf("%i32", DataWriter_Format(self)));
+ return metadata;
+}
+
+
diff --git a/core/Lucy/Index/DataWriter.cfh b/core/Lucy/Index/DataWriter.cfh
new file mode 100644
index 0000000..7ae669f
--- /dev/null
+++ b/core/Lucy/Index/DataWriter.cfh
@@ -0,0 +1,146 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Write data to an index.
+ *
+ * DataWriter is an abstract base class for writing index data, generally in
+ * segment-sized chunks. Each component of an index -- e.g. stored fields,
+ * lexicon, postings, deletions -- is represented by a
+ * DataWriter/L<DataReader|Lucy::Index::DataReader> pair.
+ *
+ * Components may be specified per index by subclassing
+ * L<Architecture|Lucy::Plan::Architecture>.
+ */
+
+public class Lucy::Index::DataWriter inherits Lucy::Object::Obj {
+
+ Snapshot *snapshot;
+ Segment *segment;
+ PolyReader *polyreader;
+ Schema *schema;
+ Folder *folder;
+
+ /**
+ * @param snapshot The Snapshot that will be committed at the end of the
+ * indexing session.
+ * @param segment The Segment in progress.
+ * @param polyreader A PolyReader representing all existing data in the
+ * index. (If the index is brand new, the PolyReader will have no
+ * sub-readers).
+ */
+ public inert DataWriter*
+ init(DataWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ /** Process a document, previously inverted by <code>inverter</code>.
+ *
+ * @param inverter An Inverter wrapping an inverted document.
+ * @param doc_id Internal number assigned to this document within the
+ * segment.
+ */
+ public abstract void
+ Add_Inverted_Doc(DataWriter *self, Inverter *inverter, int32_t doc_id);
+
+ /** Add content from an existing segment into the one currently being
+ * written.
+ *
+ * @param reader The SegReader containing content to add.
+ * @param doc_map An array of integers mapping old document ids to
+ * new. Deleted documents are mapped to 0, indicating that they should be
+ * skipped.
+ */
+ public abstract void
+ Add_Segment(DataWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ /** Remove a segment's data. The default implementation is a no-op, as
+ * all files within the segment directory will be automatically deleted.
+ * Subclasses which manage their own files outside of the segment system
+ * should override this method and use it as a trigger for cleaning up
+ * obsolete data.
+ *
+ * @param reader The SegReader containing content to merge, which must
+ * represent a segment which is part of the the current snapshot.
+ */
+ public void
+ Delete_Segment(DataWriter *self, SegReader *reader);
+
+ /** Move content from an existing segment into the one currently being
+ * written.
+ *
+ * The default implementation calls Add_Segment() then Delete_Segment().
+ *
+ * @param reader The SegReader containing content to merge, which must
+ * represent a segment which is part of the the current snapshot.
+ * @param doc_map An array of integers mapping old document ids to
+ * new. Deleted documents are mapped to 0, indicating that they should be
+ * skipped.
+ */
+ public void
+ Merge_Segment(DataWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ /** Complete the segment: close all streams, store metadata, etc.
+ */
+ public abstract void
+ Finish(DataWriter *self);
+
+ /** Arbitrary metadata to be serialized and stored by the Segment. The
+ * default implementation supplies a Hash with a single key-value pair for
+ * "format".
+ */
+ public incremented Hash*
+ Metadata(DataWriter *self);
+
+ /** Every writer must specify a file format revision number, which should
+ * increment each time the format changes. Responsibility for revision
+ * checking is left to the companion DataReader.
+ */
+ public abstract int32_t
+ Format(DataWriter *self);
+
+ /** Accessor for "snapshot" member var.
+ */
+ public Snapshot*
+ Get_Snapshot(DataWriter *self);
+
+ /** Accessor for "segment" member var.
+ */
+ public Segment*
+ Get_Segment(DataWriter *self);
+
+ /** Accessor for "polyreader" member var.
+ */
+ public PolyReader*
+ Get_PolyReader(DataWriter *self);
+
+ /** Accessor for "schema" member var.
+ */
+ public Schema*
+ Get_Schema(DataWriter *self);
+
+ /** Accessor for "folder" member var.
+ */
+ public Folder*
+ Get_Folder(DataWriter *self);
+
+ public void
+ Destroy(DataWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/DeletionsReader.c b/core/Lucy/Index/DeletionsReader.c
new file mode 100644
index 0000000..c059a68
--- /dev/null
+++ b/core/Lucy/Index/DeletionsReader.c
@@ -0,0 +1,209 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DELETIONSREADER
+#define C_LUCY_POLYDELETIONSREADER
+#define C_LUCY_DEFAULTDELETIONSREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/BitVecDelDocs.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/BitVecMatcher.h"
+#include "Lucy/Search/SeriesMatcher.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+DeletionsReader*
+DelReader_init(DeletionsReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ ABSTRACT_CLASS_CHECK(self, DELETIONSREADER);
+ return self;
+}
+
+DeletionsReader*
+DelReader_aggregator(DeletionsReader *self, VArray *readers,
+ I32Array *offsets) {
+ UNUSED_VAR(self);
+ return (DeletionsReader*)PolyDelReader_new(readers, offsets);
+}
+
+PolyDeletionsReader*
+PolyDelReader_new(VArray *readers, I32Array *offsets) {
+ PolyDeletionsReader *self
+ = (PolyDeletionsReader*)VTable_Make_Obj(POLYDELETIONSREADER);
+ return PolyDelReader_init(self, readers, offsets);
+}
+
+PolyDeletionsReader*
+PolyDelReader_init(PolyDeletionsReader *self, VArray *readers,
+ I32Array *offsets) {
+ uint32_t i, max;
+ DelReader_init((DeletionsReader*)self, NULL, NULL, NULL, NULL, -1);
+ self->del_count = 0;
+ for (i = 0, max = VA_Get_Size(readers); i < max; i++) {
+ DeletionsReader *reader = (DeletionsReader*)CERTIFY(
+ VA_Fetch(readers, i), DELETIONSREADER);
+ self->del_count += DelReader_Del_Count(reader);
+ }
+ self->readers = (VArray*)INCREF(readers);
+ self->offsets = (I32Array*)INCREF(offsets);
+ return self;
+}
+
+void
+PolyDelReader_close(PolyDeletionsReader *self) {
+ if (self->readers) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ DeletionsReader *reader
+ = (DeletionsReader*)VA_Fetch(self->readers, i);
+ if (reader) { DelReader_Close(reader); }
+ }
+ VA_Clear(self->readers);
+ }
+}
+
+void
+PolyDelReader_destroy(PolyDeletionsReader *self) {
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYDELETIONSREADER);
+}
+
+int32_t
+PolyDelReader_del_count(PolyDeletionsReader *self) {
+ return self->del_count;
+}
+
+Matcher*
+PolyDelReader_iterator(PolyDeletionsReader *self) {
+ SeriesMatcher *deletions = NULL;
+ if (self->del_count) {
+ uint32_t num_readers = VA_Get_Size(self->readers);
+ VArray *matchers = VA_new(num_readers);
+ uint32_t i;
+ for (i = 0; i < num_readers; i++) {
+ DeletionsReader *reader
+ = (DeletionsReader*)VA_Fetch(self->readers, i);
+ Matcher *matcher = DelReader_Iterator(reader);
+ if (matcher) { VA_Store(matchers, i, (Obj*)matcher); }
+ }
+ deletions = SeriesMatcher_new(matchers, self->offsets);
+ DECREF(matchers);
+ }
+ return (Matcher*)deletions;
+}
+
+DefaultDeletionsReader*
+DefDelReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick) {
+ DefaultDeletionsReader *self
+ = (DefaultDeletionsReader*)VTable_Make_Obj(DEFAULTDELETIONSREADER);
+ return DefDelReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+DefaultDeletionsReader*
+DefDelReader_init(DefaultDeletionsReader *self, Schema *schema,
+ Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick) {
+ DelReader_init((DeletionsReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ DefDelReader_Read_Deletions(self);
+ if (!self->deldocs) {
+ self->del_count = 0;
+ self->deldocs = BitVec_new(0);
+ }
+ return self;
+}
+
+void
+DefDelReader_close(DefaultDeletionsReader *self) {
+ DECREF(self->deldocs);
+ self->deldocs = NULL;
+}
+
+void
+DefDelReader_destroy(DefaultDeletionsReader *self) {
+ DECREF(self->deldocs);
+ SUPER_DESTROY(self, DEFAULTDELETIONSREADER);
+}
+
+BitVector*
+DefDelReader_read_deletions(DefaultDeletionsReader *self) {
+ VArray *segments = DefDelReader_Get_Segments(self);
+ Segment *segment = DefDelReader_Get_Segment(self);
+ CharBuf *my_seg_name = Seg_Get_Name(segment);
+ CharBuf *del_file = NULL;
+ int32_t del_count = 0;
+ int32_t i;
+
+ // Start with deletions files in the most recently added segments and work
+ // backwards. The first one we find which addresses our segment is the
+ // one we need.
+ for (i = VA_Get_Size(segments) - 1; i >= 0; i--) {
+ Segment *other_seg = (Segment*)VA_Fetch(segments, i);
+ Hash *metadata
+ = (Hash*)Seg_Fetch_Metadata_Str(other_seg, "deletions", 9);
+ if (metadata) {
+ Hash *files = (Hash*)CERTIFY(
+ Hash_Fetch_Str(metadata, "files", 5), HASH);
+ Hash *seg_files_data
+ = (Hash*)Hash_Fetch(files, (Obj*)my_seg_name);
+ if (seg_files_data) {
+ Obj *count = (Obj*)CERTIFY(
+ Hash_Fetch_Str(seg_files_data, "count", 5),
+ OBJ);
+ del_count = (int32_t)Obj_To_I64(count);
+ del_file = (CharBuf*)CERTIFY(
+ Hash_Fetch_Str(seg_files_data, "filename", 8),
+ CHARBUF);
+ break;
+ }
+ }
+ }
+
+ DECREF(self->deldocs);
+ if (del_file) {
+ self->deldocs = (BitVector*)BitVecDelDocs_new(self->folder, del_file);
+ self->del_count = del_count;
+ }
+ else {
+ self->deldocs = NULL;
+ self->del_count = 0;
+ }
+
+ return self->deldocs;
+}
+
+Matcher*
+DefDelReader_iterator(DefaultDeletionsReader *self) {
+ return (Matcher*)BitVecMatcher_new(self->deldocs);
+}
+
+int32_t
+DefDelReader_del_count(DefaultDeletionsReader *self) {
+ return self->del_count;
+}
+
+
diff --git a/core/Lucy/Index/DeletionsReader.cfh b/core/Lucy/Index/DeletionsReader.cfh
new file mode 100644
index 0000000..00d9f44
--- /dev/null
+++ b/core/Lucy/Index/DeletionsReader.cfh
@@ -0,0 +1,99 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::DeletionsReader cnick DelReader
+ inherits Lucy::Index::DataReader {
+
+ inert DeletionsReader*
+ init(DeletionsReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Return the number of docs which have been marked as deleted in this
+ * segment.
+ */
+ abstract int32_t
+ Del_Count(DeletionsReader *self);
+
+ /** Return a Matcher which iterates over the set of all deleted doc nums
+ * for this segment.
+ */
+ abstract incremented Matcher*
+ Iterator(DeletionsReader *self);
+
+ public incremented nullable DeletionsReader*
+ Aggregator(DeletionsReader *self, VArray *readers, I32Array *offsets);
+}
+
+class Lucy::Index::PolyDeletionsReader cnick PolyDelReader
+ inherits Lucy::Index::DeletionsReader {
+
+ VArray *readers;
+ I32Array *offsets;
+ int32_t del_count;
+
+ inert incremented PolyDeletionsReader*
+ new(VArray *readers, I32Array *offsets);
+
+ inert PolyDeletionsReader*
+ init(PolyDeletionsReader *self, VArray *readers, I32Array *offsets);
+
+ int32_t
+ Del_Count(PolyDeletionsReader *self);
+
+ incremented Matcher*
+ Iterator(PolyDeletionsReader *self);
+
+ public void
+ Close(PolyDeletionsReader *self);
+
+ public void
+ Destroy(PolyDeletionsReader *self);
+}
+
+class Lucy::Index::DefaultDeletionsReader cnick DefDelReader
+ inherits Lucy::Index::DeletionsReader {
+
+ BitVector *deldocs;
+ int32_t del_count;
+
+ inert incremented DefaultDeletionsReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultDeletionsReader*
+ init(DefaultDeletionsReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ int32_t
+ Del_Count(DefaultDeletionsReader *self);
+
+ incremented Matcher*
+ Iterator(DefaultDeletionsReader *self);
+
+ nullable BitVector*
+ Read_Deletions(DefaultDeletionsReader *self);
+
+ public void
+ Close(DefaultDeletionsReader *self);
+
+ public void
+ Destroy(DefaultDeletionsReader *self);
+}
+
+
diff --git a/core/Lucy/Index/DeletionsWriter.c b/core/Lucy/Index/DeletionsWriter.c
new file mode 100644
index 0000000..d866854
--- /dev/null
+++ b/core/Lucy/Index/DeletionsWriter.c
@@ -0,0 +1,383 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DELETIONSWRITER
+#define C_LUCY_DEFAULTDELETIONSWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include <math.h>
+
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/IndexReader.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/BitVecMatcher.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/IndexSearcher.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+
+DeletionsWriter*
+DelWriter_init(DeletionsWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ ABSTRACT_CLASS_CHECK(self, DELETIONSWRITER);
+ return self;
+}
+
+I32Array*
+DelWriter_generate_doc_map(DeletionsWriter *self, Matcher *deletions,
+ int32_t doc_max, int32_t offset) {
+ int32_t *doc_map = (int32_t*)CALLOCATE(doc_max + 1, sizeof(int32_t));
+ int32_t new_doc_id;
+ int32_t i;
+ int32_t next_deletion = deletions ? Matcher_Next(deletions) : I32_MAX;
+ UNUSED_VAR(self);
+
+ // 0 for a deleted doc, a new number otherwise
+ for (i = 1, new_doc_id = 1; i <= doc_max; i++) {
+ if (i == next_deletion) {
+ next_deletion = Matcher_Next(deletions);
+ }
+ else {
+ doc_map[i] = offset + new_doc_id++;
+ }
+ }
+
+ return I32Arr_new_steal(doc_map, doc_max + 1);
+}
+
+int32_t DefDelWriter_current_file_format = 1;
+
+DefaultDeletionsWriter*
+DefDelWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ DefaultDeletionsWriter *self
+ = (DefaultDeletionsWriter*)VTable_Make_Obj(DEFAULTDELETIONSWRITER);
+ return DefDelWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+DefaultDeletionsWriter*
+DefDelWriter_init(DefaultDeletionsWriter *self, Schema *schema,
+ Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ uint32_t i;
+ uint32_t num_seg_readers;
+
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ self->seg_readers = PolyReader_Seg_Readers(polyreader);
+ num_seg_readers = VA_Get_Size(self->seg_readers);
+ self->seg_starts = PolyReader_Offsets(polyreader);
+ self->bit_vecs = VA_new(num_seg_readers);
+ self->updated = (bool_t*)CALLOCATE(num_seg_readers, sizeof(bool_t));
+ self->searcher = IxSearcher_new((Obj*)polyreader);
+ self->name_to_tick = Hash_new(num_seg_readers);
+
+ // Materialize a BitVector of deletions for each segment.
+ for (i = 0; i < num_seg_readers; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ BitVector *bit_vec = BitVec_new(SegReader_Doc_Max(seg_reader));
+ DeletionsReader *del_reader
+ = (DeletionsReader*)SegReader_Fetch(
+ seg_reader, VTable_Get_Name(DELETIONSREADER));
+ Matcher *seg_dels = del_reader
+ ? DelReader_Iterator(del_reader)
+ : NULL;
+
+ if (seg_dels) {
+ int32_t del;
+ while (0 != (del = Matcher_Next(seg_dels))) {
+ BitVec_Set(bit_vec, del);
+ }
+ DECREF(seg_dels);
+ }
+ VA_Store(self->bit_vecs, i, (Obj*)bit_vec);
+ Hash_Store(self->name_to_tick,
+ (Obj*)SegReader_Get_Seg_Name(seg_reader),
+ (Obj*)Int32_new(i));
+ }
+
+ return self;
+}
+
+void
+DefDelWriter_destroy(DefaultDeletionsWriter *self) {
+ DECREF(self->seg_readers);
+ DECREF(self->seg_starts);
+ DECREF(self->bit_vecs);
+ DECREF(self->searcher);
+ DECREF(self->name_to_tick);
+ FREEMEM(self->updated);
+ SUPER_DESTROY(self, DEFAULTDELETIONSWRITER);
+}
+
+static CharBuf*
+S_del_filename(DefaultDeletionsWriter *self, SegReader *target_reader) {
+ Segment *target_seg = SegReader_Get_Segment(target_reader);
+ return CB_newf("%o/deletions-%o.bv", Seg_Get_Name(self->segment),
+ Seg_Get_Name(target_seg));
+}
+
+void
+DefDelWriter_finish(DefaultDeletionsWriter *self) {
+ Folder *const folder = self->folder;
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ if (self->updated[i]) {
+ BitVector *deldocs = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ int32_t doc_max = SegReader_Doc_Max(seg_reader);
+ double used = (doc_max + 1) / 8.0;
+ uint32_t byte_size = (uint32_t)ceil(used);
+ uint32_t new_max = byte_size * 8 - 1;
+ CharBuf *filename = S_del_filename(self, seg_reader);
+ OutStream *outstream = Folder_Open_Out(folder, filename);
+ if (!outstream) { RETHROW(INCREF(Err_get_error())); }
+
+ // Ensure that we have 1 bit for each doc in segment.
+ BitVec_Grow(deldocs, new_max);
+
+ // Write deletions data and clean up.
+ OutStream_Write_Bytes(outstream,
+ (char*)BitVec_Get_Raw_Bits(deldocs),
+ byte_size);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ DECREF(filename);
+ }
+ }
+
+ Seg_Store_Metadata_Str(self->segment, "deletions", 9,
+ (Obj*)DefDelWriter_Metadata(self));
+}
+
+Hash*
+DefDelWriter_metadata(DefaultDeletionsWriter *self) {
+ Hash *const metadata = DataWriter_metadata((DataWriter*)self);
+ Hash *const files = Hash_new(0);
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ if (self->updated[i]) {
+ BitVector *deldocs = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ Segment *segment = SegReader_Get_Segment(seg_reader);
+ Hash *mini_meta = Hash_new(2);
+ Hash_Store_Str(mini_meta, "count", 5,
+ (Obj*)CB_newf("%u32", (uint32_t)BitVec_Count(deldocs)));
+ Hash_Store_Str(mini_meta, "filename", 8,
+ (Obj*)S_del_filename(self, seg_reader));
+ Hash_Store(files, (Obj*)Seg_Get_Name(segment), (Obj*)mini_meta);
+ }
+ }
+ Hash_Store_Str(metadata, "files", 5, (Obj*)files);
+
+ return metadata;
+}
+
+int32_t
+DefDelWriter_format(DefaultDeletionsWriter *self) {
+ UNUSED_VAR(self);
+ return DefDelWriter_current_file_format;
+}
+
+Matcher*
+DefDelWriter_seg_deletions(DefaultDeletionsWriter *self,
+ SegReader *seg_reader) {
+ Matcher *deletions = NULL;
+ Segment *segment = SegReader_Get_Segment(seg_reader);
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ Integer32 *tick_obj = (Integer32*)Hash_Fetch(self->name_to_tick,
+ (Obj*)seg_name);
+ int32_t tick = tick_obj ? Int32_Get_Value(tick_obj) : 0;
+ SegReader *candidate = tick_obj
+ ? (SegReader*)VA_Fetch(self->seg_readers, tick)
+ : NULL;
+
+ if (tick_obj) {
+ DeletionsReader *del_reader
+ = (DeletionsReader*)SegReader_Obtain(
+ candidate, VTable_Get_Name(DELETIONSREADER));
+ if (self->updated[tick] || DelReader_Del_Count(del_reader)) {
+ BitVector *deldocs = (BitVector*)VA_Fetch(self->bit_vecs, tick);
+ deletions = (Matcher*)BitVecMatcher_new(deldocs);
+ }
+ }
+ else { // Sanity check.
+ THROW(ERR, "Couldn't find SegReader %o", seg_reader);
+ }
+
+ return deletions;
+}
+
+int32_t
+DefDelWriter_seg_del_count(DefaultDeletionsWriter *self,
+ const CharBuf *seg_name) {
+ Integer32 *tick
+ = (Integer32*)Hash_Fetch(self->name_to_tick, (Obj*)seg_name);
+ BitVector *deldocs = tick
+ ? (BitVector*)VA_Fetch(self->bit_vecs, Int32_Get_Value(tick))
+ : NULL;
+ return deldocs ? BitVec_Count(deldocs) : 0;
+}
+
+void
+DefDelWriter_delete_by_term(DefaultDeletionsWriter *self,
+ const CharBuf *field, Obj *term) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ PostingListReader *plist_reader
+ = (PostingListReader*)SegReader_Fetch(
+ seg_reader, VTable_Get_Name(POSTINGLISTREADER));
+ BitVector *bit_vec = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ PostingList *plist = plist_reader
+ ? PListReader_Posting_List(plist_reader, field, term)
+ : NULL;
+ int32_t doc_id;
+ int32_t num_zapped = 0;
+
+ // Iterate through postings, marking each doc as deleted.
+ if (plist) {
+ while (0 != (doc_id = PList_Next(plist))) {
+ num_zapped += !BitVec_Get(bit_vec, doc_id);
+ BitVec_Set(bit_vec, doc_id);
+ }
+ if (num_zapped) { self->updated[i] = true; }
+ DECREF(plist);
+ }
+ }
+}
+
+void
+DefDelWriter_delete_by_query(DefaultDeletionsWriter *self, Query *query) {
+ Compiler *compiler = Query_Make_Compiler(query, (Searcher*)self->searcher,
+ Query_Get_Boost(query));
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->seg_readers, i);
+ BitVector *bit_vec = (BitVector*)VA_Fetch(self->bit_vecs, i);
+ Matcher *matcher = Compiler_Make_Matcher(compiler, seg_reader, false);
+
+ if (matcher) {
+ int32_t doc_id;
+ int32_t num_zapped = 0;
+
+ // Iterate through matches, marking each doc as deleted.
+ while (0 != (doc_id = Matcher_Next(matcher))) {
+ num_zapped += !BitVec_Get(bit_vec, doc_id);
+ BitVec_Set(bit_vec, doc_id);
+ }
+ if (num_zapped) { self->updated[i] = true; }
+
+ DECREF(matcher);
+ }
+ }
+
+ DECREF(compiler);
+}
+
+void
+DefDelWriter_delete_by_doc_id(DefaultDeletionsWriter *self, int32_t doc_id) {
+ uint32_t sub_tick = PolyReader_sub_tick(self->seg_starts, doc_id);
+ BitVector *bit_vec = (BitVector*)VA_Fetch(self->bit_vecs, sub_tick);
+ uint32_t offset = I32Arr_Get(self->seg_starts, sub_tick);
+ int32_t seg_doc_id = doc_id - offset;
+
+ if (!BitVec_Get(bit_vec, seg_doc_id)) {
+ self->updated[sub_tick] = true;
+ BitVec_Set(bit_vec, seg_doc_id);
+ }
+}
+
+bool_t
+DefDelWriter_updated(DefaultDeletionsWriter *self) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->seg_readers); i < max; i++) {
+ if (self->updated[i]) { return true; }
+ }
+ return false;
+}
+
+void
+DefDelWriter_add_segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ // This method is a no-op, because the only reason it would be called is
+ // if we are adding an entire index. If that's the case, all deletes are
+ // already being applied.
+ UNUSED_VAR(self);
+ UNUSED_VAR(reader);
+ UNUSED_VAR(doc_map);
+}
+
+void
+DefDelWriter_merge_segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ UNUSED_VAR(doc_map);
+ Segment *segment = SegReader_Get_Segment(reader);
+ Hash *del_meta = (Hash*)Seg_Fetch_Metadata_Str(segment, "deletions", 9);
+
+ if (del_meta) {
+ VArray *seg_readers = self->seg_readers;
+ Hash *files = (Hash*)Hash_Fetch_Str(del_meta, "files", 5);
+ if (files) {
+ CharBuf *seg;
+ Hash *mini_meta;
+ Hash_Iterate(files);
+ while (Hash_Next(files, (Obj**)&seg, (Obj**)&mini_meta)) {
+ uint32_t i, max;
+
+ /* Find the segment the deletions from the SegReader
+ * we're adding correspond to. If it's gone, we don't
+ * need to worry about losing deletions files that point
+ * at it. */
+ for (i = 0, max = VA_Get_Size(seg_readers); i < max; i++) {
+ SegReader *candidate
+ = (SegReader*)VA_Fetch(seg_readers, i);
+ CharBuf *candidate_name
+ = Seg_Get_Name(SegReader_Get_Segment(candidate));
+
+ if (CB_Equals(seg, (Obj*)candidate_name)) {
+ /* If the count hasn't changed, we're about to
+ * merge away the most recent deletions file
+ * pointing at this target segment -- so force a
+ * new file to be written out. */
+ int32_t count = (int32_t)Obj_To_I64(Hash_Fetch_Str(mini_meta, "count", 5));
+ DeletionsReader *del_reader
+ = (DeletionsReader*)SegReader_Obtain(
+ candidate, VTable_Get_Name(DELETIONSREADER));
+ if (count == DelReader_Del_Count(del_reader)) {
+ self->updated[i] = true;
+ }
+ break;
+ }
+ }
+ }
+ }
+ }
+}
+
+
diff --git a/core/Lucy/Index/DeletionsWriter.cfh b/core/Lucy/Index/DeletionsWriter.cfh
new file mode 100644
index 0000000..15a0bfd
--- /dev/null
+++ b/core/Lucy/Index/DeletionsWriter.cfh
@@ -0,0 +1,166 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Abstract base class for marking documents as deleted.
+ *
+ * Subclasses of DeletionsWriter provide a low-level mechanism for declaring a
+ * document deleted from an index.
+ *
+ * Because files in an index are never modified, and because it is not
+ * practical to delete entire segments, a DeletionsWriter does not actually
+ * remove documents from the index. Instead, it communicates to a search-time
+ * companion DeletionsReader which documents are deleted in such a way that it
+ * can create a Matcher iterator.
+ *
+ * Documents are truly deleted only when the segments which contain them are
+ * merged into new ones.
+ */
+
+abstract class Lucy::Index::DeletionsWriter cnick DelWriter
+ inherits Lucy::Index::DataWriter {
+
+ inert DeletionsWriter*
+ init(DeletionsWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ /** Delete all documents in the index that index the supplied term.
+ *
+ * @param field The name of an indexed field. (If it is not spec'd as
+ * <code>indexed</code>, an error will occur.)
+ * @param term The term which identifies docs to be marked as deleted. If
+ * <code>field</code> is associated with an Analyzer, <code>term</code>
+ * will be processed automatically (so don't pre-process it yourself).
+ */
+ public abstract void
+ Delete_By_Term(DeletionsWriter *self, const CharBuf *field, Obj *term);
+
+ /** Delete all documents in the index that match <code>query</code>.
+ *
+ * @param query A L<Query|Lucy::Search::Query>.
+ */
+ public abstract void
+ Delete_By_Query(DeletionsWriter *self, Query *query);
+
+ /** Delete the document identified in the PolyReader by the supplied id.
+ */
+ public abstract void
+ Delete_By_Doc_ID(DeletionsWriter *self, int32_t doc_id);
+
+ /** Returns true if there are updates that need to be written.
+ */
+ public abstract bool_t
+ Updated(DeletionsWriter *self);
+
+ /** Produce an array of int32_t which wraps around deleted documents. The
+ * position in the array represents the original doc id, and the value
+ * represents the new doc id. Deleted docs are assigned the value - 0, so
+ * if you had 4 docs and doc 2 was deleted, the array would have the
+ * values... (1, 0, 2, 3).
+ *
+ * @param offset Value which gets added to each valid document id.
+ * With an offset of 1000, the array in the previous example would be
+ * { 1001, 0, 1002, 1003 }.
+ */
+ public incremented I32Array*
+ Generate_Doc_Map(DeletionsWriter *self, Matcher *deletions,
+ int32_t doc_max, int32_t offset);
+
+ /** Return a deletions iterator for the supplied SegReader, which must be
+ * a component within the PolyReader that was supplied at
+ * construction-time.
+ */
+ public abstract incremented nullable Matcher*
+ Seg_Deletions(DeletionsWriter *self, SegReader *seg_reader);
+
+ /** Return the number of deletions for a given segment.
+ *
+ * @param seg_name The name of the segment.
+ */
+ public abstract int32_t
+ Seg_Del_Count(DeletionsWriter *self, const CharBuf *seg_name);
+}
+
+/** Implements DeletionsWriter using BitVector files.
+ */
+class Lucy::Index::DefaultDeletionsWriter cnick DefDelWriter
+ inherits Lucy::Index::DeletionsWriter {
+
+ VArray *seg_readers;
+ Hash *name_to_tick;
+ I32Array *seg_starts;
+ VArray *bit_vecs;
+ bool_t *updated;
+ IndexSearcher *searcher;
+
+ inert int32_t current_file_format;
+
+ /**
+ * @param schema A Schema.
+ * @param segment A Segment.
+ * @param snapshot A Snapshot.
+ * @param polyreader An PolyReader.
+ */
+ inert incremented DefaultDeletionsWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert DefaultDeletionsWriter*
+ init(DefaultDeletionsWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ public void
+ Delete_By_Term(DefaultDeletionsWriter *self, const CharBuf *field,
+ Obj *term);
+
+ public void
+ Delete_By_Query(DefaultDeletionsWriter *self, Query *query);
+
+ public void
+ Delete_By_Doc_ID(DefaultDeletionsWriter *self, int32_t doc_id);
+
+ public bool_t
+ Updated(DefaultDeletionsWriter *self);
+
+ public incremented nullable Matcher*
+ Seg_Deletions(DefaultDeletionsWriter *self, SegReader *seg_reader);
+
+ public int32_t
+ Seg_Del_Count(DefaultDeletionsWriter *self, const CharBuf *seg_name);
+
+ public void
+ Add_Segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Merge_Segment(DefaultDeletionsWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Finish(DefaultDeletionsWriter *self);
+
+ public int32_t
+ Format(DefaultDeletionsWriter* self);
+
+ public incremented Hash*
+ Metadata(DefaultDeletionsWriter *self);
+
+ public void
+ Destroy(DefaultDeletionsWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/DocReader.c b/core/Lucy/Index/DocReader.c
new file mode 100644
index 0000000..7d6ddfd
--- /dev/null
+++ b/core/Lucy/Index/DocReader.c
@@ -0,0 +1,204 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DOCREADER
+#define C_LUCY_POLYDOCREADER
+#define C_LUCY_DEFAULTDOCREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/DocReader.h"
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Index/DocWriter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+
+DocReader*
+DocReader_init(DocReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ return (DocReader*)DataReader_init((DataReader*)self, schema, folder,
+ snapshot, segments, seg_tick);
+}
+
+DocReader*
+DocReader_aggregator(DocReader *self, VArray *readers, I32Array *offsets) {
+ UNUSED_VAR(self);
+ return (DocReader*)PolyDocReader_new(readers, offsets);
+}
+
+PolyDocReader*
+PolyDocReader_new(VArray *readers, I32Array *offsets) {
+ PolyDocReader *self = (PolyDocReader*)VTable_Make_Obj(POLYDOCREADER);
+ return PolyDocReader_init(self, readers, offsets);
+}
+
+PolyDocReader*
+PolyDocReader_init(PolyDocReader *self, VArray *readers, I32Array *offsets) {
+ uint32_t i, max;
+ DocReader_init((DocReader*)self, NULL, NULL, NULL, NULL, -1);
+ for (i = 0, max = VA_Get_Size(readers); i < max; i++) {
+ CERTIFY(VA_Fetch(readers, i), DOCREADER);
+ }
+ self->readers = (VArray*)INCREF(readers);
+ self->offsets = (I32Array*)INCREF(offsets);
+ return self;
+}
+
+void
+PolyDocReader_close(PolyDocReader *self) {
+ if (self->readers) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ DocReader *reader = (DocReader*)VA_Fetch(self->readers, i);
+ if (reader) { DocReader_Close(reader); }
+ }
+ VA_Clear(self->readers);
+ }
+}
+
+void
+PolyDocReader_destroy(PolyDocReader *self) {
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYDOCREADER);
+}
+
+HitDoc*
+PolyDocReader_fetch_doc(PolyDocReader *self, int32_t doc_id) {
+ uint32_t seg_tick = PolyReader_sub_tick(self->offsets, doc_id);
+ int32_t offset = I32Arr_Get(self->offsets, seg_tick);
+ DocReader *doc_reader = (DocReader*)VA_Fetch(self->readers, seg_tick);
+ HitDoc *hit_doc = NULL;
+ if (!doc_reader) {
+ THROW(ERR, "Invalid doc_id: %i32", doc_id);
+ }
+ else {
+ hit_doc = DocReader_Fetch_Doc(doc_reader, doc_id - offset);
+ HitDoc_Set_Doc_ID(hit_doc, doc_id);
+ }
+ return hit_doc;
+}
+
+DefaultDocReader*
+DefDocReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick) {
+ DefaultDocReader *self
+ = (DefaultDocReader*)VTable_Make_Obj(DEFAULTDOCREADER);
+ return DefDocReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+void
+DefDocReader_close(DefaultDocReader *self) {
+ if (self->dat_in != NULL) {
+ InStream_Close(self->dat_in);
+ DECREF(self->dat_in);
+ self->dat_in = NULL;
+ }
+ if (self->ix_in != NULL) {
+ InStream_Close(self->ix_in);
+ DECREF(self->ix_in);
+ self->ix_in = NULL;
+ }
+}
+
+void
+DefDocReader_destroy(DefaultDocReader *self) {
+ DECREF(self->ix_in);
+ DECREF(self->dat_in);
+ SUPER_DESTROY(self, DEFAULTDOCREADER);
+}
+
+DefaultDocReader*
+DefDocReader_init(DefaultDocReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ Hash *metadata;
+ Segment *segment;
+ DocReader_init((DocReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ segment = DefDocReader_Get_Segment(self);
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "documents", 9);
+
+ if (metadata) {
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *ix_file = CB_newf("%o/documents.ix", seg_name);
+ CharBuf *dat_file = CB_newf("%o/documents.dat", seg_name);
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+
+ // Check format.
+ if (!format) { THROW(ERR, "Missing 'format' var"); }
+ else {
+ int64_t format_val = Obj_To_I64(format);
+ if (format_val < DocWriter_current_file_format) {
+ THROW(ERR, "Obsolete doc storage format %i64; "
+ "Index regeneration is required", format_val);
+ }
+ else if (format_val != DocWriter_current_file_format) {
+ THROW(ERR, "Unsupported doc storage format: %i64", format_val);
+ }
+ }
+
+ // Get streams.
+ if (Folder_Exists(folder, ix_file)) {
+ self->ix_in = Folder_Open_In(folder, ix_file);
+ if (!self->ix_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->dat_in = Folder_Open_In(folder, dat_file);
+ if (!self->dat_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ }
+ DECREF(ix_file);
+ DECREF(dat_file);
+ }
+
+ return self;
+}
+
+void
+DefDocReader_read_record(DefaultDocReader *self, ByteBuf *buffer,
+ int32_t doc_id) {
+ int64_t start;
+ int64_t end;
+ size_t size;
+ char *buf;
+
+ // Find start and length of variable length record.
+ InStream_Seek(self->ix_in, (int64_t)doc_id * 8);
+ start = InStream_Read_I64(self->ix_in);
+ end = InStream_Read_I64(self->ix_in);
+ size = (size_t)(end - start);
+
+ // Read in the record.
+ buf = BB_Grow(buffer, size);
+ InStream_Seek(self->dat_in, start);
+ InStream_Read_Bytes(self->dat_in, buf, size);
+ BB_Set_Size(buffer, size);
+}
+
+
diff --git a/core/Lucy/Index/DocReader.cfh b/core/Lucy/Index/DocReader.cfh
new file mode 100644
index 0000000..ee712b1
--- /dev/null
+++ b/core/Lucy/Index/DocReader.cfh
@@ -0,0 +1,102 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Retrieve stored documents.
+ *
+ * DocReader defines the interface by which documents (with all stored fields)
+ * are retrieved from the index. The default implementation returns
+ * L<HitDoc|Lucy::Document::HitDoc> objects.
+ */
+class Lucy::Index::DocReader inherits Lucy::Index::DataReader {
+
+ inert DocReader*
+ init(DocReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Retrieve the document identified by <code>doc_id</code>.
+ *
+ * @return a HitDoc.
+ */
+ public abstract incremented HitDoc*
+ Fetch_Doc(DocReader *self, int32_t doc_id);
+
+ /** Returns a DocReader which divvies up requests to its sub-readers
+ * according to the offset range.
+ *
+ * @param readers An array of DocReaders.
+ * @param offsets Doc id start offsets for each reader.
+ */
+ public incremented nullable DocReader*
+ Aggregator(DocReader *self, VArray *readers, I32Array *offsets);
+}
+
+/** Aggregate multiple DocReaders.
+ */
+class Lucy::Index::PolyDocReader inherits Lucy::Index::DocReader {
+
+ VArray *readers;
+ I32Array *offsets;
+
+ inert incremented PolyDocReader*
+ new(VArray *readers, I32Array *offsets);
+
+ inert PolyDocReader*
+ init(PolyDocReader *self, VArray *readers, I32Array *offsets);
+
+ public incremented HitDoc*
+ Fetch_Doc(PolyDocReader *self, int32_t doc_id);
+
+ public void
+ Close(PolyDocReader *self);
+
+ public void
+ Destroy(PolyDocReader *self);
+}
+
+class Lucy::Index::DefaultDocReader cnick DefDocReader
+ inherits Lucy::Index::DocReader {
+
+ InStream *dat_in;
+ InStream *ix_in;
+
+ inert incremented DefaultDocReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultDocReader*
+ init(DefaultDocReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ public incremented HitDoc*
+ Fetch_Doc(DefaultDocReader *self, int32_t doc_id);
+
+ /** Read the raw byte content for the specified doc into the supplied
+ * buffer.
+ */
+ void
+ Read_Record(DefaultDocReader *self, ByteBuf *buffer, int32_t doc_id);
+
+ public void
+ Close(DefaultDocReader *self);
+
+ public void
+ Destroy(DefaultDocReader *self);
+}
+
+
diff --git a/core/Lucy/Index/DocVector.c b/core/Lucy/Index/DocVector.c
new file mode 100644
index 0000000..7d39ae6
--- /dev/null
+++ b/core/Lucy/Index/DocVector.c
@@ -0,0 +1,194 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DOCVECTOR
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/TermVector.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+// Extract a document's compressed TermVector data into (term_text =>
+// compressed positional data) pairs.
+static Hash*
+S_extract_tv_cache(ByteBuf *field_buf);
+
+// Pull a TermVector object out from compressed positional data.
+static TermVector*
+S_extract_tv_from_tv_buf(const CharBuf *field, const CharBuf *term_text,
+ ByteBuf *tv_buf);
+
+DocVector*
+DocVec_new() {
+ DocVector *self = (DocVector*)VTable_Make_Obj(DOCVECTOR);
+ return DocVec_init(self);
+}
+
+DocVector*
+DocVec_init(DocVector *self) {
+ self->field_bufs = Hash_new(0);
+ self->field_vectors = Hash_new(0);
+ return self;
+}
+
+void
+DocVec_serialize(DocVector *self, OutStream *outstream) {
+ Hash_Serialize(self->field_bufs, outstream);
+ Hash_Serialize(self->field_vectors, outstream);
+}
+
+DocVector*
+DocVec_deserialize(DocVector *self, InStream *instream) {
+ self = self ? self : (DocVector*)VTable_Make_Obj(DOCVECTOR);
+ self->field_bufs = Hash_deserialize(NULL, instream);
+ self->field_vectors = Hash_deserialize(NULL, instream);
+ return self;
+}
+
+void
+DocVec_destroy(DocVector *self) {
+ DECREF(self->field_bufs);
+ DECREF(self->field_vectors);
+ SUPER_DESTROY(self, DOCVECTOR);
+}
+
+void
+DocVec_add_field_buf(DocVector *self, const CharBuf *field,
+ ByteBuf *field_buf) {
+ Hash_Store(self->field_bufs, (Obj*)field, INCREF(field_buf));
+}
+
+ByteBuf*
+DocVec_field_buf(DocVector *self, const CharBuf *field) {
+ return (ByteBuf*)Hash_Fetch(self->field_bufs, (Obj*)field);
+}
+
+VArray*
+DocVec_field_names(DocVector *self) {
+ return Hash_Keys(self->field_bufs);
+}
+
+TermVector*
+DocVec_term_vector(DocVector *self, const CharBuf *field,
+ const CharBuf *term_text) {
+ ByteBuf *tv_buf;
+ Hash *field_vector = (Hash*)Hash_Fetch(self->field_vectors, (Obj*)field);
+
+ // If no cache hit, try to fill cache.
+ if (field_vector == NULL) {
+ ByteBuf *field_buf
+ = (ByteBuf*)Hash_Fetch(self->field_bufs, (Obj*)field);
+
+ // Bail if there's no content or the field isn't highlightable.
+ if (field_buf == NULL) { return NULL; }
+
+ field_vector = S_extract_tv_cache(field_buf);
+ Hash_Store(self->field_vectors, (Obj*)field, (Obj*)field_vector);
+ }
+
+ // Get a buf for the term text or bail.
+ tv_buf = (ByteBuf*)Hash_Fetch(field_vector, (Obj*)term_text);
+ if (tv_buf == NULL) {
+ return NULL;
+ }
+
+ return S_extract_tv_from_tv_buf(field, term_text, tv_buf);
+}
+
+static Hash*
+S_extract_tv_cache(ByteBuf *field_buf) {
+ Hash *tv_cache = Hash_new(0);
+ char *tv_string = BB_Get_Buf(field_buf);
+ int32_t num_terms = NumUtil_decode_c32(&tv_string);
+ CharBuf *text = CB_new(0);
+ int32_t i;
+
+ // Read the number of highlightable terms in the field.
+ for (i = 0; i < num_terms; i++) {
+ char *bookmark_ptr;
+ size_t overlap = NumUtil_decode_c32(&tv_string);
+ size_t len = NumUtil_decode_c32(&tv_string);
+ int32_t num_positions;
+
+ // Decompress the term text.
+ CB_Set_Size(text, overlap);
+ CB_Cat_Trusted_Str(text, tv_string, len);
+ tv_string += len;
+
+ // Get positions & offsets string.
+ bookmark_ptr = tv_string;
+ num_positions = NumUtil_decode_c32(&tv_string);
+ while (num_positions--) {
+ // Leave nums compressed to save a little mem.
+ NumUtil_skip_cint(&tv_string);
+ NumUtil_skip_cint(&tv_string);
+ NumUtil_skip_cint(&tv_string);
+ }
+ len = tv_string - bookmark_ptr;
+
+ // Store the $text => $posdata pair in the output hash.
+ Hash_Store(tv_cache, (Obj*)text,
+ (Obj*)BB_new_bytes(bookmark_ptr, len));
+ }
+ DECREF(text);
+
+ return tv_cache;
+}
+
+static TermVector*
+S_extract_tv_from_tv_buf(const CharBuf *field, const CharBuf *term_text,
+ ByteBuf *tv_buf) {
+ TermVector *retval = NULL;
+ char *posdata = BB_Get_Buf(tv_buf);
+ char *posdata_end = posdata + BB_Get_Size(tv_buf);
+ int32_t *positions = NULL;
+ int32_t *starts = NULL;
+ int32_t *ends = NULL;
+ uint32_t num_pos = 0;
+ uint32_t i;
+
+ if (posdata != posdata_end) {
+ num_pos = NumUtil_decode_c32(&posdata);
+ positions = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ }
+
+ // Expand C32s.
+ for (i = 0; i < num_pos; i++) {
+ positions[i] = NumUtil_decode_c32(&posdata);
+ starts[i] = NumUtil_decode_c32(&posdata);
+ ends[i] = NumUtil_decode_c32(&posdata);
+ }
+
+ if (posdata != posdata_end) {
+ THROW(ERR, "Bad encoding of posdata");
+ }
+ else {
+ I32Array *posits_map = I32Arr_new_steal(positions, num_pos);
+ I32Array *starts_map = I32Arr_new_steal(starts, num_pos);
+ I32Array *ends_map = I32Arr_new_steal(ends, num_pos);
+ retval = TV_new(field, term_text, posits_map, starts_map, ends_map);
+ DECREF(posits_map);
+ DECREF(starts_map);
+ DECREF(ends_map);
+ }
+
+ return retval;
+}
+
+
diff --git a/core/Lucy/Index/DocVector.cfh b/core/Lucy/Index/DocVector.cfh
new file mode 100644
index 0000000..5568700
--- /dev/null
+++ b/core/Lucy/Index/DocVector.cfh
@@ -0,0 +1,60 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** A collection of TermVectors.
+ */
+
+class Lucy::Index::DocVector cnick DocVec
+ inherits Lucy::Object::Obj {
+
+ Hash *field_bufs;
+ Hash *field_vectors;
+
+ /** Constructor.
+ */
+ inert incremented DocVector*
+ new();
+
+ inert DocVector*
+ init(DocVector *self);
+
+ incremented TermVector*
+ Term_Vector(DocVector *self, const CharBuf *field, const CharBuf *term);
+
+ /** Add a compressed, encoded TermVector to the object.
+ */
+ void
+ Add_Field_Buf(DocVector *self, const CharBuf *field, ByteBuf *field_buf);
+
+ /** Return the compressed, encoded TermVector associated with a particular
+ * field.
+ */
+ ByteBuf*
+ Field_Buf(DocVector *self, const CharBuf *field);
+
+ public void
+ Serialize(DocVector *self, OutStream *outstream);
+
+ public incremented DocVector*
+ Deserialize(DocVector *self, InStream *instream);
+
+ public void
+ Destroy(DocVector *self);
+}
+
+
diff --git a/core/Lucy/Index/DocWriter.c b/core/Lucy/Index/DocWriter.c
new file mode 100644
index 0000000..3f2fd95
--- /dev/null
+++ b/core/Lucy/Index/DocWriter.c
@@ -0,0 +1,186 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DOCWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/DocWriter.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Index/DocReader.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+
+static OutStream*
+S_lazy_init(DocWriter *self);
+
+int32_t DocWriter_current_file_format = 2;
+
+DocWriter*
+DocWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ DocWriter *self = (DocWriter*)VTable_Make_Obj(DOCWRITER);
+ return DocWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+DocWriter*
+DocWriter_init(DocWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ return self;
+}
+
+void
+DocWriter_destroy(DocWriter *self) {
+ DECREF(self->dat_out);
+ DECREF(self->ix_out);
+ SUPER_DESTROY(self, DOCWRITER);
+}
+
+static OutStream*
+S_lazy_init(DocWriter *self) {
+ if (!self->dat_out) {
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+
+ // Get streams.
+ {
+ CharBuf *ix_file = CB_newf("%o/documents.ix", seg_name);
+ self->ix_out = Folder_Open_Out(folder, ix_file);
+ DECREF(ix_file);
+ if (!self->ix_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+ {
+ CharBuf *dat_file = CB_newf("%o/documents.dat", seg_name);
+ self->dat_out = Folder_Open_Out(folder, dat_file);
+ DECREF(dat_file);
+ if (!self->dat_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+
+ // Go past non-doc #0.
+ OutStream_Write_I64(self->ix_out, 0);
+ }
+
+ return self->dat_out;
+}
+
+void
+DocWriter_add_inverted_doc(DocWriter *self, Inverter *inverter,
+ int32_t doc_id) {
+ OutStream *dat_out = S_lazy_init(self);
+ OutStream *ix_out = self->ix_out;
+ uint32_t num_stored = 0;
+ int64_t start = OutStream_Tell(dat_out);
+ int64_t expected = OutStream_Tell(ix_out) / 8;
+
+ // Verify doc id.
+ if (doc_id != expected) {
+ THROW(ERR, "Expected doc id %i64 but got %i32", expected, doc_id);
+ }
+
+ // Write the number of stored fields.
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Stored(type)) { num_stored++; }
+ }
+ OutStream_Write_C32(dat_out, num_stored);
+
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ // Only store fields marked as "stored".
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Stored(type)) {
+ CharBuf *field = Inverter_Get_Field_Name(inverter);
+ Obj *value = Inverter_Get_Value(inverter);
+ CB_Serialize(field, dat_out);
+ Obj_Serialize(value, dat_out);
+ }
+ }
+
+ // Write file pointer.
+ OutStream_Write_I64(ix_out, start);
+}
+
+void
+DocWriter_add_segment(DocWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ int32_t doc_max = SegReader_Doc_Max(reader);
+
+ if (doc_max == 0) {
+ // Bail if the supplied segment is empty.
+ return;
+ }
+ else {
+ OutStream *const dat_out = S_lazy_init(self);
+ OutStream *const ix_out = self->ix_out;
+ ByteBuf *const buffer = BB_new(0);
+ DefaultDocReader *const doc_reader
+ = (DefaultDocReader*)CERTIFY(
+ SegReader_Obtain(reader, VTable_Get_Name(DOCREADER)),
+ DEFAULTDOCREADER);
+ int32_t i, max;
+
+ for (i = 1, max = SegReader_Doc_Max(reader); i <= max; i++) {
+ if (I32Arr_Get(doc_map, i)) {
+ int64_t start = OutStream_Tell(dat_out);
+ char *buf;
+ size_t size;
+
+ // Copy record over.
+ DefDocReader_Read_Record(doc_reader, buffer, i);
+ buf = BB_Get_Buf(buffer);
+ size = BB_Get_Size(buffer);
+ OutStream_Write_Bytes(dat_out, buf, size);
+
+ // Write file pointer.
+ OutStream_Write_I64(ix_out, start);
+ }
+ }
+
+ DECREF(buffer);
+ }
+}
+
+void
+DocWriter_finish(DocWriter *self) {
+ if (self->dat_out) {
+ // Write one final file pointer, so that we can derive the length of
+ // the last record.
+ int64_t end = OutStream_Tell(self->dat_out);
+ OutStream_Write_I64(self->ix_out, end);
+
+ // Close down output streams.
+ OutStream_Close(self->dat_out);
+ OutStream_Close(self->ix_out);
+ Seg_Store_Metadata_Str(self->segment, "documents", 9,
+ (Obj*)DocWriter_Metadata(self));
+ }
+}
+
+int32_t
+DocWriter_format(DocWriter *self) {
+ UNUSED_VAR(self);
+ return DocWriter_current_file_format;
+}
+
+
diff --git a/core/Lucy/Index/DocWriter.cfh b/core/Lucy/Index/DocWriter.cfh
new file mode 100644
index 0000000..a3548b0
--- /dev/null
+++ b/core/Lucy/Index/DocWriter.cfh
@@ -0,0 +1,54 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Default doc writer.
+ */
+class Lucy::Index::DocWriter inherits Lucy::Index::DataWriter {
+
+ OutStream *ix_out;
+ OutStream *dat_out;
+
+ inert int32_t current_file_format;
+
+ /** Constructors.
+ */
+ inert incremented DocWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert DocWriter*
+ init(DocWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ public void
+ Add_Inverted_Doc(DocWriter *self, Inverter *inverter, int32_t doc_id);
+
+ public void
+ Add_Segment(DocWriter *self, SegReader *reader, I32Array *doc_map = NULL);
+
+ public void
+ Finish(DocWriter *self);
+
+ public int32_t
+ Format(DocWriter *self);
+
+ public void
+ Destroy(DocWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/FilePurger.c b/core/Lucy/Index/FilePurger.c
new file mode 100644
index 0000000..4745002
--- /dev/null
+++ b/core/Lucy/Index/FilePurger.c
@@ -0,0 +1,288 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FILEPURGER
+#include <ctype.h>
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/FilePurger.h"
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/Lock.h"
+
+// Place unused files into purgables array and obsolete Snapshots into
+// snapshots array.
+static void
+S_discover_unused(FilePurger *self, VArray **purgables, VArray **snapshots);
+
+// Clean up after a failed background merge session, adding all dead files to
+// the list of candidates to be zapped.
+static void
+S_zap_dead_merge(FilePurger *self, Hash *candidates);
+
+// Return an array of recursively expanded filepath entries.
+static VArray*
+S_find_all_referenced(Folder *folder, VArray *entries);
+
+FilePurger*
+FilePurger_new(Folder *folder, Snapshot *snapshot, IndexManager *manager) {
+ FilePurger *self = (FilePurger*)VTable_Make_Obj(FILEPURGER);
+ return FilePurger_init(self, folder, snapshot, manager);
+}
+
+FilePurger*
+FilePurger_init(FilePurger *self, Folder *folder, Snapshot *snapshot,
+ IndexManager *manager) {
+ self->folder = (Folder*)INCREF(folder);
+ self->snapshot = (Snapshot*)INCREF(snapshot);
+ self->manager = manager
+ ? (IndexManager*)INCREF(manager)
+ : IxManager_new(NULL, NULL);
+ IxManager_Set_Folder(self->manager, folder);
+
+ // Don't allow the locks directory to be zapped.
+ self->disallowed = Hash_new(0);
+ Hash_Store_Str(self->disallowed, "locks", 5, INCREF(&EMPTY));
+
+ return self;
+}
+
+void
+FilePurger_destroy(FilePurger *self) {
+ DECREF(self->folder);
+ DECREF(self->snapshot);
+ DECREF(self->manager);
+ DECREF(self->disallowed);
+ SUPER_DESTROY(self, FILEPURGER);
+}
+
+void
+FilePurger_purge(FilePurger *self) {
+ Lock *deletion_lock = IxManager_Make_Deletion_Lock(self->manager);
+
+ // Obtain deletion lock, purge files, release deletion lock.
+ Lock_Clear_Stale(deletion_lock);
+ if (Lock_Obtain(deletion_lock)) {
+ Folder *folder = self->folder;
+ Hash *failures = Hash_new(0);
+ VArray *purgables;
+ VArray *snapshots;
+
+ S_discover_unused(self, &purgables, &snapshots);
+
+ // Attempt to delete entries -- if failure, no big deal, just try
+ // again later. Proceed in reverse lexical order so that directories
+ // get deleted after they've been emptied.
+ VA_Sort(purgables, NULL, NULL);
+ for (uint32_t i = VA_Get_Size(purgables); i--;) {
+ CharBuf *entry = (CharBuf*)VA_fetch(purgables, i);
+ if (Hash_Fetch(self->disallowed, (Obj*)entry)) { continue; }
+ if (!Folder_Delete(folder, entry)) {
+ if (Folder_Exists(folder, entry)) {
+ Hash_Store(failures, (Obj*)entry, INCREF(&EMPTY));
+ }
+ }
+ }
+
+ for (uint32_t i = 0, max = VA_Get_Size(snapshots); i < max; i++) {
+ Snapshot *snapshot = (Snapshot*)VA_Fetch(snapshots, i);
+ bool_t snapshot_has_failures = false;
+ if (Hash_Get_Size(failures)) {
+ // Only delete snapshot files if all of their entries were
+ // successfully deleted.
+ VArray *entries = Snapshot_List(snapshot);
+ for (uint32_t j = VA_Get_Size(entries); j--;) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(entries, j);
+ if (Hash_Fetch(failures, (Obj*)entry)) {
+ snapshot_has_failures = true;
+ break;
+ }
+ }
+ DECREF(entries);
+ }
+ if (!snapshot_has_failures) {
+ CharBuf *snapfile = Snapshot_Get_Path(snapshot);
+ Folder_Delete(folder, snapfile);
+ }
+ }
+
+ DECREF(failures);
+ DECREF(purgables);
+ DECREF(snapshots);
+ Lock_Release(deletion_lock);
+ }
+ else {
+ WARN("Can't obtain deletion lock, skipping deletion of "
+ "obsolete files");
+ }
+
+ DECREF(deletion_lock);
+}
+
+static void
+S_zap_dead_merge(FilePurger *self, Hash *candidates) {
+ IndexManager *manager = self->manager;
+ Lock *merge_lock = IxManager_Make_Merge_Lock(manager);
+
+ Lock_Clear_Stale(merge_lock);
+ if (!Lock_Is_Locked(merge_lock)) {
+ Hash *merge_data = IxManager_Read_Merge_Data(manager);
+ Obj *cutoff = merge_data
+ ? Hash_Fetch_Str(merge_data, "cutoff", 6)
+ : NULL;
+
+ if (cutoff) {
+ CharBuf *cutoff_seg = Seg_num_to_name(Obj_To_I64(cutoff));
+ if (Folder_Exists(self->folder, cutoff_seg)) {
+ ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
+ DirHandle *dh = Folder_Open_Dir(self->folder, cutoff_seg);
+ CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
+ CharBuf *filepath = CB_new(32);
+
+ if (!dh) {
+ THROW(ERR, "Can't open segment dir '%o'", filepath);
+ }
+
+ Hash_Store(candidates, (Obj*)cutoff_seg, INCREF(&EMPTY));
+ Hash_Store(candidates, (Obj*)merge_json, INCREF(&EMPTY));
+ while (DH_Next(dh)) {
+ // TODO: recursively delete subdirs within seg dir.
+ CB_setf(filepath, "%o/%o", cutoff_seg, entry);
+ Hash_Store(candidates, (Obj*)filepath, INCREF(&EMPTY));
+ }
+ DECREF(filepath);
+ DECREF(dh);
+ }
+ DECREF(cutoff_seg);
+ }
+
+ DECREF(merge_data);
+ }
+
+ DECREF(merge_lock);
+ return;
+}
+
+static void
+S_discover_unused(FilePurger *self, VArray **purgables_ptr,
+ VArray **snapshots_ptr) {
+ Folder *folder = self->folder;
+ DirHandle *dh = Folder_Open_Dir(folder, NULL);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+ VArray *spared = VA_new(1);
+ VArray *snapshots = VA_new(1);
+ CharBuf *snapfile = NULL;
+
+ // Start off with the list of files in the current snapshot.
+ if (self->snapshot) {
+ VArray *entries = Snapshot_List(self->snapshot);
+ VArray *referenced = S_find_all_referenced(folder, entries);
+ VA_Push_VArray(spared, referenced);
+ DECREF(entries);
+ DECREF(referenced);
+ snapfile = Snapshot_Get_Path(self->snapshot);
+ if (snapfile) { VA_Push(spared, INCREF(snapfile)); }
+ }
+
+ CharBuf *entry = DH_Get_Entry(dh);
+ Hash *candidates = Hash_new(64);
+ while (DH_Next(dh)) {
+ if (!CB_Starts_With_Str(entry, "snapshot_", 9)) { continue; }
+ else if (!CB_Ends_With_Str(entry, ".json", 5)) { continue; }
+ else if (snapfile && CB_Equals(entry, (Obj*)snapfile)) { continue; }
+ else {
+ Snapshot *snapshot
+ = Snapshot_Read_File(Snapshot_new(), folder, entry);
+ Lock *lock
+ = IxManager_Make_Snapshot_Read_Lock(self->manager, entry);
+ VArray *snap_list = Snapshot_List(snapshot);
+ VArray *referenced = S_find_all_referenced(folder, snap_list);
+
+ // DON'T obtain the lock -- only see whether another
+ // entity holds a lock on the snapshot file.
+ if (lock) {
+ Lock_Clear_Stale(lock);
+ }
+ if (lock && Lock_Is_Locked(lock)) {
+ // The snapshot file is locked, which means someone's using
+ // that version of the index -- protect all of its entries.
+ uint32_t new_size = VA_Get_Size(spared)
+ + VA_Get_Size(referenced)
+ + 1;
+ VA_Grow(spared, new_size);
+ VA_Push(spared, (Obj*)CB_Clone(entry));
+ VA_Push_VArray(spared, referenced);
+ }
+ else {
+ // No one's using this snapshot, so all of its entries are
+ // candidates for deletion.
+ for (uint32_t i = 0, max = VA_Get_Size(referenced); i < max; i++) {
+ CharBuf *file = (CharBuf*)VA_Fetch(referenced, i);
+ Hash_Store(candidates, (Obj*)file, INCREF(&EMPTY));
+ }
+ VA_Push(snapshots, INCREF(snapshot));
+ }
+
+ DECREF(referenced);
+ DECREF(snap_list);
+ DECREF(snapshot);
+ DECREF(lock);
+ }
+ }
+ DECREF(dh);
+
+ // Clean up after a dead segment consolidation.
+ S_zap_dead_merge(self, candidates);
+
+ // Eliminate any current files from the list of files to be purged.
+ for (uint32_t i = 0, max = VA_Get_Size(spared); i < max; i++) {
+ CharBuf *filename = (CharBuf*)VA_Fetch(spared, i);
+ DECREF(Hash_Delete(candidates, (Obj*)filename));
+ }
+
+ // Pass back purgables and Snapshots.
+ *purgables_ptr = Hash_Keys(candidates);
+ *snapshots_ptr = snapshots;
+
+ DECREF(candidates);
+ DECREF(spared);
+}
+
+static VArray*
+S_find_all_referenced(Folder *folder, VArray *entries) {
+ Hash *uniqued = Hash_new(VA_Get_Size(entries));
+ for (uint32_t i = 0, max = VA_Get_Size(entries); i < max; i++) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(entries, i);
+ Hash_Store(uniqued, (Obj*)entry, INCREF(&EMPTY));
+ if (Folder_Is_Directory(folder, entry)) {
+ VArray *contents = Folder_List_R(folder, entry);
+ for (uint32_t j = VA_Get_Size(contents); j--;) {
+ CharBuf *sub_entry = (CharBuf*)VA_Fetch(contents, j);
+ Hash_Store(uniqued, (Obj*)sub_entry, INCREF(&EMPTY));
+ }
+ DECREF(contents);
+ }
+ }
+ VArray *referenced = Hash_Keys(uniqued);
+ DECREF(uniqued);
+ return referenced;
+}
+
+
diff --git a/core/Lucy/Index/FilePurger.cfh b/core/Lucy/Index/FilePurger.cfh
new file mode 100644
index 0000000..fb05196
--- /dev/null
+++ b/core/Lucy/Index/FilePurger.cfh
@@ -0,0 +1,46 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Delete obsolete index files.
+ */
+
+class Lucy::Index::FilePurger inherits Lucy::Object::Obj {
+
+ Folder *folder;
+ Snapshot *snapshot;
+ IndexManager *manager;
+ Hash *disallowed;
+
+ inert incremented FilePurger*
+ new(Folder *folder, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ inert FilePurger*
+ init(FilePurger *self, Folder *folder, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ /** Purge obsolete files from the index.
+ */
+ void
+ Purge(FilePurger *self);
+
+ public void
+ Destroy(FilePurger *self);
+}
+
+
diff --git a/core/Lucy/Index/HighlightReader.c b/core/Lucy/Index/HighlightReader.c
new file mode 100644
index 0000000..821c8b4
--- /dev/null
+++ b/core/Lucy/Index/HighlightReader.c
@@ -0,0 +1,231 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HIGHLIGHTREADER
+#define C_LUCY_POLYHIGHLIGHTREADER
+#define C_LUCY_DEFAULTHIGHLIGHTREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/HighlightReader.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/HighlightWriter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/Folder.h"
+
+HighlightReader*
+HLReader_init(HighlightReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ ABSTRACT_CLASS_CHECK(self, HIGHLIGHTREADER);
+ return self;
+}
+
+HighlightReader*
+HLReader_aggregator(HighlightReader *self, VArray *readers,
+ I32Array *offsets) {
+ UNUSED_VAR(self);
+ return (HighlightReader*)PolyHLReader_new(readers, offsets);
+}
+
+PolyHighlightReader*
+PolyHLReader_new(VArray *readers, I32Array *offsets) {
+ PolyHighlightReader *self
+ = (PolyHighlightReader*)VTable_Make_Obj(POLYHIGHLIGHTREADER);
+ return PolyHLReader_init(self, readers, offsets);
+}
+
+PolyHighlightReader*
+PolyHLReader_init(PolyHighlightReader *self, VArray *readers,
+ I32Array *offsets) {
+ uint32_t i, max;
+ HLReader_init((HighlightReader*)self, NULL, NULL, NULL, NULL, -1);
+ for (i = 0, max = VA_Get_Size(readers); i < max; i++) {
+ CERTIFY(VA_Fetch(readers, i), HIGHLIGHTREADER);
+ }
+ self->readers = (VArray*)INCREF(readers);
+ self->offsets = (I32Array*)INCREF(offsets);
+ return self;
+}
+
+void
+PolyHLReader_close(PolyHighlightReader *self) {
+ if (self->readers) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ HighlightReader *sub_reader
+ = (HighlightReader*)VA_Fetch(self->readers, i);
+ if (sub_reader) { HLReader_Close(sub_reader); }
+ }
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ self->readers = NULL;
+ self->offsets = NULL;
+ }
+}
+
+void
+PolyHLReader_destroy(PolyHighlightReader *self) {
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYHIGHLIGHTREADER);
+}
+
+DocVector*
+PolyHLReader_fetch_doc_vec(PolyHighlightReader *self, int32_t doc_id) {
+ uint32_t seg_tick = PolyReader_sub_tick(self->offsets, doc_id);
+ int32_t offset = I32Arr_Get(self->offsets, seg_tick);
+ HighlightReader *sub_reader
+ = (HighlightReader*)VA_Fetch(self->readers, seg_tick);
+ if (!sub_reader) { THROW(ERR, "Invalid doc_id: %i32", doc_id); }
+ return HLReader_Fetch_Doc_Vec(sub_reader, doc_id - offset);
+}
+
+DefaultHighlightReader*
+DefHLReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick) {
+ DefaultHighlightReader *self = (DefaultHighlightReader*)VTable_Make_Obj(
+ DEFAULTHIGHLIGHTREADER);
+ return DefHLReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+DefaultHighlightReader*
+DefHLReader_init(DefaultHighlightReader *self, Schema *schema,
+ Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick) {
+ Segment *segment;
+ Hash *metadata;
+ HLReader_init((HighlightReader*)self, schema, folder, snapshot,
+ segments, seg_tick);
+ segment = DefHLReader_Get_Segment(self);
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "highlight", 9);
+ if (!metadata) {
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "term_vectors", 12);
+ }
+
+ // Check format.
+ if (metadata) {
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+ if (!format) { THROW(ERR, "Missing 'format' var"); }
+ else {
+ if (Obj_To_I64(format) != HLWriter_current_file_format) {
+ THROW(ERR, "Unsupported highlight data format: %i64",
+ Obj_To_I64(format));
+ }
+ }
+ }
+
+
+ // Open instreams.
+ {
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
+ CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
+ if (Folder_Exists(folder, ix_file)) {
+ self->ix_in = Folder_Open_In(folder, ix_file);
+ if (!self->ix_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->dat_in = Folder_Open_In(folder, dat_file);
+ if (!self->dat_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(dat_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ }
+ DECREF(ix_file);
+ DECREF(dat_file);
+ }
+
+ return self;
+}
+
+void
+DefHLReader_close(DefaultHighlightReader *self) {
+ if (self->dat_in != NULL) {
+ InStream_Close(self->dat_in);
+ DECREF(self->dat_in);
+ self->dat_in = NULL;
+ }
+ if (self->ix_in != NULL) {
+ InStream_Close(self->ix_in);
+ DECREF(self->ix_in);
+ self->ix_in = NULL;
+ }
+}
+
+void
+DefHLReader_destroy(DefaultHighlightReader *self) {
+ DECREF(self->ix_in);
+ DECREF(self->dat_in);
+ SUPER_DESTROY(self, DEFAULTHIGHLIGHTREADER);
+}
+
+DocVector*
+DefHLReader_fetch_doc_vec(DefaultHighlightReader *self, int32_t doc_id) {
+ DocVector *doc_vec = DocVec_new();
+ int64_t file_pos;
+ uint32_t num_fields;
+
+ InStream_Seek(self->ix_in, doc_id * 8);
+ file_pos = InStream_Read_I64(self->ix_in);
+ InStream_Seek(self->dat_in, file_pos);
+
+ num_fields = InStream_Read_C32(self->dat_in);
+ while (num_fields--) {
+ CharBuf *field = CB_deserialize(NULL, self->dat_in);
+ ByteBuf *field_buf = BB_deserialize(NULL, self->dat_in);
+ DocVec_Add_Field_Buf(doc_vec, field, field_buf);
+ DECREF(field_buf);
+ DECREF(field);
+ }
+
+ return doc_vec;
+}
+
+void
+DefHLReader_read_record(DefaultHighlightReader *self, int32_t doc_id,
+ ByteBuf *target) {
+ InStream *dat_in = self->dat_in;
+ InStream *ix_in = self->ix_in;
+
+ InStream_Seek(ix_in, doc_id * 8);
+
+ {
+ // Copy the whole record.
+ int64_t filepos = InStream_Read_I64(ix_in);
+ int64_t end = InStream_Read_I64(ix_in);
+ size_t size = (size_t)(end - filepos);
+ char *buf = BB_Grow(target, size);
+ InStream_Seek(dat_in, filepos);
+ InStream_Read_Bytes(dat_in, buf, size);
+ BB_Set_Size(target, size);
+ }
+}
+
+
diff --git a/core/Lucy/Index/HighlightReader.cfh b/core/Lucy/Index/HighlightReader.cfh
new file mode 100644
index 0000000..2259d7d
--- /dev/null
+++ b/core/Lucy/Index/HighlightReader.cfh
@@ -0,0 +1,96 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read a segment's highlighting data.
+ *
+ * Read segment data used for creating highlighted excerpts.
+ */
+class Lucy::Index::HighlightReader cnick HLReader
+ inherits Lucy::Index::DataReader {
+
+ inert HighlightReader*
+ init(HighlightReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Return the DocVector object for the given doc id.
+ */
+ public abstract incremented DocVector*
+ Fetch_Doc_Vec(HighlightReader *self, int32_t doc_id);
+
+ public incremented nullable HighlightReader*
+ Aggregator(HighlightReader *self, VArray *readers, I32Array *offsets);
+}
+
+class Lucy::Index::PolyHighlightReader cnick PolyHLReader
+ inherits Lucy::Index::HighlightReader {
+
+ VArray *readers;
+ I32Array *offsets;
+
+ inert incremented PolyHighlightReader*
+ new(VArray *readers, I32Array *offsets);
+
+ inert PolyHighlightReader*
+ init(PolyHighlightReader *self, VArray *readers, I32Array *offsets);
+
+ public incremented DocVector*
+ Fetch_Doc_Vec(PolyHighlightReader *self, int32_t doc_id);
+
+ public void
+ Close(PolyHighlightReader *self);
+
+ public void
+ Destroy(PolyHighlightReader *self);
+}
+
+class Lucy::Index::DefaultHighlightReader cnick DefHLReader
+ inherits Lucy::Index::HighlightReader {
+
+ InStream *ix_in;
+ InStream *dat_in;
+
+ /** Constructors.
+ */
+ inert incremented DefaultHighlightReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultHighlightReader*
+ init(DefaultHighlightReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ /** Return the DocVector object for the given doc id.
+ */
+ public incremented DocVector*
+ Fetch_Doc_Vec(DefaultHighlightReader *self, int32_t doc_id);
+
+ /** Return the raw bytes of an entry.
+ */
+ void
+ Read_Record(DefaultHighlightReader *self, int32_t doc_id,
+ ByteBuf *buffer);
+
+ public void
+ Close(DefaultHighlightReader *self);
+
+ public void
+ Destroy(DefaultHighlightReader *self);
+}
+
+
diff --git a/core/Lucy/Index/HighlightWriter.c b/core/Lucy/Index/HighlightWriter.c
new file mode 100644
index 0000000..68ae6fa
--- /dev/null
+++ b/core/Lucy/Index/HighlightWriter.c
@@ -0,0 +1,269 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HIGHLIGHTWRITER
+#define C_LUCY_DEFAULTHIGHLIGHTWRITER
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include <stdio.h>
+
+#include "Lucy/Index/HighlightWriter.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Index/HighlightReader.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/InStream.h"
+
+static OutStream*
+S_lazy_init(HighlightWriter *self);
+
+int32_t HLWriter_current_file_format = 1;
+
+HighlightWriter*
+HLWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ HighlightWriter *self
+ = (HighlightWriter*)VTable_Make_Obj(HIGHLIGHTWRITER);
+ return HLWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+HighlightWriter*
+HLWriter_init(HighlightWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ return self;
+}
+
+void
+HLWriter_destroy(HighlightWriter *self) {
+ DECREF(self->dat_out);
+ DECREF(self->ix_out);
+ SUPER_DESTROY(self, HIGHLIGHTWRITER);
+}
+
+static OutStream*
+S_lazy_init(HighlightWriter *self) {
+ if (!self->dat_out) {
+ Segment *segment = self->segment;
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(segment);
+
+ // Open outstreams.
+ {
+ CharBuf *ix_file = CB_newf("%o/highlight.ix", seg_name);
+ self->ix_out = Folder_Open_Out(folder, ix_file);
+ DECREF(ix_file);
+ if (!self->ix_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+ {
+ CharBuf *dat_file = CB_newf("%o/highlight.dat", seg_name);
+ self->dat_out = Folder_Open_Out(folder, dat_file);
+ DECREF(dat_file);
+ if (!self->dat_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+
+ // Go past invalid doc 0.
+ OutStream_Write_I64(self->ix_out, 0);
+ }
+
+ return self->dat_out;
+}
+
+void
+HLWriter_add_inverted_doc(HighlightWriter *self, Inverter *inverter,
+ int32_t doc_id) {
+ OutStream *dat_out = S_lazy_init(self);
+ OutStream *ix_out = self->ix_out;
+ int64_t filepos = OutStream_Tell(dat_out);
+ uint32_t num_highlightable = 0;
+ int32_t expected = (int32_t)(OutStream_Tell(ix_out) / 8);
+
+ // Verify doc id.
+ if (doc_id != expected) {
+ THROW(ERR, "Expected doc id %i32 but got %i32", expected, doc_id);
+ }
+
+ // Write index data.
+ OutStream_Write_I64(ix_out, filepos);
+
+ // Count, then write number of highlightable fields.
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Is_A(type, FULLTEXTTYPE)
+ && FullTextType_Highlightable((FullTextType*)type)
+ ) {
+ num_highlightable++;
+ }
+ }
+ OutStream_Write_C32(dat_out, num_highlightable);
+
+ Inverter_Iterate(inverter);
+ while (Inverter_Next(inverter)) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Is_A(type, FULLTEXTTYPE)
+ && FullTextType_Highlightable((FullTextType*)type)
+ ) {
+ CharBuf *field = Inverter_Get_Field_Name(inverter);
+ Inversion *inversion = Inverter_Get_Inversion(inverter);
+ ByteBuf *tv_buf = HLWriter_TV_Buf(self, inversion);
+ CB_Serialize(field, dat_out);
+ BB_Serialize(tv_buf, dat_out);
+ DECREF(tv_buf);
+ }
+ }
+}
+
+ByteBuf*
+HLWriter_tv_buf(HighlightWriter *self, Inversion *inversion) {
+ char *last_text = "";
+ size_t last_len = 0;
+ ByteBuf *tv_buf = BB_new(20 + Inversion_Get_Size(inversion) * 8);
+ uint32_t num_postings = 0;
+ char *dest;
+ Token **tokens;
+ uint32_t freq;
+ UNUSED_VAR(self);
+
+ // Leave space for a c32 indicating the number of postings.
+ BB_Set_Size(tv_buf, C32_MAX_BYTES);
+
+ Inversion_Reset(inversion);
+ while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
+ Token *token = *tokens;
+ int32_t overlap = StrHelp_overlap(last_text, token->text,
+ last_len, token->len);
+ char *ptr;
+ char *orig;
+ size_t old_size = BB_Get_Size(tv_buf);
+ size_t new_size = old_size
+ + C32_MAX_BYTES // overlap
+ + C32_MAX_BYTES // length of string diff
+ + (token->len - overlap) // diff char data
+ + C32_MAX_BYTES // num prox
+ + (C32_MAX_BYTES * freq * 3); // pos data
+
+ // Allocate for worst-case scenario.
+ ptr = BB_Grow(tv_buf, new_size);
+ orig = ptr;
+ ptr += old_size;
+
+ // Track number of postings.
+ num_postings += 1;
+
+ // Append the string diff to the tv_buf.
+ NumUtil_encode_c32(overlap, &ptr);
+ NumUtil_encode_c32((token->len - overlap), &ptr);
+ memcpy(ptr, (token->text + overlap), (token->len - overlap));
+ ptr += token->len - overlap;
+
+ // Save text and text_len for comparison next loop.
+ last_text = token->text;
+ last_len = token->len;
+
+ // Append the number of positions for this term.
+ NumUtil_encode_c32(freq, &ptr);
+
+ do {
+ // Add position, start_offset, and end_offset to tv_buf.
+ NumUtil_encode_c32(token->pos, &ptr);
+ NumUtil_encode_c32(token->start_offset, &ptr);
+ NumUtil_encode_c32(token->end_offset, &ptr);
+
+ } while (--freq && (token = *++tokens));
+
+ // Set new byte length.
+ BB_Set_Size(tv_buf, ptr - orig);
+ }
+
+ // Go back and start the term vector string with the posting count.
+ dest = BB_Get_Buf(tv_buf);
+ NumUtil_encode_padded_c32(num_postings, &dest);
+
+ return tv_buf;
+}
+
+void
+HLWriter_add_segment(HighlightWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ int32_t doc_max = SegReader_Doc_Max(reader);
+
+ if (doc_max == 0) {
+ // Bail if the supplied segment is empty.
+ return;
+ }
+ else {
+ DefaultHighlightReader *hl_reader
+ = (DefaultHighlightReader*)CERTIFY(
+ SegReader_Obtain(reader, VTable_Get_Name(HIGHLIGHTREADER)),
+ DEFAULTHIGHLIGHTREADER);
+ OutStream *dat_out = S_lazy_init(self);
+ OutStream *ix_out = self->ix_out;
+ int32_t orig;
+ ByteBuf *bb = BB_new(0);
+
+ for (orig = 1; orig <= doc_max; orig++) {
+ // Skip deleted docs.
+ if (doc_map && !I32Arr_Get(doc_map, orig)) {
+ continue;
+ }
+
+ // Write file pointer.
+ OutStream_Write_I64(ix_out, OutStream_Tell(dat_out));
+
+ // Copy the raw record.
+ DefHLReader_Read_Record(hl_reader, orig, bb);
+ OutStream_Write_Bytes(dat_out, BB_Get_Buf(bb), BB_Get_Size(bb));
+
+ BB_Set_Size(bb, 0);
+ }
+ DECREF(bb);
+ }
+}
+
+void
+HLWriter_finish(HighlightWriter *self) {
+ if (self->dat_out) {
+ // Write one final file pointer, so that we can derive the length of
+ // the last record.
+ int64_t end = OutStream_Tell(self->dat_out);
+ OutStream_Write_I64(self->ix_out, end);
+
+ // Close down the output streams.
+ OutStream_Close(self->dat_out);
+ OutStream_Close(self->ix_out);
+ Seg_Store_Metadata_Str(self->segment, "highlight", 9,
+ (Obj*)HLWriter_Metadata(self));
+ }
+}
+
+int32_t
+HLWriter_format(HighlightWriter *self) {
+ UNUSED_VAR(self);
+ return HLWriter_current_file_format;
+}
+
+
diff --git a/core/Lucy/Index/HighlightWriter.cfh b/core/Lucy/Index/HighlightWriter.cfh
new file mode 100644
index 0000000..b8f8efb
--- /dev/null
+++ b/core/Lucy/Index/HighlightWriter.cfh
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Add highlighting data to index.
+ */
+
+class Lucy::Index::HighlightWriter cnick HLWriter
+ inherits Lucy::Index::DataWriter {
+
+ OutStream *ix_out;
+ OutStream *dat_out;
+
+ inert int32_t current_file_format;
+
+ inert incremented HighlightWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert HighlightWriter*
+ init(HighlightWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ incremented ByteBuf*
+ TV_Buf(HighlightWriter *self, Inversion *inversion);
+
+ public void
+ Add_Inverted_Doc(HighlightWriter *self, Inverter *inverter, int32_t doc_id);
+
+ public void
+ Add_Segment(HighlightWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Finish(HighlightWriter *self);
+
+ public int32_t
+ Format(HighlightWriter *self);
+
+ public void
+ Destroy(HighlightWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/IndexManager.c b/core/Lucy/Index/IndexManager.c
new file mode 100644
index 0000000..f17c593
--- /dev/null
+++ b/core/Lucy/Index/IndexManager.c
@@ -0,0 +1,391 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INDEXMANAGER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/LockFactory.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/StringHelper.h"
+
+IndexManager*
+IxManager_new(const CharBuf *host, LockFactory *lock_factory) {
+ IndexManager *self = (IndexManager*)VTable_Make_Obj(INDEXMANAGER);
+ return IxManager_init(self, host, lock_factory);
+}
+
+IndexManager*
+IxManager_init(IndexManager *self, const CharBuf *host,
+ LockFactory *lock_factory) {
+ self->host = host
+ ? CB_Clone(host)
+ : CB_new_from_trusted_utf8("", 0);
+ self->lock_factory = (LockFactory*)INCREF(lock_factory);
+ self->folder = NULL;
+ self->write_lock_timeout = 1000;
+ self->write_lock_interval = 100;
+ self->merge_lock_timeout = 0;
+ self->merge_lock_interval = 1000;
+ self->deletion_lock_timeout = 1000;
+ self->deletion_lock_interval = 100;
+
+ return self;
+}
+
+void
+IxManager_destroy(IndexManager *self) {
+ DECREF(self->host);
+ DECREF(self->folder);
+ DECREF(self->lock_factory);
+ SUPER_DESTROY(self, INDEXMANAGER);
+}
+
+int64_t
+IxManager_highest_seg_num(IndexManager *self, Snapshot *snapshot) {
+ VArray *files = Snapshot_List(snapshot);
+ uint32_t i, max;
+ uint64_t highest_seg_num = 0;
+ UNUSED_VAR(self);
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *file = (CharBuf*)VA_Fetch(files, i);
+ if (Seg_valid_seg_name(file)) {
+ uint64_t seg_num = IxFileNames_extract_gen(file);
+ if (seg_num > highest_seg_num) { highest_seg_num = seg_num; }
+ }
+ }
+ DECREF(files);
+ return (int64_t)highest_seg_num;
+}
+
+CharBuf*
+IxManager_make_snapshot_filename(IndexManager *self) {
+ Folder *folder = (Folder*)CERTIFY(self->folder, FOLDER);
+ DirHandle *dh = Folder_Open_Dir(folder, NULL);
+ CharBuf *entry;
+ uint64_t max_gen = 0;
+
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+ entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ if (CB_Starts_With_Str(entry, "snapshot_", 9)
+ && CB_Ends_With_Str(entry, ".json", 5)
+ ) {
+ uint64_t gen = IxFileNames_extract_gen(entry);
+ if (gen > max_gen) { max_gen = gen; }
+ }
+ }
+ DECREF(dh);
+
+ {
+ uint64_t new_gen = max_gen + 1;
+ char base36[StrHelp_MAX_BASE36_BYTES];
+ StrHelp_to_base36(new_gen, &base36);
+ return CB_newf("snapshot_%s.json", &base36);
+ }
+}
+
+static int
+S_compare_doc_count(void *context, const void *va, const void *vb) {
+ SegReader *a = *(SegReader**)va;
+ SegReader *b = *(SegReader**)vb;
+ UNUSED_VAR(context);
+ return SegReader_Doc_Count(a) - SegReader_Doc_Count(b);
+}
+
+static bool_t
+S_check_cutoff(VArray *array, uint32_t tick, void *data) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(array, tick);
+ int64_t cutoff = *(int64_t*)data;
+ return SegReader_Get_Seg_Num(seg_reader) > cutoff;
+}
+
+static uint32_t
+S_fibonacci(uint32_t n) {
+ uint32_t result = 0;
+ if (n > 46) {
+ THROW(ERR, "input %u32 too high", n);
+ }
+ else if (n < 2) {
+ result = n;
+ }
+ else {
+ result = S_fibonacci(n - 1) + S_fibonacci(n - 2);
+ }
+ return result;
+}
+
+VArray*
+IxManager_recycle(IndexManager *self, PolyReader *reader,
+ DeletionsWriter *del_writer, int64_t cutoff,
+ bool_t optimize) {
+ VArray *seg_readers = PolyReader_Get_Seg_Readers(reader);
+ VArray *candidates = VA_Gather(seg_readers, S_check_cutoff, &cutoff);
+ VArray *recyclables = VA_new(VA_Get_Size(candidates));
+ const uint32_t num_candidates = VA_Get_Size(candidates);
+
+ if (optimize) {
+ DECREF(recyclables);
+ return candidates;
+ }
+
+ // Sort by ascending size in docs, choose sparsely populated segments.
+ VA_Sort(candidates, S_compare_doc_count, NULL);
+ int32_t *counts = (int32_t*)MALLOCATE(num_candidates * sizeof(int32_t));
+ for (uint32_t i = 0; i < num_candidates; i++) {
+ SegReader *seg_reader
+ = (SegReader*)CERTIFY(VA_Fetch(candidates, i), SEGREADER);
+ counts[i] = SegReader_Doc_Count(seg_reader);
+ }
+ I32Array *doc_counts = I32Arr_new_steal(counts, num_candidates);
+ uint32_t threshold = IxManager_Choose_Sparse(self, doc_counts);
+ DECREF(doc_counts);
+
+ // Move SegReaders to be recycled.
+ for (uint32_t i = 0; i < threshold; i++) {
+ VA_Store(recyclables, i, VA_Delete(candidates, i));
+ }
+
+ // Find segments where at least 10% of all docs have been deleted.
+ for (uint32_t i = threshold; i < num_candidates; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Delete(candidates, i);
+ CharBuf *seg_name = SegReader_Get_Seg_Name(seg_reader);
+ double doc_max = SegReader_Doc_Max(seg_reader);
+ double num_deletions = DelWriter_Seg_Del_Count(del_writer, seg_name);
+ double del_proportion = num_deletions / doc_max;
+ if (del_proportion >= 0.1) {
+ VA_Push(recyclables, (Obj*)seg_reader);
+ }
+ else {
+ DECREF(seg_reader);
+ }
+ }
+
+ DECREF(candidates);
+ return recyclables;
+}
+
+uint32_t
+IxManager_choose_sparse(IndexManager *self, I32Array *doc_counts) {
+ UNUSED_VAR(self);
+ uint32_t threshold = 0;
+ uint32_t total_docs = 0;
+ const uint32_t num_candidates = I32Arr_Get_Size(doc_counts);
+
+ // Find sparsely populated segments.
+ for (uint32_t i = 0; i < num_candidates; i++) {
+ uint32_t num_segs_when_done = num_candidates - threshold + 1;
+ total_docs += I32Arr_Get(doc_counts, i);
+ if (total_docs < S_fibonacci(num_segs_when_done + 5)) {
+ threshold = i + 1;
+ }
+ }
+
+ // If recycling, try not to get stuck merging the same big segment over
+ // and over on small commits.
+ if (threshold == 1 && num_candidates > 2) {
+ int32_t this_seg_doc_count = I32Arr_Get(doc_counts, 0);
+ int32_t next_seg_doc_count = I32Arr_Get(doc_counts, 1);
+ // Try to merge 2 segments worth of stuff, so long as the next segment
+ // is less than double the size.
+ if (next_seg_doc_count / 2 < this_seg_doc_count) {
+ threshold = 2;
+ }
+ }
+
+ return threshold;
+}
+
+static LockFactory*
+S_obtain_lock_factory(IndexManager *self) {
+ if (!self->lock_factory) {
+ if (!self->folder) {
+ THROW(ERR, "Can't create a LockFactory without a Folder");
+ }
+ self->lock_factory = LockFact_new(self->folder, self->host);
+ }
+ return self->lock_factory;
+}
+
+Lock*
+IxManager_make_write_lock(IndexManager *self) {
+ ZombieCharBuf *write_lock_name = ZCB_WRAP_STR("write", 5);
+ LockFactory *lock_factory = S_obtain_lock_factory(self);
+ return LockFact_Make_Lock(lock_factory, (CharBuf*)write_lock_name,
+ self->write_lock_timeout,
+ self->write_lock_interval);
+}
+
+Lock*
+IxManager_make_deletion_lock(IndexManager *self) {
+ ZombieCharBuf *lock_name = ZCB_WRAP_STR("deletion", 8);
+ LockFactory *lock_factory = S_obtain_lock_factory(self);
+ return LockFact_Make_Lock(lock_factory, (CharBuf*)lock_name,
+ self->deletion_lock_timeout,
+ self->deletion_lock_interval);
+}
+
+Lock*
+IxManager_make_merge_lock(IndexManager *self) {
+ ZombieCharBuf *merge_lock_name = ZCB_WRAP_STR("merge", 5);
+ LockFactory *lock_factory = S_obtain_lock_factory(self);
+ return LockFact_Make_Lock(lock_factory, (CharBuf*)merge_lock_name,
+ self->merge_lock_timeout,
+ self->merge_lock_interval);
+}
+
+void
+IxManager_write_merge_data(IndexManager *self, int64_t cutoff) {
+ ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
+ Hash *data = Hash_new(1);
+ bool_t success;
+ Hash_Store_Str(data, "cutoff", 6, (Obj*)CB_newf("%i64", cutoff));
+ success = Json_spew_json((Obj*)data, self->folder, (CharBuf*)merge_json);
+ DECREF(data);
+ if (!success) {
+ THROW(ERR, "Failed to write to %o", merge_json);
+ }
+}
+
+Hash*
+IxManager_read_merge_data(IndexManager *self) {
+ ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
+ if (Folder_Exists(self->folder, (CharBuf*)merge_json)) {
+ Hash *stuff
+ = (Hash*)Json_slurp_json(self->folder, (CharBuf*)merge_json);
+ if (stuff) {
+ CERTIFY(stuff, HASH);
+ return stuff;
+ }
+ else {
+ return Hash_new(0);
+ }
+ }
+ else {
+ return NULL;
+ }
+}
+
+bool_t
+IxManager_remove_merge_data(IndexManager *self) {
+ ZombieCharBuf *merge_json = ZCB_WRAP_STR("merge.json", 10);
+ return Folder_Delete(self->folder, (CharBuf*)merge_json) != 0;
+}
+
+Lock*
+IxManager_make_snapshot_read_lock(IndexManager *self,
+ const CharBuf *filename) {
+ ZombieCharBuf *lock_name = ZCB_WRAP(filename);
+ LockFactory *lock_factory = S_obtain_lock_factory(self);
+
+ if (!CB_Starts_With_Str(filename, "snapshot_", 9)
+ || !CB_Ends_With_Str(filename, ".json", 5)
+ ) {
+ THROW(ERR, "Not a snapshot filename: %o", filename);
+ }
+
+ // Truncate ".json" from end of snapshot file name.
+ ZCB_Chop(lock_name, sizeof(".json") - 1);
+
+ return LockFact_Make_Shared_Lock(lock_factory, (CharBuf*)lock_name, 1000, 100);
+}
+
+void
+IxManager_set_folder(IndexManager *self, Folder *folder) {
+ DECREF(self->folder);
+ self->folder = (Folder*)INCREF(folder);
+}
+
+Folder*
+IxManager_get_folder(IndexManager *self) {
+ return self->folder;
+}
+
+CharBuf*
+IxManager_get_host(IndexManager *self) {
+ return self->host;
+}
+
+uint32_t
+IxManager_get_write_lock_timeout(IndexManager *self) {
+ return self->write_lock_timeout;
+}
+
+uint32_t
+IxManager_get_write_lock_interval(IndexManager *self) {
+ return self->write_lock_interval;
+}
+
+uint32_t
+IxManager_get_merge_lock_timeout(IndexManager *self) {
+ return self->merge_lock_timeout;
+}
+
+uint32_t
+IxManager_get_merge_lock_interval(IndexManager *self) {
+ return self->merge_lock_interval;
+}
+
+uint32_t
+IxManager_get_deletion_lock_timeout(IndexManager *self) {
+ return self->deletion_lock_timeout;
+}
+
+uint32_t
+IxManager_get_deletion_lock_interval(IndexManager *self) {
+ return self->deletion_lock_interval;
+}
+
+void
+IxManager_set_write_lock_timeout(IndexManager *self, uint32_t timeout) {
+ self->write_lock_timeout = timeout;
+}
+
+void
+IxManager_set_write_lock_interval(IndexManager *self, uint32_t interval) {
+ self->write_lock_interval = interval;
+}
+
+void
+IxManager_set_merge_lock_timeout(IndexManager *self, uint32_t timeout) {
+ self->merge_lock_timeout = timeout;
+}
+
+void
+IxManager_set_merge_lock_interval(IndexManager *self, uint32_t interval) {
+ self->merge_lock_interval = interval;
+}
+
+void
+IxManager_set_deletion_lock_timeout(IndexManager *self, uint32_t timeout) {
+ self->deletion_lock_timeout = timeout;
+}
+
+void
+IxManager_set_deletion_lock_interval(IndexManager *self, uint32_t interval) {
+ self->deletion_lock_interval = interval;
+}
+
+
diff --git a/core/Lucy/Index/IndexManager.cfh b/core/Lucy/Index/IndexManager.cfh
new file mode 100644
index 0000000..de420bb
--- /dev/null
+++ b/core/Lucy/Index/IndexManager.cfh
@@ -0,0 +1,210 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Policies governing index updating, locking, and file deletion.
+ *
+ * IndexManager is an advanced-use class for controlling index locking,
+ * updating, merging, and deletion behaviors.
+ *
+ * IndexManager and L<Architecture|Lucy::Plan::Architecture> are
+ * complementary classes: Architecture is used to define traits and behaviors
+ * which cannot change for the life of an index; IndexManager is used for
+ * defining rules which may change from process to process.
+ */
+class Lucy::Index::IndexManager cnick IxManager
+ inherits Lucy::Object::Obj {
+
+ Folder *folder;
+ CharBuf *host;
+ LockFactory *lock_factory;
+ uint32_t write_lock_timeout;
+ uint32_t write_lock_interval;
+ uint32_t merge_lock_timeout;
+ uint32_t merge_lock_interval;
+ uint32_t deletion_lock_timeout;
+ uint32_t deletion_lock_interval;
+
+ public inert incremented IndexManager*
+ new(const CharBuf *host = NULL, LockFactory *lock_factory = NULL);
+
+ /**
+ * @param host An identifier which should be unique per-machine.
+ * @param lock_factory A LockFactory.
+ */
+ public inert IndexManager*
+ init(IndexManager *self, const CharBuf *host = NULL,
+ LockFactory *lock_factory = NULL);
+
+ public void
+ Destroy(IndexManager *self);
+
+ /**
+ * Setter for <code>folder</code> member. Typical clients (Indexer,
+ * IndexReader) will use this method to install their own Folder instance.
+ */
+ public void
+ Set_Folder(IndexManager *self, Folder *folder = NULL);
+
+ /** Getter for <code>folder</code> member.
+ */
+ public nullable Folder*
+ Get_Folder(IndexManager *self);
+
+ /** Getter for <code>host</code> member.
+ */
+ public CharBuf*
+ Get_Host(IndexManager *self);
+
+ /** Return an array of SegReaders representing segments that should be
+ * consolidated. Implementations must balance index-time churn against
+ * search-time degradation due to segment proliferation. The default
+ * implementation prefers small segments or segments with a high
+ * proportion of deletions.
+ *
+ * @param reader A PolyReader.
+ * @param del_writer A DeletionsWriter.
+ * @param cutoff A segment number which all returned SegReaders must
+ * exceed.
+ * @param optimize A boolean indicating whether to spend extra time
+ * optimizing the index for search-time performance.
+ */
+ public incremented VArray*
+ Recycle(IndexManager *self, PolyReader *reader,
+ DeletionsWriter *del_writer, int64_t cutoff,
+ bool_t optimize = false);
+
+ /** Return a tick. All segments below that tick will be merged.
+ * Exposed for testing purposes only.
+ *
+ * @param doc_counts Segment doc counts, in ascending order.
+ */
+ uint32_t
+ Choose_Sparse(IndexManager *self, I32Array *doc_counts);
+
+ /** Create the Lock which controls access to modifying the logical content
+ * of the index.
+ */
+ public incremented Lock*
+ Make_Write_Lock(IndexManager *self);
+
+ /** Create the Lock which grants permission to delete obsolete snapshot
+ * files or any file listed within an existing snapshot file.
+ */
+ public incremented Lock*
+ Make_Deletion_Lock(IndexManager *self);
+
+ public incremented Lock*
+ Make_Merge_Lock(IndexManager *self);
+
+ /** Write supplied data to "merge.json". Throw an exception if the write
+ * fails.
+ */
+ public void
+ Write_Merge_Data(IndexManager *self, int64_t cutoff);
+
+ /** Look for the "merge.json" file dropped by BackgroundMerger. If it's
+ * not there, return NULL. If it's there but can't be decoded, return an
+ * empty Hash. If successfully decoded, return contents.
+ */
+ public incremented Hash*
+ Read_Merge_Data(IndexManager *self);
+
+ public bool_t
+ Remove_Merge_Data(IndexManager *self);
+
+ /** Create a shared lock on a snapshot file, which serves as a proxy for
+ * all the files it lists and indicates that they must not be deleted.
+ */
+ public incremented Lock*
+ Make_Snapshot_Read_Lock(IndexManager *self, const CharBuf *filename);
+
+ /** Return the highest number for a segment directory which contains a
+ * segmeta file in the snapshot.
+ */
+ public int64_t
+ Highest_Seg_Num(IndexManager *self, Snapshot *snapshot);
+
+ /** Return the name of a new snapshot file, which shall contain a base-36
+ * "generation" embedded inside it greater than the generation of any
+ * snapshot file currently in the index folder.
+ */
+ public incremented CharBuf*
+ Make_Snapshot_Filename(IndexManager *self);
+
+ /** Setter for write lock timeout. Default: 1000 milliseconds.
+ */
+ public void
+ Set_Write_Lock_Timeout(IndexManager *self, uint32_t timeout);
+
+ /** Getter for write lock timeout.
+ */
+ public uint32_t
+ Get_Write_Lock_Timeout(IndexManager *self);
+
+ /** Setter for write lock retry interval. Default: 100 milliseconds.
+ */
+ public void
+ Set_Write_Lock_Interval(IndexManager *self, uint32_t timeout);
+
+ /** Getter for write lock retry interval.
+ */
+ public uint32_t
+ Get_Write_Lock_Interval(IndexManager *self);
+
+ /** Setter for merge lock timeout. Default: 0 milliseconds (no retries).
+ */
+ public void
+ Set_Merge_Lock_Timeout(IndexManager *self, uint32_t timeout);
+
+ /** Getter for merge lock timeout.
+ */
+ public uint32_t
+ Get_Merge_Lock_Timeout(IndexManager *self);
+
+ /** Setter for merge lock retry interval. Default: 1000 milliseconds.
+ */
+ public void
+ Set_Merge_Lock_Interval(IndexManager *self, uint32_t timeout);
+
+ /** Getter for merge lock retry interval.
+ */
+ public uint32_t
+ Get_Merge_Lock_Interval(IndexManager *self);
+
+ /** Setter for deletion lock timeout. Default: 1000 milliseconds.
+ */
+ public void
+ Set_Deletion_Lock_Timeout(IndexManager *self, uint32_t timeout);
+
+ /** Getter for deletion lock timeout.
+ */
+ public uint32_t
+ Get_Deletion_Lock_Timeout(IndexManager *self);
+
+ /** Setter for deletion lock retry interval. Default: 100 milliseconds.
+ */
+ public void
+ Set_Deletion_Lock_Interval(IndexManager *self, uint32_t timeout);
+
+ /** Getter for deletion lock retry interval.
+ */
+ public uint32_t
+ Get_Deletion_Lock_Interval(IndexManager *self);
+}
+
+
diff --git a/core/Lucy/Index/IndexReader.c b/core/Lucy/Index/IndexReader.c
new file mode 100644
index 0000000..99250c2
--- /dev/null
+++ b/core/Lucy/Index/IndexReader.c
@@ -0,0 +1,122 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INDEXREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/IndexReader.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/Lock.h"
+
+IndexReader*
+IxReader_open(Obj *index, Snapshot *snapshot, IndexManager *manager) {
+ return IxReader_do_open(NULL, index, snapshot, manager);
+}
+
+IndexReader*
+IxReader_do_open(IndexReader *temp_self, Obj *index, Snapshot *snapshot,
+ IndexManager *manager) {
+ PolyReader *polyreader = PolyReader_open(index, snapshot, manager);
+ if (!VA_Get_Size(PolyReader_Get_Seg_Readers(polyreader))) {
+ THROW(ERR, "Index doesn't seem to contain any data");
+ }
+ DECREF(temp_self);
+ return (IndexReader*)polyreader;
+}
+
+IndexReader*
+IxReader_init(IndexReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick,
+ IndexManager *manager) {
+ snapshot = snapshot ? (Snapshot*)INCREF(snapshot) : Snapshot_new();
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ DECREF(snapshot);
+ self->components = Hash_new(0);
+ self->read_lock = NULL;
+ self->deletion_lock = NULL;
+ if (manager) {
+ self->manager = (IndexManager*)INCREF(manager);
+ IxManager_Set_Folder(self->manager, self->folder);
+ }
+ else {
+ self->manager = NULL;
+ }
+ return self;
+}
+
+void
+IxReader_close(IndexReader *self) {
+ if (self->components) {
+ CharBuf *key;
+ DataReader *component;
+ Hash_Iterate(self->components);
+ while (Hash_Next(self->components, (Obj**)&key,
+ (Obj**)&component)
+ ) {
+ if (Obj_Is_A((Obj*)component, DATAREADER)) {
+ DataReader_Close(component);
+ }
+ }
+ Hash_Clear(self->components);
+ }
+ if (self->read_lock) {
+ Lock_Release(self->read_lock);
+ DECREF(self->read_lock);
+ self->read_lock = NULL;
+ }
+}
+
+void
+IxReader_destroy(IndexReader *self) {
+ DECREF(self->components);
+ if (self->read_lock) {
+ Lock_Release(self->read_lock);
+ DECREF(self->read_lock);
+ }
+ DECREF(self->manager);
+ DECREF(self->deletion_lock);
+ SUPER_DESTROY(self, INDEXREADER);
+}
+
+Hash*
+IxReader_get_components(IndexReader *self) {
+ return self->components;
+}
+
+DataReader*
+IxReader_obtain(IndexReader *self, const CharBuf *api) {
+ DataReader *component
+ = (DataReader*)Hash_Fetch(self->components, (Obj*)api);
+ if (!component) {
+ THROW(ERR, "No component registered for '%o'", api);
+ }
+ return component;
+}
+
+DataReader*
+IxReader_fetch(IndexReader *self, const CharBuf *api) {
+ return (DataReader*)Hash_Fetch(self->components, (Obj*)api);
+}
+
+
diff --git a/core/Lucy/Index/IndexReader.cfh b/core/Lucy/Index/IndexReader.cfh
new file mode 100644
index 0000000..4ab34ca
--- /dev/null
+++ b/core/Lucy/Index/IndexReader.cfh
@@ -0,0 +1,125 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read from an inverted index.
+ *
+ * IndexReader is the interface through which
+ * L<IndexSearcher|Lucy::Search::IndexSearcher> objects access the
+ * content of an index.
+ *
+ * IndexReader objects always represent a point-in-time view of an index as it
+ * existed at the moment the reader was created. If you want search results
+ * to reflect modifications to an index, you must create a new IndexReader
+ * after the update process completes.
+ *
+ * IndexReaders are composites; most of the work is done by individual
+ * L<DataReader|Lucy::Index::DataReader> sub-components, which may be
+ * accessed via Fetch() and Obtain(). The most efficient and powerful access
+ * to index data happens at the segment level via
+ * L<SegReader|Lucy::Index::SegReader>'s sub-components.
+ */
+
+class Lucy::Index::IndexReader cnick IxReader
+ inherits Lucy::Index::DataReader {
+
+ Hash *components;
+ IndexManager *manager;
+ Lock *read_lock;
+ Lock *deletion_lock;
+
+ public inert nullable IndexReader*
+ init(IndexReader *self, Schema *schema = NULL, Folder *folder,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1, IndexManager *manager = NULL);
+
+ public inert incremented nullable IndexReader*
+ open(Obj *index, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ /** IndexReader is an abstract base class; open() returns the IndexReader
+ * subclass PolyReader, which channels the output of 0 or more SegReaders.
+ *
+ * @param index Either a string filepath or a Folder.
+ * @param snapshot A Snapshot. If not supplied, the most recent snapshot
+ * file will be used.
+ * @param manager An L<IndexManager|Lucy::Index::IndexManager>.
+ * Read-locking is off by default; supplying this argument turns it on.
+ */
+ public inert nullable IndexReader*
+ do_open(IndexReader *self, Obj *index, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ /** Return the maximum number of documents available to the reader, which
+ * is also the highest possible internal document id. Documents which
+ * have been marked as deleted but not yet purged from the index are
+ * included in this count.
+ */
+ public abstract int32_t
+ Doc_Max(IndexReader *self);
+
+ /** Return the number of documents available to the reader, subtracting
+ * any that are marked as deleted.
+ */
+ public abstract int32_t
+ Doc_Count(IndexReader *self);
+
+ /** Return the number of documents which have been marked as deleted but
+ * not yet purged from the index.
+ */
+ public abstract int32_t
+ Del_Count(IndexReader *self);
+
+ /** Return an array with one entry for each segment, corresponding to
+ * segment doc_id start offset.
+ */
+ public abstract incremented I32Array*
+ Offsets(IndexReader *self);
+
+ /** Return an array of all the SegReaders represented within the
+ * IndexReader.
+ */
+ public abstract incremented VArray*
+ Seg_Readers(IndexReader *self);
+
+ /** Fetch a component, or throw an error if the component can't be found.
+ *
+ * @param api The name of the DataReader subclass that the desired
+ * component must implement.
+ */
+ public DataReader*
+ Obtain(IndexReader *self, const CharBuf *api);
+
+ /** Fetch a component, or return NULL if the component can't be found.
+ *
+ * @param api The name of the DataReader subclass that the desired
+ * component must implement.
+ */
+ public nullable DataReader*
+ Fetch(IndexReader *self, const CharBuf *api);
+
+ public void
+ Close(IndexReader *self);
+
+ Hash*
+ Get_Components(IndexReader *self);
+
+ public void
+ Destroy(IndexReader *self);
+}
+
+
diff --git a/core/Lucy/Index/Indexer.c b/core/Lucy/Index/Indexer.c
new file mode 100644
index 0000000..82f18c6
--- /dev/null
+++ b/core/Lucy/Index/Indexer.c
@@ -0,0 +1,603 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INDEXER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Indexer.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/FilePurger.h"
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SegWriter.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/Json.h"
+
+int32_t Indexer_CREATE = 0x00000001;
+int32_t Indexer_TRUNCATE = 0x00000002;
+
+// Release the write lock - if it's there.
+static void
+S_release_write_lock(Indexer *self);
+
+// Release the merge lock - if it's there.
+static void
+S_release_merge_lock(Indexer *self);
+
+// Verify a Folder or derive an FSFolder from a CharBuf path. Call
+// Folder_Initialize() if "create" is true.
+static Folder*
+S_init_folder(Obj *index, bool_t create);
+
+// Find the schema file within a snapshot.
+static CharBuf*
+S_find_schema_file(Snapshot *snapshot);
+
+Indexer*
+Indexer_new(Schema *schema, Obj *index, IndexManager *manager, int32_t flags) {
+ Indexer *self = (Indexer*)VTable_Make_Obj(INDEXER);
+ return Indexer_init(self, schema, index, manager, flags);
+}
+
+Indexer*
+Indexer_init(Indexer *self, Schema *schema, Obj *index,
+ IndexManager *manager, int32_t flags) {
+ bool_t create = (flags & Indexer_CREATE) ? true : false;
+ bool_t truncate = (flags & Indexer_TRUNCATE) ? true : false;
+ Folder *folder = S_init_folder(index, create);
+ Lock *write_lock;
+ CharBuf *latest_snapfile;
+ Snapshot *latest_snapshot = Snapshot_new();
+
+ // Init.
+ self->stock_doc = Doc_new(NULL, 0);
+ self->truncate = false;
+ self->optimize = false;
+ self->prepared = false;
+ self->needs_commit = false;
+ self->snapfile = NULL;
+ self->merge_lock = NULL;
+
+ // Assign.
+ self->folder = folder;
+ self->manager = manager
+ ? (IndexManager*)INCREF(manager)
+ : IxManager_new(NULL, NULL);
+ IxManager_Set_Folder(self->manager, folder);
+
+ // Get a write lock for this folder.
+ write_lock = IxManager_Make_Write_Lock(self->manager);
+ Lock_Clear_Stale(write_lock);
+ if (Lock_Obtain(write_lock)) {
+ // Only assign if successful, otherwise DESTROY unlocks -- bad!
+ self->write_lock = write_lock;
+ }
+ else {
+ DECREF(write_lock);
+ DECREF(self);
+ RETHROW(INCREF(Err_get_error()));
+ }
+
+ // Find the latest snapshot or create a new one.
+ latest_snapfile = IxFileNames_latest_snapshot(folder);
+ if (latest_snapfile) {
+ Snapshot_Read_File(latest_snapshot, folder, latest_snapfile);
+ }
+
+ // Look for an existing Schema if one wasn't supplied.
+ if (schema) {
+ self->schema = (Schema*)INCREF(schema);
+ }
+ else {
+ if (!latest_snapfile) {
+ THROW(ERR, "No Schema supplied, and can't find one in the index");
+ }
+ else {
+ CharBuf *schema_file = S_find_schema_file(latest_snapshot);
+ Hash *dump = (Hash*)Json_slurp_json(folder, schema_file);
+ if (dump) { // read file successfully
+ self->schema = (Schema*)CERTIFY(
+ VTable_Load_Obj(SCHEMA, (Obj*)dump),
+ SCHEMA);
+ schema = self->schema;
+ DECREF(dump);
+ schema_file = NULL;
+ }
+ else {
+ THROW(ERR, "Failed to parse %o", schema_file);
+ }
+ }
+ }
+
+ // If we're clobbering, start with an empty Snapshot and an empty
+ // PolyReader. Otherwise, start with the most recent Snapshot and an
+ // up-to-date PolyReader.
+ if (truncate) {
+ self->snapshot = Snapshot_new();
+ self->polyreader = PolyReader_new(schema, folder, NULL, NULL, NULL);
+ self->truncate = true;
+ }
+ else {
+ // TODO: clone most recent snapshot rather than read it twice.
+ self->snapshot = (Snapshot*)INCREF(latest_snapshot);
+ self->polyreader = latest_snapfile
+ ? PolyReader_open((Obj*)folder, NULL, NULL)
+ : PolyReader_new(schema, folder, NULL, NULL, NULL);
+
+ if (latest_snapfile) {
+ // Make sure than any existing fields which may have been
+ // dynamically added during past indexing sessions get added.
+ Schema *old_schema = PolyReader_Get_Schema(self->polyreader);
+ Schema_Eat(schema, old_schema);
+ }
+ }
+
+ // Zap detritus from previous sessions.
+ {
+ // Note: we have to feed FilePurger with the most recent snapshot file
+ // now, but with the Indexer's snapshot later.
+ FilePurger *file_purger
+ = FilePurger_new(folder, latest_snapshot, self->manager);
+ FilePurger_Purge(file_purger);
+ DECREF(file_purger);
+ }
+
+ // Create a new segment.
+ {
+ int64_t new_seg_num
+ = IxManager_Highest_Seg_Num(self->manager, latest_snapshot) + 1;
+ Lock *merge_lock = IxManager_Make_Merge_Lock(self->manager);
+ uint32_t i, max;
+
+ if (Lock_Is_Locked(merge_lock)) {
+ // If there's a background merge process going on, stay out of its
+ // way.
+ Hash *merge_data = IxManager_Read_Merge_Data(self->manager);
+ Obj *cutoff_obj = merge_data
+ ? Hash_Fetch_Str(merge_data, "cutoff", 6)
+ : NULL;
+ if (!cutoff_obj) {
+ DECREF(merge_lock);
+ DECREF(merge_data);
+ THROW(ERR, "Background merge detected, but can't read merge data");
+ }
+ else {
+ int64_t cutoff = Obj_To_I64(cutoff_obj);
+ if (cutoff >= new_seg_num) {
+ new_seg_num = cutoff + 1;
+ }
+ }
+ DECREF(merge_data);
+ }
+
+ self->segment = Seg_new(new_seg_num);
+
+ // Add all known fields to Segment.
+ {
+ VArray *fields = Schema_All_Fields(schema);
+ for (i = 0, max = VA_Get_Size(fields); i < max; i++) {
+ Seg_Add_Field(self->segment, (CharBuf*)VA_Fetch(fields, i));
+ }
+ DECREF(fields);
+ }
+
+ DECREF(merge_lock);
+ }
+
+ // Create new SegWriter and FilePurger.
+ self->file_purger
+ = FilePurger_new(folder, self->snapshot, self->manager);
+ self->seg_writer = SegWriter_new(self->schema, self->snapshot,
+ self->segment, self->polyreader);
+ SegWriter_Prep_Seg_Dir(self->seg_writer);
+
+ // Grab a local ref to the DeletionsWriter.
+ self->del_writer = (DeletionsWriter*)INCREF(
+ SegWriter_Get_Del_Writer(self->seg_writer));
+
+ DECREF(latest_snapfile);
+ DECREF(latest_snapshot);
+
+ return self;
+}
+
+void
+Indexer_destroy(Indexer *self) {
+ S_release_merge_lock(self);
+ S_release_write_lock(self);
+ DECREF(self->schema);
+ DECREF(self->folder);
+ DECREF(self->segment);
+ DECREF(self->manager);
+ DECREF(self->stock_doc);
+ DECREF(self->polyreader);
+ DECREF(self->del_writer);
+ DECREF(self->snapshot);
+ DECREF(self->seg_writer);
+ DECREF(self->file_purger);
+ DECREF(self->write_lock);
+ DECREF(self->snapfile);
+ SUPER_DESTROY(self, INDEXER);
+}
+
+static Folder*
+S_init_folder(Obj *index, bool_t create) {
+ Folder *folder = NULL;
+
+ // Validate or acquire a Folder.
+ if (Obj_Is_A(index, FOLDER)) {
+ folder = (Folder*)INCREF(index);
+ }
+ else if (Obj_Is_A(index, CHARBUF)) {
+ folder = (Folder*)FSFolder_new((CharBuf*)index);
+ }
+ else {
+ THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
+ }
+
+ // Validate or create the index directory.
+ if (create) {
+ Folder_Initialize(folder);
+ }
+ else {
+ if (!Folder_Check(folder)) {
+ THROW(ERR, "Folder '%o' failed check", Folder_Get_Path(folder));
+ }
+ }
+
+ return folder;
+}
+
+void
+Indexer_add_doc(Indexer *self, Doc *doc, float boost) {
+ SegWriter_Add_Doc(self->seg_writer, doc, boost);
+}
+
+void
+Indexer_delete_by_term(Indexer *self, CharBuf *field, Obj *term) {
+ Schema *schema = self->schema;
+ FieldType *type = Schema_Fetch_Type(schema, field);
+
+ // Raise exception if the field isn't indexed.
+ if (!type || !FType_Indexed(type)) {
+ THROW(ERR, "%o is not an indexed field", field);
+ }
+
+ // Analyze term if appropriate, then zap.
+ if (FType_Is_A(type, FULLTEXTTYPE)) {
+ CERTIFY(term, CHARBUF);
+ {
+ Analyzer *analyzer = Schema_Fetch_Analyzer(schema, field);
+ VArray *terms = Analyzer_Split(analyzer, (CharBuf*)term);
+ Obj *analyzed_term = VA_Fetch(terms, 0);
+ if (analyzed_term) {
+ DelWriter_Delete_By_Term(self->del_writer, field,
+ analyzed_term);
+ }
+ DECREF(terms);
+ }
+ }
+ else {
+ DelWriter_Delete_By_Term(self->del_writer, field, term);
+ }
+}
+
+void
+Indexer_delete_by_query(Indexer *self, Query *query) {
+ DelWriter_Delete_By_Query(self->del_writer, query);
+}
+
+void
+Indexer_add_index(Indexer *self, Obj *index) {
+ Folder *other_folder = NULL;
+ IndexReader *reader = NULL;
+
+ if (Obj_Is_A(index, FOLDER)) {
+ other_folder = (Folder*)INCREF(index);
+ }
+ else if (Obj_Is_A(index, CHARBUF)) {
+ other_folder = (Folder*)FSFolder_new((CharBuf*)index);
+ }
+ else {
+ THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
+ }
+
+ reader = IxReader_open((Obj*)other_folder, NULL, NULL);
+ if (reader == NULL) {
+ THROW(ERR, "Index doesn't seem to contain any data");
+ }
+ else {
+ Schema *schema = self->schema;
+ Schema *other_schema = IxReader_Get_Schema(reader);
+ VArray *other_fields = Schema_All_Fields(other_schema);
+ VArray *seg_readers = IxReader_Seg_Readers(reader);
+ uint32_t i, max;
+
+ // Validate schema compatibility and add fields.
+ Schema_Eat(schema, other_schema);
+
+ // Add fields to Segment.
+ for (i = 0, max = VA_Get_Size(other_fields); i < max; i++) {
+ CharBuf *other_field = (CharBuf*)VA_Fetch(other_fields, i);
+ Seg_Add_Field(self->segment, other_field);
+ }
+ DECREF(other_fields);
+
+ // Add all segments.
+ for (i = 0, max = VA_Get_Size(seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(seg_readers, i);
+ DeletionsReader *del_reader
+ = (DeletionsReader*)SegReader_Fetch(
+ seg_reader, VTable_Get_Name(DELETIONSREADER));
+ Matcher *deletions = del_reader
+ ? DelReader_Iterator(del_reader)
+ : NULL;
+ I32Array *doc_map = DelWriter_Generate_Doc_Map(
+ self->del_writer, deletions,
+ SegReader_Doc_Max(seg_reader),
+ (int32_t)Seg_Get_Count(self->segment));
+ SegWriter_Add_Segment(self->seg_writer, seg_reader, doc_map);
+ DECREF(deletions);
+ DECREF(doc_map);
+ }
+ DECREF(seg_readers);
+ }
+
+ DECREF(reader);
+ DECREF(other_folder);
+}
+
+void
+Indexer_optimize(Indexer *self) {
+ self->optimize = true;
+}
+
+static CharBuf*
+S_find_schema_file(Snapshot *snapshot) {
+ VArray *files = Snapshot_List(snapshot);
+ uint32_t i, max;
+ CharBuf *retval = NULL;
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *file = (CharBuf*)VA_Fetch(files, i);
+ if (CB_Starts_With_Str(file, "schema_", 7)
+ && CB_Ends_With_Str(file, ".json", 5)
+ ) {
+ retval = file;
+ break;
+ }
+ }
+ DECREF(files);
+ return retval;
+}
+
+static bool_t
+S_maybe_merge(Indexer *self, VArray *seg_readers) {
+ bool_t merge_happened = false;
+ uint32_t num_seg_readers = VA_Get_Size(seg_readers);
+ Lock *merge_lock = IxManager_Make_Merge_Lock(self->manager);
+ bool_t got_merge_lock = Lock_Obtain(merge_lock);
+ int64_t cutoff;
+ VArray *to_merge;
+ uint32_t i, max;
+
+ if (got_merge_lock) {
+ self->merge_lock = merge_lock;
+ cutoff = 0;
+ }
+ else {
+ // If something else holds the merge lock, don't interfere.
+ Hash *merge_data = IxManager_Read_Merge_Data(self->manager);
+ if (merge_data) {
+ Obj *cutoff_obj = Hash_Fetch_Str(merge_data, "cutoff", 6);
+ if (cutoff_obj) {
+ cutoff = Obj_To_I64(cutoff_obj);
+ }
+ else {
+ cutoff = I64_MAX;
+ }
+ DECREF(merge_data);
+ }
+ else {
+ cutoff = I64_MAX;
+ }
+ DECREF(merge_lock);
+ }
+
+ // Get a list of segments to recycle. Validate and confirm that there are
+ // no dupes in the list.
+ to_merge = IxManager_Recycle(self->manager, self->polyreader,
+ self->del_writer, cutoff, self->optimize);
+ {
+ Hash *seen = Hash_new(VA_Get_Size(to_merge));
+ for (i = 0, max = VA_Get_Size(to_merge); i < max; i++) {
+ SegReader *seg_reader
+ = (SegReader*)CERTIFY(VA_Fetch(to_merge, i), SEGREADER);
+ CharBuf *seg_name = SegReader_Get_Seg_Name(seg_reader);
+ if (Hash_Fetch(seen, (Obj*)seg_name)) {
+ THROW(ERR, "Recycle() tried to merge segment '%o' twice",
+ seg_name);
+ }
+ Hash_Store(seen, (Obj*)seg_name, INCREF(&EMPTY));
+ }
+ DECREF(seen);
+ }
+
+ // Consolidate segments if either sparse or optimizing forced.
+ for (i = 0, max = VA_Get_Size(to_merge); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(to_merge, i);
+ int64_t seg_num = SegReader_Get_Seg_Num(seg_reader);
+ Matcher *deletions
+ = DelWriter_Seg_Deletions(self->del_writer, seg_reader);
+ I32Array *doc_map = DelWriter_Generate_Doc_Map(
+ self->del_writer, deletions,
+ SegReader_Doc_Max(seg_reader),
+ (int32_t)Seg_Get_Count(self->segment));
+ if (seg_num <= cutoff) {
+ THROW(ERR, "Segment %o violates cutoff (%i64 <= %i64)",
+ SegReader_Get_Seg_Name(seg_reader), seg_num, cutoff);
+ }
+ SegWriter_Merge_Segment(self->seg_writer, seg_reader, doc_map);
+ merge_happened = true;
+ DECREF(deletions);
+ DECREF(doc_map);
+ }
+
+ // Write out new deletions.
+ if (DelWriter_Updated(self->del_writer)) {
+ // Only write out if they haven't all been applied.
+ if (VA_Get_Size(to_merge) != num_seg_readers) {
+ DelWriter_Finish(self->del_writer);
+ }
+ }
+
+ DECREF(to_merge);
+ return merge_happened;
+}
+
+void
+Indexer_prepare_commit(Indexer *self) {
+ VArray *seg_readers = PolyReader_Get_Seg_Readers(self->polyreader);
+ uint32_t num_seg_readers = VA_Get_Size(seg_readers);
+ bool_t merge_happened = false;
+
+ if (!self->write_lock || self->prepared) {
+ THROW(ERR, "Can't call Prepare_Commit() more than once");
+ }
+
+ // Merge existing index data.
+ if (num_seg_readers) {
+ merge_happened = S_maybe_merge(self, seg_readers);
+ }
+
+ // Add a new segment and write a new snapshot file if...
+ if (Seg_Get_Count(self->segment) // Docs/segs added.
+ || merge_happened // Some segs merged.
+ || !Snapshot_Num_Entries(self->snapshot) // Initializing index.
+ || DelWriter_Updated(self->del_writer)
+ ) {
+ Folder *folder = self->folder;
+ Schema *schema = self->schema;
+ Snapshot *snapshot = self->snapshot;
+ CharBuf *old_schema_name = S_find_schema_file(snapshot);
+ uint64_t schema_gen = old_schema_name
+ ? IxFileNames_extract_gen(old_schema_name) + 1
+ : 1;
+ char base36[StrHelp_MAX_BASE36_BYTES];
+ CharBuf *new_schema_name;
+
+ StrHelp_to_base36(schema_gen, &base36);
+ new_schema_name = CB_newf("schema_%s.json", base36);
+
+ // Finish the segment, write schema file.
+ SegWriter_Finish(self->seg_writer);
+ Schema_Write(schema, folder, new_schema_name);
+ if (old_schema_name) {
+ Snapshot_Delete_Entry(snapshot, old_schema_name);
+ }
+ Snapshot_Add_Entry(snapshot, new_schema_name);
+ DECREF(new_schema_name);
+
+ // Write temporary snapshot file.
+ DECREF(self->snapfile);
+ self->snapfile = IxManager_Make_Snapshot_Filename(self->manager);
+ CB_Cat_Trusted_Str(self->snapfile, ".temp", 5);
+ Folder_Delete(folder, self->snapfile);
+ Snapshot_Write_File(snapshot, folder, self->snapfile);
+
+ self->needs_commit = true;
+ }
+
+ // Close reader, so that we can delete its files if appropriate.
+ PolyReader_Close(self->polyreader);
+
+ self->prepared = true;
+}
+
+void
+Indexer_commit(Indexer *self) {
+ // Safety check.
+ if (!self->write_lock) {
+ THROW(ERR, "Can't call commit() more than once");
+ }
+
+ if (!self->prepared) {
+ Indexer_Prepare_Commit(self);
+ }
+
+ if (self->needs_commit) {
+ bool_t success;
+
+ // Rename temp snapshot file.
+ CharBuf *temp_snapfile = CB_Clone(self->snapfile);
+ CB_Chop(self->snapfile, sizeof(".temp") - 1);
+ Snapshot_Set_Path(self->snapshot, self->snapfile);
+ success = Folder_Rename(self->folder, temp_snapfile, self->snapfile);
+ DECREF(temp_snapfile);
+ if (!success) { RETHROW(INCREF(Err_get_error())); }
+
+ // Purge obsolete files.
+ FilePurger_Purge(self->file_purger);
+ }
+
+ // Release locks, invalidating the Indexer.
+ S_release_merge_lock(self);
+ S_release_write_lock(self);
+}
+
+SegWriter*
+Indexer_get_seg_writer(Indexer *self) {
+ return self->seg_writer;
+}
+
+Doc*
+Indexer_get_stock_doc(Indexer *self) {
+ return self->stock_doc;
+}
+
+static void
+S_release_write_lock(Indexer *self) {
+ if (self->write_lock) {
+ Lock_Release(self->write_lock);
+ DECREF(self->write_lock);
+ self->write_lock = NULL;
+ }
+}
+
+static void
+S_release_merge_lock(Indexer *self) {
+ if (self->merge_lock) {
+ Lock_Release(self->merge_lock);
+ DECREF(self->merge_lock);
+ self->merge_lock = NULL;
+ }
+}
+
+
diff --git a/core/Lucy/Index/Indexer.cfh b/core/Lucy/Index/Indexer.cfh
new file mode 100644
index 0000000..ad30281
--- /dev/null
+++ b/core/Lucy/Index/Indexer.cfh
@@ -0,0 +1,149 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Build inverted indexes.
+ *
+ * The Indexer class is Apache Lucy's primary tool for managing the content of
+ * inverted indexes, which may later be searched using
+ * L<IndexSearcher|Lucy::Search::IndexSearcher>.
+ *
+ * In general, only one Indexer at a time may write to an index safely. If a
+ * write lock cannot be secured, new() will throw an exception.
+ *
+ * If an index is located on a shared volume, each writer application must
+ * identify itself by supplying an
+ * L<IndexManager|Lucy::Index::IndexManager> with a unique
+ * <code>host</code> id to Indexer's constructor or index corruption will
+ * occur. See L<Lucy::Docs::FileLocking> for a detailed discussion.
+ *
+ * Note: at present, Delete_By_Term() and Delete_By_Query() only affect
+ * documents which had been previously committed to the index -- and not any
+ * documents added this indexing session but not yet committed. This may
+ * change in a future update.
+ */
+class Lucy::Index::Indexer inherits Lucy::Object::Obj {
+
+ Schema *schema;
+ Folder *folder;
+ Segment *segment;
+ IndexManager *manager;
+ PolyReader *polyreader;
+ Snapshot *snapshot;
+ SegWriter *seg_writer;
+ DeletionsWriter *del_writer;
+ FilePurger *file_purger;
+ Lock *write_lock;
+ Lock *merge_lock;
+ Doc *stock_doc;
+ CharBuf *snapfile;
+ bool_t truncate;
+ bool_t optimize;
+ bool_t needs_commit;
+ bool_t prepared;
+
+ public inert int32_t TRUNCATE;
+ public inert int32_t CREATE;
+
+ public inert incremented Indexer*
+ new(Schema *schema = NULL, Obj *index, IndexManager *manager = NULL,
+ int32_t flags = 0);
+
+ /** Open a new Indexer. If the index already exists, update it.
+ *
+ * @param schema A Schema.
+ * @param index Either a string filepath or a Folder.
+ * @param manager An IndexManager.
+ * @param flags Flags governing behavior.
+ */
+ public inert Indexer*
+ init(Indexer *self, Schema *schema = NULL, Obj *index,
+ IndexManager *manager = NULL, int32_t flags = 0);
+
+ /** Add a document to the index.
+ *
+ * @param doc A Lucy::Document::Doc object.
+ * @param boost A floating point weight which affects how this document
+ * scores.
+ */
+ public void
+ Add_Doc(Indexer *self, Doc *doc, float boost = 1.0);
+
+ /** Absorb an existing index into this one. The two indexes must
+ * have matching Schemas.
+ *
+ * @param index Either an index path name or a Folder.
+ */
+ public void
+ Add_Index(Indexer *self, Obj *index);
+
+ /** Mark documents which contain the supplied term as deleted, so that
+ * they will be excluded from search results and eventually removed
+ * altogether. The change is not apparent to search apps until after
+ * Commit() succeeds.
+ *
+ * @param field The name of an indexed field. (If it is not spec'd as
+ * <code>indexed</code>, an error will occur.)
+ * @param term The term which identifies docs to be marked as deleted. If
+ * <code>field</code> is associated with an Analyzer, <code>term</code>
+ * will be processed automatically (so don't pre-process it yourself).
+ */
+ public void
+ Delete_By_Term(Indexer *self, CharBuf *field, Obj *term);
+
+ /** Mark documents which match the supplied Query as deleted.
+ *
+ * @param query A L<Query|Lucy::Search::Query>.
+ */
+ public void
+ Delete_By_Query(Indexer *self, Query *query);
+
+ /** Optimize the index for search-time performance. This may take a
+ * while, as it can involve rewriting large amounts of data.
+ */
+ public void
+ Optimize(Indexer *self);
+
+ /** Commit any changes made to the index. Until this is called, none of
+ * the changes made during an indexing session are permanent.
+ *
+ * Calling Commit() invalidates the Indexer, so if you want to make more
+ * changes you'll need a new one.
+ */
+ public void
+ Commit(Indexer *self);
+
+ /** Perform the expensive setup for Commit() in advance, so that Commit()
+ * completes quickly. (If Prepare_Commit() is not called explicitly by
+ * the user, Commit() will call it internally.)
+ */
+ public void
+ Prepare_Commit(Indexer *self);
+
+ /** Accessor for seg_writer member var.
+ */
+ public SegWriter*
+ Get_Seg_Writer(Indexer *self);
+
+ Doc*
+ Get_Stock_Doc(Indexer *self);
+
+ public void
+ Destroy(Indexer *self);
+}
+
+
diff --git a/core/Lucy/Index/Inverter.c b/core/Lucy/Index/Inverter.c
new file mode 100644
index 0000000..6e5615a
--- /dev/null
+++ b/core/Lucy/Index/Inverter.c
@@ -0,0 +1,259 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INVERTER
+#define C_LUCY_INVERTERENTRY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/BlobType.h"
+#include "Lucy/Plan/NumericType.h"
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Plan/TextType.h"
+#include "Lucy/Plan/Schema.h"
+
+Inverter*
+Inverter_new(Schema *schema, Segment *segment) {
+ Inverter *self = (Inverter*)VTable_Make_Obj(INVERTER);
+ return Inverter_init(self, schema, segment);
+}
+
+Inverter*
+Inverter_init(Inverter *self, Schema *schema, Segment *segment) {
+ // Init.
+ self->tick = -1;
+ self->doc = NULL;
+ self->sorted = false;
+ self->blank = InvEntry_new(NULL, NULL, 0);
+ self->current = self->blank;
+
+ // Derive.
+ self->entry_pool = VA_new(Schema_Num_Fields(schema));
+ self->entries = VA_new(Schema_Num_Fields(schema));
+
+ // Assign.
+ self->schema = (Schema*)INCREF(schema);
+ self->segment = (Segment*)INCREF(segment);
+
+ return self;
+}
+
+void
+Inverter_destroy(Inverter *self) {
+ Inverter_Clear(self);
+ DECREF(self->blank);
+ DECREF(self->entries);
+ DECREF(self->entry_pool);
+ DECREF(self->schema);
+ DECREF(self->segment);
+ SUPER_DESTROY(self, INVERTER);
+}
+
+uint32_t
+Inverter_iterate(Inverter *self) {
+ self->tick = -1;
+ if (!self->sorted) {
+ VA_Sort(self->entries, NULL, NULL);
+ self->sorted = true;
+ }
+ return VA_Get_Size(self->entries);
+}
+
+int32_t
+Inverter_next(Inverter *self) {
+ self->current = (InverterEntry*)VA_Fetch(self->entries, ++self->tick);
+ if (!self->current) { self->current = self->blank; } // Exhausted.
+ return self->current->field_num;
+}
+
+void
+Inverter_set_doc(Inverter *self, Doc *doc) {
+ Inverter_Clear(self); // Zap all cached field values and Inversions.
+ self->doc = (Doc*)INCREF(doc);
+}
+
+void
+Inverter_set_boost(Inverter *self, float boost) {
+ self->boost = boost;
+}
+
+float
+Inverter_get_boost(Inverter *self) {
+ return self->boost;
+}
+
+Doc*
+Inverter_get_doc(Inverter *self) {
+ return self->doc;
+}
+
+CharBuf*
+Inverter_get_field_name(Inverter *self) {
+ return self->current->field;
+}
+
+Obj*
+Inverter_get_value(Inverter *self) {
+ return self->current->value;
+}
+
+FieldType*
+Inverter_get_type(Inverter *self) {
+ return self->current->type;
+}
+
+Analyzer*
+Inverter_get_analyzer(Inverter *self) {
+ return self->current->analyzer;
+}
+
+Similarity*
+Inverter_get_similarity(Inverter *self) {
+ return self->current->sim;
+}
+
+Inversion*
+Inverter_get_inversion(Inverter *self) {
+ return self->current->inversion;
+}
+
+
+void
+Inverter_add_field(Inverter *self, InverterEntry *entry) {
+ // Get an Inversion, going through analyzer if appropriate.
+ if (entry->analyzer) {
+ DECREF(entry->inversion);
+ entry->inversion = Analyzer_Transform_Text(entry->analyzer,
+ (CharBuf*)entry->value);
+ Inversion_Invert(entry->inversion);
+ }
+ else if (entry->indexed || entry->highlightable) {
+ ViewCharBuf *value = (ViewCharBuf*)entry->value;
+ size_t token_len = ViewCB_Get_Size(value);
+ Token *seed = Token_new((char*)ViewCB_Get_Ptr8(value),
+ token_len, 0, token_len, 1.0f, 1);
+ DECREF(entry->inversion);
+ entry->inversion = Inversion_new(seed);
+ DECREF(seed);
+ Inversion_Invert(entry->inversion); // Nearly a no-op.
+ }
+
+ // Prime the iterator.
+ VA_Push(self->entries, INCREF(entry));
+ self->sorted = false;
+}
+
+void
+Inverter_clear(Inverter *self) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->entries); i < max; i++) {
+ InvEntry_Clear(VA_Fetch(self->entries, i));
+ }
+ VA_Clear(self->entries);
+ self->tick = -1;
+ DECREF(self->doc);
+ self->doc = NULL;
+}
+
+InverterEntry*
+InvEntry_new(Schema *schema, const CharBuf *field, int32_t field_num) {
+ InverterEntry *self = (InverterEntry*)VTable_Make_Obj(INVERTERENTRY);
+ return InvEntry_init(self, schema, field, field_num);
+}
+
+InverterEntry*
+InvEntry_init(InverterEntry *self, Schema *schema, const CharBuf *field,
+ int32_t field_num) {
+ self->field_num = field_num;
+ self->field = field ? CB_Clone(field) : NULL;
+ self->inversion = NULL;
+
+ if (schema) {
+ self->analyzer
+ = (Analyzer*)INCREF(Schema_Fetch_Analyzer(schema, field));
+ self->sim = (Similarity*)INCREF(Schema_Fetch_Sim(schema, field));
+ self->type = (FieldType*)INCREF(Schema_Fetch_Type(schema, field));
+ if (!self->type) { THROW(ERR, "Unknown field: '%o'", field); }
+
+ uint8_t prim_id = FType_Primitive_ID(self->type);
+ switch (prim_id & FType_PRIMITIVE_ID_MASK) {
+ case FType_TEXT:
+ self->value = (Obj*)ViewCB_new_from_trusted_utf8(NULL, 0);
+ break;
+ case FType_BLOB:
+ self->value = (Obj*)ViewBB_new(NULL, 0);
+ break;
+ case FType_INT32:
+ self->value = (Obj*)Int32_new(0);
+ break;
+ case FType_INT64:
+ self->value = (Obj*)Int64_new(0);
+ break;
+ case FType_FLOAT32:
+ self->value = (Obj*)Float32_new(0);
+ break;
+ case FType_FLOAT64:
+ self->value = (Obj*)Float64_new(0);
+ break;
+ default:
+ THROW(ERR, "Unrecognized primitive id: %i8", prim_id);
+ }
+
+ self->indexed = FType_Indexed(self->type);
+ if (self->indexed && FType_Is_A(self->type, NUMERICTYPE)) {
+ THROW(ERR, "Field '%o' spec'd as indexed, but numerical types cannot "
+ "be indexed yet", field);
+ }
+ if (FType_Is_A(self->type, FULLTEXTTYPE)) {
+ self->highlightable
+ = FullTextType_Highlightable((FullTextType*)self->type);
+ }
+ }
+ return self;
+}
+
+void
+InvEntry_destroy(InverterEntry *self) {
+ DECREF(self->field);
+ DECREF(self->value);
+ DECREF(self->analyzer);
+ DECREF(self->type);
+ DECREF(self->sim);
+ DECREF(self->inversion);
+ SUPER_DESTROY(self, INVERTERENTRY);
+}
+
+void
+InvEntry_clear(InverterEntry *self) {
+ DECREF(self->inversion);
+ self->inversion = NULL;
+}
+
+int32_t
+InvEntry_compare_to(InverterEntry *self, Obj *other) {
+ InverterEntry *competitor
+ = (InverterEntry*)CERTIFY(other, INVERTERENTRY);
+ return self->field_num - competitor->field_num;
+}
+
+
diff --git a/core/Lucy/Index/Inverter.cfh b/core/Lucy/Index/Inverter.cfh
new file mode 100644
index 0000000..ac07964
--- /dev/null
+++ b/core/Lucy/Index/Inverter.cfh
@@ -0,0 +1,176 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Invert documents.
+ *
+ * Inverter's role is to prepare the content of a Doc for addition to various
+ * DataWriters, by associating fields with FieldTypes, inverting their
+ * content when appropriate, and marshalling them into an iterable form.
+ */
+
+class Lucy::Index::Inverter inherits Lucy::Object::Obj {
+
+ Schema *schema;
+ Segment *segment;
+ Doc *doc;
+ VArray *entries; /* Entries for the current Doc. */
+ VArray *entry_pool; /* Cached entry per field. */
+ InverterEntry *current; /* Current entry while iterating. */
+ InverterEntry *blank; /* Used when iterator is exhausted. */
+ float boost;
+ int32_t tick;
+ bool_t sorted;
+
+ inert incremented Inverter*
+ new(Schema *schema, Segment *segment);
+
+ inert Inverter*
+ init(Inverter *self, Schema *schema, Segment *segment);
+
+ /** Invert the document, first calling Set_Doc(), then Add_Field() for
+ * each field in the Doc.
+ */
+ public void
+ Invert_Doc(Inverter *self, Doc *doc);
+
+ /** Set the object's <code>doc</code> member. Calls Clear() as side
+ * effect.
+ */
+ public void
+ Set_Doc(Inverter *self, Doc *doc);
+
+ /** Set the object's <code>boost</code> member.
+ */
+ public void
+ Set_Boost(Inverter *self, float boost);
+
+ /** Add a field to the Inverter. If the field is indexed/analyzed, invert
+ * it.
+ *
+ * The InverterEntry's value should have already been set to the field's
+ * value when Add_Field() is called.
+ */
+ void
+ Add_Field(Inverter *self, InverterEntry *entry);
+
+ /** Remove the cached Doc and everything derived from it.
+ */
+ public void
+ Clear(Inverter *self);
+
+ /** Reset the iterator and prepare to cycle through any fields that have
+ * been added.
+ *
+ * @return the number of fields which will be iterated over.
+ */
+ public uint32_t
+ Iterate(Inverter *self);
+
+ /** Proceed to the next field. Fields are iterated in order of Segment
+ * field number.
+ */
+ public int32_t
+ Next(Inverter *self);
+
+ /** Return the current doc, or NULL if there isn't one.
+ */
+ public nullable Doc*
+ Get_Doc(Inverter *self);
+
+ /** Return the current boost.
+ */
+ public float
+ Get_Boost(Inverter *self);
+
+ /** Return the current field's name, or NULL if the iterator is exhausted.
+ */
+ public nullable CharBuf*
+ Get_Field_Name(Inverter *self);
+
+ /** Return the current field's value, or NULL if the iterator is
+ * exhausted.
+ */
+ public nullable Obj*
+ Get_Value(Inverter *self);
+
+ /** Return the FieldType for the current field, or NULL if the iterator is
+ * exhausted.
+ */
+ public nullable FieldType*
+ Get_Type(Inverter *self);
+
+ /** Return the Analyzer for the current field, or NULL if the iterator is
+ * exhausted.
+ */
+ public nullable Analyzer*
+ Get_Analyzer(Inverter *self);
+
+ /** Return the Analyzer for the current field, or NULL if the iterator is
+ * exhausted.
+ */
+ public nullable Similarity*
+ Get_Similarity(Inverter *self);
+
+ /** Return the Inversion for the current field, provided that that field
+ * is indexed; return NULL if the iterator is exhausted.
+ */
+ public nullable Inversion*
+ Get_Inversion(Inverter *self);
+
+ public void
+ Destroy(Inverter *self);
+}
+
+/** Cached information about fields.
+ *
+ * Inverter needs to check certain field characteristics frequently. Since
+ * field definitions are unchanging, we cache them in an InverterEntry object.
+ */
+private final class Lucy::Index::Inverter::InverterEntry cnick InvEntry
+ inherits Lucy::Object::Obj {
+
+ int32_t field_num;
+ CharBuf *field;
+ Obj *value;
+ Inversion *inversion;
+ FieldType *type;
+ Analyzer *analyzer;
+ Similarity *sim;
+ bool_t indexed;
+ bool_t highlightable;
+
+ inert incremented InverterEntry*
+ new(Schema *schema = NULL, const CharBuf *field_name, int32_t field_num);
+
+ inert InverterEntry*
+ init(InverterEntry *self = NULL, Schema *schema,
+ const CharBuf *field_name, int32_t field_num);
+
+ public int32_t
+ Compare_To(InverterEntry *self, Obj *other);
+
+ /** Expunge all transient data.
+ */
+ void
+ Clear(InverterEntry *self);
+
+ public void
+ Destroy(InverterEntry *self);
+}
+
+
diff --git a/core/Lucy/Index/LexIndex.c b/core/Lucy/Index/LexIndex.c
new file mode 100644
index 0000000..dca9adf
--- /dev/null
+++ b/core/Lucy/Index/LexIndex.c
@@ -0,0 +1,196 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LEXINDEX
+#define C_LUCY_TERMINFO
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/LexIndex.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Index/TermStepper.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+
+// Read the data we've arrived at after a seek operation.
+static void
+S_read_entry(LexIndex *self);
+
+LexIndex*
+LexIndex_new(Schema *schema, Folder *folder, Segment *segment,
+ const CharBuf *field) {
+ LexIndex *self = (LexIndex*)VTable_Make_Obj(LEXINDEX);
+ return LexIndex_init(self, schema, folder, segment, field);
+}
+
+LexIndex*
+LexIndex_init(LexIndex *self, Schema *schema, Folder *folder,
+ Segment *segment, const CharBuf *field) {
+ int32_t field_num = Seg_Field_Num(segment, field);
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *ixix_file = CB_newf("%o/lexicon-%i32.ixix", seg_name, field_num);
+ CharBuf *ix_file = CB_newf("%o/lexicon-%i32.ix", seg_name, field_num);
+ Architecture *arch = Schema_Get_Architecture(schema);
+
+ // Init.
+ Lex_init((Lexicon*)self, field);
+ self->tinfo = TInfo_new(0);
+ self->tick = 0;
+
+ // Derive
+ self->field_type = Schema_Fetch_Type(schema, field);
+ if (!self->field_type) {
+ CharBuf *mess = MAKE_MESS("Unknown field: '%o'", field);
+ DECREF(ix_file);
+ DECREF(ixix_file);
+ DECREF(self);
+ Err_throw_mess(ERR, mess);
+ }
+ INCREF(self->field_type);
+ self->term_stepper = FType_Make_Term_Stepper(self->field_type);
+ self->ixix_in = Folder_Open_In(folder, ixix_file);
+ if (!self->ixix_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(ixix_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->ix_in = Folder_Open_In(folder, ix_file);
+ if (!self->ix_in) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(ix_file);
+ DECREF(ixix_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->index_interval = Arch_Index_Interval(arch);
+ self->skip_interval = Arch_Skip_Interval(arch);
+ self->size = (int32_t)(InStream_Length(self->ixix_in) / sizeof(int64_t));
+ self->offsets = (int64_t*)InStream_Buf(self->ixix_in,
+ (size_t)InStream_Length(self->ixix_in));
+
+ DECREF(ixix_file);
+ DECREF(ix_file);
+
+ return self;
+}
+
+void
+LexIndex_destroy(LexIndex *self) {
+ DECREF(self->field_type);
+ DECREF(self->ixix_in);
+ DECREF(self->ix_in);
+ DECREF(self->term_stepper);
+ DECREF(self->tinfo);
+ SUPER_DESTROY(self, LEXINDEX);
+}
+
+int32_t
+LexIndex_get_term_num(LexIndex *self) {
+ return (self->index_interval * self->tick) - 1;
+}
+
+Obj*
+LexIndex_get_term(LexIndex *self) {
+ return TermStepper_Get_Value(self->term_stepper);
+}
+
+TermInfo*
+LexIndex_get_term_info(LexIndex *self) {
+ return self->tinfo;
+}
+
+static void
+S_read_entry(LexIndex *self) {
+ InStream *ix_in = self->ix_in;
+ TermInfo *tinfo = self->tinfo;
+ int64_t offset = (int64_t)NumUtil_decode_bigend_u64(self->offsets + self->tick);
+ InStream_Seek(ix_in, offset);
+ TermStepper_Read_Key_Frame(self->term_stepper, ix_in);
+ tinfo->doc_freq = InStream_Read_C32(ix_in);
+ tinfo->post_filepos = InStream_Read_C64(ix_in);
+ tinfo->skip_filepos = tinfo->doc_freq >= self->skip_interval
+ ? InStream_Read_C64(ix_in)
+ : 0;
+ tinfo->lex_filepos = InStream_Read_C64(ix_in);
+}
+
+void
+LexIndex_seek(LexIndex *self, Obj *target) {
+ TermStepper *term_stepper = self->term_stepper;
+ InStream *ix_in = self->ix_in;
+ FieldType *type = self->field_type;
+ int32_t lo = 0;
+ int32_t hi = self->size - 1;
+ int32_t result = -100;
+
+ if (target == NULL || self->size == 0) {
+ self->tick = 0;
+ return;
+ }
+ else {
+ if (!Obj_Is_A(target, CHARBUF)) {
+ THROW(ERR, "Target is a %o, and not comparable to a %o",
+ Obj_Get_Class_Name(target), VTable_Get_Name(CHARBUF));
+ }
+ /* TODO:
+ Obj *first_obj = VA_Fetch(terms, 0);
+ if (!Obj_Is_A(target, Obj_Get_VTable(first_obj))) {
+ THROW(ERR, "Target is a %o, and not comparable to a %o",
+ Obj_Get_Class_Name(target), Obj_Get_Class_Name(first_obj));
+ }
+ */
+ }
+
+ // Divide and conquer.
+ while (hi >= lo) {
+ const int32_t mid = lo + ((hi - lo) / 2);
+ const int64_t offset
+ = (int64_t)NumUtil_decode_bigend_u64(self->offsets + mid);
+ InStream_Seek(ix_in, offset);
+ TermStepper_Read_Key_Frame(term_stepper, ix_in);
+
+ // Compare values. There is no need for a NULL-check because the term
+ // number is alway between 0 and self->size - 1.
+ Obj *value = TermStepper_Get_Value(term_stepper);
+ int32_t comparison = FType_Compare_Values(type, target, value);
+
+ if (comparison < 0) {
+ hi = mid - 1;
+ }
+ else if (comparison > 0) {
+ lo = mid + 1;
+ }
+ else {
+ result = mid;
+ break;
+ }
+ }
+
+ // Record the index of the entry we've seeked to, then read entry.
+ self->tick = hi == -1 // indicating that target lt first entry
+ ? 0
+ : result == -100 // if result is still -100, it wasn't set
+ ? hi
+ : result;
+ S_read_entry(self);
+}
+
+
diff --git a/core/Lucy/Index/LexIndex.cfh b/core/Lucy/Index/LexIndex.cfh
new file mode 100644
index 0000000..1897947
--- /dev/null
+++ b/core/Lucy/Index/LexIndex.cfh
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::LexIndex inherits Lucy::Index::Lexicon {
+
+ FieldType *field_type;
+ InStream *ixix_in;
+ InStream *ix_in;
+ int64_t *offsets;
+ int32_t tick;
+ int32_t size;
+ int32_t index_interval;
+ int32_t skip_interval;
+ TermStepper *term_stepper;
+ TermInfo *tinfo;
+
+ inert incremented LexIndex*
+ new(Schema *schema, Folder *folder, Segment *segment,
+ const CharBuf *field);
+
+ inert LexIndex*
+ init(LexIndex *self, Schema *schema, Folder *folder, Segment *segment,
+ const CharBuf *field);
+
+ public void
+ Seek(LexIndex *self, Obj *target = NULL);
+
+ int32_t
+ Get_Term_Num(LexIndex *self);
+
+ nullable TermInfo*
+ Get_Term_Info(LexIndex *self);
+
+ public nullable Obj*
+ Get_Term(LexIndex *self);
+
+ public void
+ Destroy(LexIndex *self);
+}
+
+
diff --git a/core/Lucy/Index/Lexicon.c b/core/Lucy/Index/Lexicon.c
new file mode 100644
index 0000000..c26b1e0
--- /dev/null
+++ b/core/Lucy/Index/Lexicon.c
@@ -0,0 +1,40 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LEXICON
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Lexicon.h"
+
+Lexicon*
+Lex_init(Lexicon *self, const CharBuf *field) {
+ self->field = CB_Clone(field);
+ ABSTRACT_CLASS_CHECK(self, LEXICON);
+ return self;
+}
+
+CharBuf*
+Lex_get_field(Lexicon *self) {
+ return self->field;
+}
+
+void
+Lex_destroy(Lexicon *self) {
+ DECREF(self->field);
+ SUPER_DESTROY(self, LEXICON);
+}
+
+
diff --git a/core/Lucy/Index/Lexicon.cfh b/core/Lucy/Index/Lexicon.cfh
new file mode 100644
index 0000000..10dae9b
--- /dev/null
+++ b/core/Lucy/Index/Lexicon.cfh
@@ -0,0 +1,80 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Iterator for a field's terms.
+ *
+ * A Lexicon is an iterator which provides access to all the unique terms for
+ * a given field in sorted order.
+ *
+ * If an index consists of two documents with a 'content' field holding "three
+ * blind mice" and "three musketeers" respectively, then iterating through the
+ * 'content' field's lexicon would produce this list:
+ *
+ * blind
+ * mice
+ * musketeers
+ * three
+ */
+
+class Lucy::Index::Lexicon cnick Lex inherits Lucy::Object::Obj {
+
+ CharBuf *field;
+
+ public inert Lexicon*
+ init(Lexicon *self, const CharBuf *field);
+
+ public void
+ Destroy(Lexicon *self);
+
+ /** Seek the Lexicon to the first iterator state which is greater than or
+ * equal to <code>target</code>. If <code>target</code> is NULL,
+ * reset the iterator.
+ */
+ public abstract void
+ Seek(Lexicon *self, Obj *target = NULL);
+
+ /** Proceed to the next term.
+ *
+ * @return true until the iterator is exhausted, then false.
+ */
+ public abstract bool_t
+ Next(Lexicon *self);
+
+ /** Reset the iterator. Next() must be called to proceed to the first
+ * element.
+ */
+ public abstract void
+ Reset(Lexicon *self);
+
+ /** Return the number of documents that the current term appears in at
+ * least once. Deleted documents may be included in the count.
+ */
+ public abstract int32_t
+ Doc_Freq(Lexicon *self);
+
+ /** Return the current term, or NULL if the iterator is not in a valid
+ * state.
+ */
+ public abstract nullable Obj*
+ Get_Term(Lexicon *self);
+
+ public CharBuf*
+ Get_Field(Lexicon *self);
+}
+
+
diff --git a/core/Lucy/Index/LexiconReader.c b/core/Lucy/Index/LexiconReader.c
new file mode 100644
index 0000000..d9f551d
--- /dev/null
+++ b/core/Lucy/Index/LexiconReader.c
@@ -0,0 +1,245 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LEXICONREADER
+#define C_LUCY_POLYLEXICONREADER
+#define C_LUCY_DEFAULTLEXICONREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Index/PolyLexicon.h"
+#include "Lucy/Index/SegLexicon.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Store/Folder.h"
+
+LexiconReader*
+LexReader_init(LexiconReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ ABSTRACT_CLASS_CHECK(self, LEXICONREADER);
+ return self;
+}
+
+LexiconReader*
+LexReader_aggregator(LexiconReader *self, VArray *readers, I32Array *offsets) {
+ UNUSED_VAR(self);
+ return (LexiconReader*)PolyLexReader_new(readers, offsets);
+}
+
+PolyLexiconReader*
+PolyLexReader_new(VArray *readers, I32Array *offsets) {
+ PolyLexiconReader *self
+ = (PolyLexiconReader*)VTable_Make_Obj(POLYLEXICONREADER);
+ return PolyLexReader_init(self, readers, offsets);
+}
+
+PolyLexiconReader*
+PolyLexReader_init(PolyLexiconReader *self, VArray *readers,
+ I32Array *offsets) {
+ uint32_t i, max;
+ Schema *schema = NULL;
+ for (i = 0, max = VA_Get_Size(readers); i < max; i++) {
+ LexiconReader *reader
+ = (LexiconReader*)CERTIFY(VA_Fetch(readers, i), LEXICONREADER);
+ if (!schema) { schema = LexReader_Get_Schema(reader); }
+ }
+ LexReader_init((LexiconReader*)self, schema, NULL, NULL, NULL, -1);
+ self->readers = (VArray*)INCREF(readers);
+ self->offsets = (I32Array*)INCREF(offsets);
+ return self;
+}
+
+void
+PolyLexReader_close(PolyLexiconReader *self) {
+ if (self->readers) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ LexiconReader *reader
+ = (LexiconReader*)VA_Fetch(self->readers, i);
+ if (reader) { LexReader_Close(reader); }
+ }
+ VA_Clear(self->readers);
+ }
+}
+
+void
+PolyLexReader_destroy(PolyLexiconReader *self) {
+ DECREF(self->readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYLEXICONREADER);
+}
+
+Lexicon*
+PolyLexReader_lexicon(PolyLexiconReader *self, const CharBuf *field,
+ Obj *term) {
+ PolyLexicon *lexicon = NULL;
+
+ if (field != NULL) {
+ Schema *schema = PolyLexReader_Get_Schema(self);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ if (type != NULL) {
+ lexicon = PolyLex_new(field, self->readers);
+ if (!PolyLex_Get_Num_Seg_Lexicons(lexicon)) {
+ DECREF(lexicon);
+ return NULL;
+ }
+ if (term) { PolyLex_Seek(lexicon, term); }
+ }
+ }
+
+ return (Lexicon*)lexicon;
+}
+
+uint32_t
+PolyLexReader_doc_freq(PolyLexiconReader *self, const CharBuf *field,
+ Obj *term) {
+ uint32_t doc_freq = 0;
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->readers); i < max; i++) {
+ LexiconReader *reader = (LexiconReader*)VA_Fetch(self->readers, i);
+ if (reader) {
+ doc_freq += LexReader_Doc_Freq(reader, field, term);
+ }
+ }
+ return doc_freq;
+}
+
+DefaultLexiconReader*
+DefLexReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick) {
+ DefaultLexiconReader *self
+ = (DefaultLexiconReader*)VTable_Make_Obj(DEFAULTLEXICONREADER);
+ return DefLexReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+// Indicate whether it is safe to build a SegLexicon using the given
+// parameters. Will return false if the field is not indexed or if no terms
+// are present for this field in this segment.
+static bool_t
+S_has_data(Schema *schema, Folder *folder, Segment *segment, CharBuf *field) {
+ FieldType *type = Schema_Fetch_Type(schema, field);
+
+ if (!type || !FType_Indexed(type)) {
+ // If the field isn't indexed, bail out.
+ return false;
+ }
+ else {
+ // Bail out if there are no terms for this field in this segment.
+ int32_t field_num = Seg_Field_Num(segment, field);
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *file = CB_newf("%o/lexicon-%i32.dat", seg_name, field_num);
+ bool_t retval = Folder_Exists(folder, file);
+ DECREF(file);
+ return retval;
+ }
+}
+
+DefaultLexiconReader*
+DefLexReader_init(DefaultLexiconReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ Segment *segment;
+ uint32_t i, max;
+
+ // Init.
+ LexReader_init((LexiconReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ segment = DefLexReader_Get_Segment(self);
+
+ // Build an array of SegLexicon objects.
+ self->lexicons = VA_new(Schema_Num_Fields(schema));
+ for (i = 1, max = Schema_Num_Fields(schema) + 1; i < max; i++) {
+ CharBuf *field = Seg_Field_Name(segment, i);
+ if (field && S_has_data(schema, folder, segment, field)) {
+ SegLexicon *lexicon = SegLex_new(schema, folder, segment, field);
+ VA_Store(self->lexicons, i, (Obj*)lexicon);
+ }
+ }
+
+ return self;
+}
+
+void
+DefLexReader_close(DefaultLexiconReader *self) {
+ DECREF(self->lexicons);
+ self->lexicons = NULL;
+}
+
+void
+DefLexReader_destroy(DefaultLexiconReader *self) {
+ DECREF(self->lexicons);
+ SUPER_DESTROY(self, DEFAULTLEXICONREADER);
+}
+
+Lexicon*
+DefLexReader_lexicon(DefaultLexiconReader *self, const CharBuf *field,
+ Obj *term) {
+ int32_t field_num = Seg_Field_Num(self->segment, field);
+ SegLexicon *orig = (SegLexicon*)VA_Fetch(self->lexicons, field_num);
+ SegLexicon *lexicon = NULL;
+
+ if (orig) { // i.e. has data
+ lexicon
+ = SegLex_new(self->schema, self->folder, self->segment, field);
+ SegLex_Seek(lexicon, term);
+ }
+
+ return (Lexicon*)lexicon;
+}
+
+static TermInfo*
+S_find_tinfo(DefaultLexiconReader *self, const CharBuf *field, Obj *target) {
+ if (field != NULL && target != NULL) {
+ int32_t field_num = Seg_Field_Num(self->segment, field);
+ SegLexicon *lexicon
+ = (SegLexicon*)VA_Fetch(self->lexicons, field_num);
+
+ if (lexicon) {
+ // Iterate until the result is ge the term.
+ SegLex_Seek(lexicon, target);
+
+ //if found matches target, return info; otherwise NULL
+ {
+ Obj *found = SegLex_Get_Term(lexicon);
+ if (found && Obj_Equals(target, found)) {
+ return SegLex_Get_Term_Info(lexicon);
+ }
+ }
+ }
+ }
+ return NULL;
+}
+
+TermInfo*
+DefLexReader_fetch_term_info(DefaultLexiconReader *self,
+ const CharBuf *field, Obj *target) {
+ TermInfo *tinfo = S_find_tinfo(self, field, target);
+ return tinfo ? TInfo_Clone(tinfo) : NULL;
+}
+
+uint32_t
+DefLexReader_doc_freq(DefaultLexiconReader *self, const CharBuf *field,
+ Obj *term) {
+ TermInfo *tinfo = S_find_tinfo(self, field, term);
+ return tinfo ? TInfo_Get_Doc_Freq(tinfo) : 0;
+}
+
+
diff --git a/core/Lucy/Index/LexiconReader.cfh b/core/Lucy/Index/LexiconReader.cfh
new file mode 100644
index 0000000..257046a
--- /dev/null
+++ b/core/Lucy/Index/LexiconReader.cfh
@@ -0,0 +1,119 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read Lexicon data.
+ *
+ * LexiconReader reads term dictionary information.
+ */
+abstract class Lucy::Index::LexiconReader cnick LexReader
+ inherits Lucy::Index::DataReader {
+
+ inert LexiconReader*
+ init(LexiconReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ /** Return a new Lexicon for the given <code>field</code>. Will return
+ * NULL if either the field is not indexed, or if no documents contain a
+ * value for the field.
+ *
+ * @param field Field name.
+ * @param term Pre-locate the Lexicon to this term.
+ */
+ public abstract incremented nullable Lexicon*
+ Lexicon(LexiconReader *self, const CharBuf *field, Obj *term = NULL);
+
+ /** Return the number of documents where the specified term is present.
+ */
+ public abstract uint32_t
+ Doc_Freq(LexiconReader *self, const CharBuf *field, Obj *term);
+
+ /** If the term can be found, return a term info, otherwise return NULL.
+ */
+ abstract incremented nullable TermInfo*
+ Fetch_Term_Info(LexiconReader *self, const CharBuf *field, Obj *term);
+
+ /** Return a LexiconReader which merges the output of other
+ * LexiconReaders.
+ *
+ * @param readers An array of LexiconReaders.
+ * @param offsets Doc id start offsets for each reader.
+ */
+ public incremented nullable LexiconReader*
+ Aggregator(LexiconReader *self, VArray *readers, I32Array *offsets);
+}
+
+class Lucy::Index::PolyLexiconReader cnick PolyLexReader
+ inherits Lucy::Index::LexiconReader {
+
+ VArray *readers;
+ I32Array *offsets;
+
+ inert incremented PolyLexiconReader*
+ new(VArray *readers, I32Array *offsets);
+
+ inert PolyLexiconReader*
+ init(PolyLexiconReader *self, VArray *readers, I32Array *offsets);
+
+ public incremented nullable Lexicon*
+ Lexicon(PolyLexiconReader *self, const CharBuf *field, Obj *term = NULL);
+
+ public uint32_t
+ Doc_Freq(PolyLexiconReader *self, const CharBuf *field, Obj *term);
+
+ public void
+ Close(PolyLexiconReader *self);
+
+ public void
+ Destroy(PolyLexiconReader *self);
+}
+
+class Lucy::Index::DefaultLexiconReader cnick DefLexReader
+ inherits Lucy::Index::LexiconReader {
+
+ VArray *lexicons;
+
+ inert incremented DefaultLexiconReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultLexiconReader*
+ init(DefaultLexiconReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ public incremented nullable Lexicon*
+ Lexicon(DefaultLexiconReader *self, const CharBuf *field,
+ Obj *term = NULL);
+
+ /** Return the number of documents in which the term appears.
+ */
+ public uint32_t
+ Doc_Freq(DefaultLexiconReader *self, const CharBuf *field, Obj *term);
+
+ incremented nullable TermInfo*
+ Fetch_Term_Info(DefaultLexiconReader *self, const CharBuf *field,
+ Obj *term);
+
+ public void
+ Close(DefaultLexiconReader *self);
+
+ public void
+ Destroy(DefaultLexiconReader *self);
+}
+
+
diff --git a/core/Lucy/Index/LexiconWriter.c b/core/Lucy/Index/LexiconWriter.c
new file mode 100644
index 0000000..9a65fa3
--- /dev/null
+++ b/core/Lucy/Index/LexiconWriter.c
@@ -0,0 +1,257 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LEXICONWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/LexiconWriter.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Posting/MatchPosting.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Index/TermStepper.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+
+int32_t LexWriter_current_file_format = 3;
+
+LexiconWriter*
+LexWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ LexiconWriter *self = (LexiconWriter*)VTable_Make_Obj(LEXICONWRITER);
+ return LexWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+LexiconWriter*
+LexWriter_init(LexiconWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ Architecture *arch = Schema_Get_Architecture(schema);
+
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+
+ // Assign.
+ self->index_interval = Arch_Index_Interval(arch);
+ self->skip_interval = Arch_Skip_Interval(arch);
+
+ // Init.
+ self->ix_out = NULL;
+ self->ixix_out = NULL;
+ self->dat_out = NULL;
+ self->count = 0;
+ self->ix_count = 0;
+ self->dat_file = CB_new(30);
+ self->ix_file = CB_new(30);
+ self->ixix_file = CB_new(30);
+ self->counts = Hash_new(0);
+ self->ix_counts = Hash_new(0);
+ self->temp_mode = false;
+ self->term_stepper = NULL;
+ self->tinfo_stepper = (TermStepper*)MatchTInfoStepper_new(schema);
+
+ return self;
+}
+
+void
+LexWriter_destroy(LexiconWriter *self) {
+ DECREF(self->term_stepper);
+ DECREF(self->tinfo_stepper);
+ DECREF(self->dat_file);
+ DECREF(self->ix_file);
+ DECREF(self->ixix_file);
+ DECREF(self->dat_out);
+ DECREF(self->ix_out);
+ DECREF(self->ixix_out);
+ DECREF(self->counts);
+ DECREF(self->ix_counts);
+ SUPER_DESTROY(self, LEXICONWRITER);
+}
+
+static void
+S_add_last_term_to_ix(LexiconWriter *self) {
+ // Write file pointer to index record.
+ OutStream_Write_I64(self->ixix_out, OutStream_Tell(self->ix_out));
+
+ // Write term and file pointer to main record. Track count of terms added
+ // to ix.
+ TermStepper_Write_Key_Frame(self->term_stepper,
+ self->ix_out, TermStepper_Get_Value(self->term_stepper));
+ TermStepper_Write_Key_Frame(self->tinfo_stepper,
+ self->ix_out, TermStepper_Get_Value(self->tinfo_stepper));
+ OutStream_Write_C64(self->ix_out, OutStream_Tell(self->dat_out));
+ self->ix_count++;
+}
+
+void
+LexWriter_add_term(LexiconWriter* self, CharBuf* term_text, TermInfo* tinfo) {
+ OutStream *dat_out = self->dat_out;
+
+ if ((self->count % self->index_interval == 0)
+ && !self->temp_mode
+ ) {
+ // Write a subset of entries to lexicon.ix.
+ S_add_last_term_to_ix(self);
+ }
+
+ TermStepper_Write_Delta(self->term_stepper, dat_out, (Obj*)term_text);
+ TermStepper_Write_Delta(self->tinfo_stepper, dat_out, (Obj*)tinfo);
+
+ // Track number of terms.
+ self->count++;
+}
+
+void
+LexWriter_start_field(LexiconWriter *self, int32_t field_num) {
+ Segment *const segment = LexWriter_Get_Segment(self);
+ Folder *const folder = LexWriter_Get_Folder(self);
+ Schema *const schema = LexWriter_Get_Schema(self);
+ CharBuf *const seg_name = Seg_Get_Name(segment);
+ CharBuf *const field = Seg_Field_Name(segment, field_num);
+ FieldType *const type = Schema_Fetch_Type(schema, field);
+
+ // Open outstreams.
+ CB_setf(self->dat_file, "%o/lexicon-%i32.dat", seg_name, field_num);
+ CB_setf(self->ix_file, "%o/lexicon-%i32.ix", seg_name, field_num);
+ CB_setf(self->ixix_file, "%o/lexicon-%i32.ixix", seg_name, field_num);
+ self->dat_out = Folder_Open_Out(folder, self->dat_file);
+ if (!self->dat_out) { RETHROW(INCREF(Err_get_error())); }
+ self->ix_out = Folder_Open_Out(folder, self->ix_file);
+ if (!self->ix_out) { RETHROW(INCREF(Err_get_error())); }
+ self->ixix_out = Folder_Open_Out(folder, self->ixix_file);
+ if (!self->ixix_out) { RETHROW(INCREF(Err_get_error())); }
+
+ // Initialize count and ix_count, term stepper and term info stepper.
+ self->count = 0;
+ self->ix_count = 0;
+ self->term_stepper = FType_Make_Term_Stepper(type);
+ TermStepper_Reset(self->tinfo_stepper);
+}
+
+void
+LexWriter_finish_field(LexiconWriter *self, int32_t field_num) {
+ CharBuf *field = Seg_Field_Name(self->segment, field_num);
+
+ // Store count of terms for this field as metadata.
+ Hash_Store(self->counts, (Obj*)field,
+ (Obj*)CB_newf("%i32", self->count));
+ Hash_Store(self->ix_counts, (Obj*)field,
+ (Obj*)CB_newf("%i32", self->ix_count));
+
+ // Close streams.
+ OutStream_Close(self->dat_out);
+ OutStream_Close(self->ix_out);
+ OutStream_Close(self->ixix_out);
+ DECREF(self->dat_out);
+ DECREF(self->ix_out);
+ DECREF(self->ixix_out);
+ self->dat_out = NULL;
+ self->ix_out = NULL;
+ self->ixix_out = NULL;
+
+ // Close term stepper.
+ DECREF(self->term_stepper);
+ self->term_stepper = NULL;
+}
+
+void
+LexWriter_enter_temp_mode(LexiconWriter *self, const CharBuf *field,
+ OutStream *temp_outstream) {
+ Schema *schema = LexWriter_Get_Schema(self);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+
+ // Assign outstream.
+ if (self->dat_out != NULL) {
+ THROW(ERR, "Can't enter temp mode (filename: %o) ", self->dat_file);
+ }
+ self->dat_out = (OutStream*)INCREF(temp_outstream);
+
+ // Initialize count and ix_count, term stepper and term info stepper.
+ self->count = 0;
+ self->ix_count = 0;
+ self->term_stepper = FType_Make_Term_Stepper(type);
+ TermStepper_Reset(self->tinfo_stepper);
+
+ // Remember that we're in temp mode.
+ self->temp_mode = true;
+}
+
+void
+LexWriter_leave_temp_mode(LexiconWriter *self) {
+ DECREF(self->term_stepper);
+ self->term_stepper = NULL;
+ DECREF(self->dat_out);
+ self->dat_out = NULL;
+ self->temp_mode = false;
+}
+
+void
+LexWriter_finish(LexiconWriter *self) {
+ // Ensure that streams were closed (by calling Finish_Field or
+ // Leave_Temp_Mode).
+ if (self->dat_out != NULL) {
+ THROW(ERR, "File '%o' never closed", self->dat_file);
+ }
+ else if (self->ix_out != NULL) {
+ THROW(ERR, "File '%o' never closed", self->ix_file);
+ }
+ else if (self->ix_out != NULL) {
+ THROW(ERR, "File '%o' never closed", self->ix_file);
+ }
+
+ // Store metadata.
+ Seg_Store_Metadata_Str(self->segment, "lexicon", 7,
+ (Obj*)LexWriter_Metadata(self));
+}
+
+Hash*
+LexWriter_metadata(LexiconWriter *self) {
+ Hash *const metadata = DataWriter_metadata((DataWriter*)self);
+ Hash *const counts = (Hash*)INCREF(self->counts);
+ Hash *const ix_counts = (Hash*)INCREF(self->ix_counts);
+
+ // Placeholders.
+ if (Hash_Get_Size(counts) == 0) {
+ Hash_Store_Str(counts, "none", 4, (Obj*)CB_newf("%i32", (int32_t)0));
+ Hash_Store_Str(ix_counts, "none", 4,
+ (Obj*)CB_newf("%i32", (int32_t)0));
+ }
+
+ Hash_Store_Str(metadata, "counts", 6, (Obj*)counts);
+ Hash_Store_Str(metadata, "index_counts", 12, (Obj*)ix_counts);
+
+ return metadata;
+}
+
+void
+LexWriter_add_segment(LexiconWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ // No-op, since the data gets added via PostingListWriter.
+ UNUSED_VAR(self);
+ UNUSED_VAR(reader);
+ UNUSED_VAR(doc_map);
+}
+
+int32_t
+LexWriter_format(LexiconWriter *self) {
+ UNUSED_VAR(self);
+ return LexWriter_current_file_format;
+}
+
+
diff --git a/core/Lucy/Index/LexiconWriter.cfh b/core/Lucy/Index/LexiconWriter.cfh
new file mode 100644
index 0000000..977aa44
--- /dev/null
+++ b/core/Lucy/Index/LexiconWriter.cfh
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Writer for a term dictionary.
+ */
+class Lucy::Index::LexiconWriter cnick LexWriter
+ inherits Lucy::Index::DataWriter {
+
+ TermStepper *term_stepper;
+ TermStepper *tinfo_stepper;
+ CharBuf *dat_file;
+ CharBuf *ix_file;
+ CharBuf *ixix_file;
+ OutStream *dat_out;
+ OutStream *ix_out;
+ OutStream *ixix_out;
+ Hash *counts;
+ Hash *ix_counts;
+ bool_t temp_mode;
+ int32_t index_interval;
+ int32_t skip_interval;
+ int32_t count;
+ int32_t ix_count;
+
+ inert int32_t current_file_format;
+
+ inert incremented LexiconWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert LexiconWriter*
+ init(LexiconWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ /** Prepare to write the .lex and .lexx files for a field.
+ */
+ void
+ Start_Field(LexiconWriter *self, int32_t field_num);
+
+ /** Finish writing the current field. Close files, generate metadata.
+ */
+ void
+ Finish_Field(LexiconWriter *self, int32_t field_num);
+
+ /** Prepare to write terms to a temporary file.
+ */
+ void
+ Enter_Temp_Mode(LexiconWriter *self, const CharBuf *field,
+ OutStream *temp_outstream);
+
+ /** Stop writing terms to temp file. Abandon (but don't close) the file.
+ */
+ void
+ Leave_Temp_Mode(LexiconWriter *self);
+
+ /** Add a Term's text and its associated TermInfo (which has the Term's
+ * field number).
+ */
+ void
+ Add_Term(LexiconWriter* self, CharBuf* term_text, TermInfo* tinfo);
+
+ public void
+ Add_Segment(LexiconWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public incremented Hash*
+ Metadata(LexiconWriter *self);
+
+ public int32_t
+ Format(LexiconWriter *self);
+
+ public void
+ Finish(LexiconWriter *self);
+
+ public void
+ Destroy(LexiconWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/PolyLexicon.c b/core/Lucy/Index/PolyLexicon.c
new file mode 100644
index 0000000..cf655f9
--- /dev/null
+++ b/core/Lucy/Index/PolyLexicon.c
@@ -0,0 +1,215 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POLYLEXICON
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PolyLexicon.h"
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/SegLexicon.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Util/PriorityQueue.h"
+
+// Empty out, then refill the Queue, seeking all elements to [target].
+static void
+S_refresh_lex_q(SegLexQueue *lex_q, VArray *seg_lexicons, Obj *target);
+
+PolyLexicon*
+PolyLex_new(const CharBuf *field, VArray *sub_readers) {
+ PolyLexicon *self = (PolyLexicon*)VTable_Make_Obj(POLYLEXICON);
+ return PolyLex_init(self, field, sub_readers);
+}
+
+PolyLexicon*
+PolyLex_init(PolyLexicon *self, const CharBuf *field, VArray *sub_readers) {
+ uint32_t i;
+ uint32_t num_sub_readers = VA_Get_Size(sub_readers);
+ VArray *seg_lexicons = VA_new(num_sub_readers);
+
+ // Init.
+ Lex_init((Lexicon*)self, field);
+ self->term = NULL;
+ self->lex_q = SegLexQ_new(num_sub_readers);
+
+ // Derive.
+ for (i = 0; i < num_sub_readers; i++) {
+ LexiconReader *lex_reader = (LexiconReader*)VA_Fetch(sub_readers, i);
+ if (lex_reader && CERTIFY(lex_reader, LEXICONREADER)) {
+ Lexicon *seg_lexicon = LexReader_Lexicon(lex_reader, field, NULL);
+ if (seg_lexicon != NULL) {
+ VA_Push(seg_lexicons, (Obj*)seg_lexicon);
+ }
+ }
+ }
+ self->seg_lexicons = seg_lexicons;
+
+ PolyLex_Reset(self);
+
+ return self;
+}
+
+void
+PolyLex_destroy(PolyLexicon *self) {
+ DECREF(self->seg_lexicons);
+ DECREF(self->lex_q);
+ DECREF(self->term);
+ SUPER_DESTROY(self, POLYLEXICON);
+}
+
+static void
+S_refresh_lex_q(SegLexQueue *lex_q, VArray *seg_lexicons, Obj *target) {
+ uint32_t i, max;
+
+ // Empty out the queue.
+ while (1) {
+ SegLexicon *seg_lex = (SegLexicon*)SegLexQ_Pop(lex_q);
+ if (seg_lex == NULL) { break; }
+ DECREF(seg_lex);
+ }
+
+ // Refill the queue.
+ for (i = 0, max = VA_Get_Size(seg_lexicons); i < max; i++) {
+ SegLexicon *const seg_lexicon
+ = (SegLexicon*)VA_Fetch(seg_lexicons, i);
+ SegLex_Seek(seg_lexicon, target);
+ if (SegLex_Get_Term(seg_lexicon) != NULL) {
+ SegLexQ_Insert(lex_q, INCREF(seg_lexicon));
+ }
+ }
+}
+
+void
+PolyLex_reset(PolyLexicon *self) {
+ uint32_t i;
+ VArray *seg_lexicons = self->seg_lexicons;
+ uint32_t num_segs = VA_Get_Size(seg_lexicons);
+ SegLexQueue *lex_q = self->lex_q;
+
+ // Empty out the queue.
+ while (1) {
+ SegLexicon *seg_lex = (SegLexicon*)SegLexQ_Pop(lex_q);
+ if (seg_lex == NULL) { break; }
+ DECREF(seg_lex);
+ }
+
+ // Fill the queue with valid SegLexicons.
+ for (i = 0; i < num_segs; i++) {
+ SegLexicon *const seg_lexicon
+ = (SegLexicon*)VA_Fetch(seg_lexicons, i);
+ SegLex_Reset(seg_lexicon);
+ if (SegLex_Next(seg_lexicon)) {
+ SegLexQ_Insert(self->lex_q, INCREF(seg_lexicon));
+ }
+ }
+
+ if (self->term != NULL) {
+ DECREF(self->term);
+ self->term = NULL;
+ }
+}
+
+bool_t
+PolyLex_next(PolyLexicon *self) {
+ SegLexQueue *lex_q = self->lex_q;
+ SegLexicon *top_seg_lexicon = (SegLexicon*)SegLexQ_Peek(lex_q);
+
+ // Churn through queue items with equal terms.
+ while (top_seg_lexicon != NULL) {
+ Obj *const candidate = SegLex_Get_Term(top_seg_lexicon);
+ if ((candidate && !self->term)
+ || Obj_Compare_To(self->term, candidate) != 0
+ ) {
+ // Succeed if the next item in the queue has a different term.
+ DECREF(self->term);
+ self->term = Obj_Clone(candidate);
+ return true;
+ }
+ else {
+ SegLexicon *seg_lex = (SegLexicon*)SegLexQ_Pop(lex_q);
+ DECREF(seg_lex);
+ if (SegLex_Next(top_seg_lexicon)) {
+ SegLexQ_Insert(lex_q, INCREF(top_seg_lexicon));
+ }
+ top_seg_lexicon = (SegLexicon*)SegLexQ_Peek(lex_q);
+ }
+ }
+
+ // If queue is empty, iterator is finished.
+ DECREF(self->term);
+ self->term = NULL;
+ return false;
+}
+
+void
+PolyLex_seek(PolyLexicon *self, Obj *target) {
+ VArray *seg_lexicons = self->seg_lexicons;
+ SegLexQueue *lex_q = self->lex_q;
+
+ if (target == NULL) {
+ PolyLex_Reset(self);
+ return;
+ }
+
+ // Refresh the queue, set vars.
+ S_refresh_lex_q(lex_q, seg_lexicons, target);
+ {
+ SegLexicon *least = (SegLexicon*)SegLexQ_Peek(lex_q);
+ DECREF(self->term);
+ self->term = NULL;
+ if (least) {
+ Obj *least_term = SegLex_Get_Term(least);
+ self->term = least_term ? Obj_Clone(least_term) : NULL;
+ }
+ }
+
+ // Scan up to the real target.
+ do {
+ if (self->term) {
+ const int32_t comparison = Obj_Compare_To(self->term, target);
+ if (comparison >= 0) { break; }
+ }
+ } while (PolyLex_Next(self));
+}
+
+Obj*
+PolyLex_get_term(PolyLexicon *self) {
+ return self->term;
+}
+
+uint32_t
+PolyLex_get_num_seg_lexicons(PolyLexicon *self) {
+ return VA_Get_Size(self->seg_lexicons);
+}
+
+SegLexQueue*
+SegLexQ_new(uint32_t max_size) {
+ SegLexQueue *self = (SegLexQueue*)VTable_Make_Obj(SEGLEXQUEUE);
+ return (SegLexQueue*)PriQ_init((PriorityQueue*)self, max_size);
+}
+
+bool_t
+SegLexQ_less_than(SegLexQueue *self, Obj *a, Obj *b) {
+ SegLexicon *const lex_a = (SegLexicon*)a;
+ SegLexicon *const lex_b = (SegLexicon*)b;
+ Obj *const term_a = SegLex_Get_Term(lex_a);
+ Obj *const term_b = SegLex_Get_Term(lex_b);
+ UNUSED_VAR(self);
+ return CB_less_than(&term_a, &term_b);
+}
+
+
+
diff --git a/core/Lucy/Index/PolyLexicon.cfh b/core/Lucy/Index/PolyLexicon.cfh
new file mode 100644
index 0000000..be39c3f
--- /dev/null
+++ b/core/Lucy/Index/PolyLexicon.cfh
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Multi-segment Lexicon.
+ *
+ * Interleaves the output of multiple SegLexicons.
+ */
+
+class Lucy::Index::PolyLexicon cnick PolyLex
+ inherits Lucy::Index::Lexicon {
+
+ Obj *term;
+ SegLexQueue *lex_q;
+ VArray *seg_lexicons;
+ int32_t size;
+
+ inert incremented PolyLexicon*
+ new(const CharBuf *field, VArray *sub_readers);
+
+ inert PolyLexicon*
+ init(PolyLexicon *self, const CharBuf *field, VArray *sub_readers);
+
+ public void
+ Seek(PolyLexicon *self, Obj *target = NULL);
+
+ public bool_t
+ Next(PolyLexicon *self);
+
+ public void
+ Reset(PolyLexicon *self);
+
+ public nullable Obj*
+ Get_Term(PolyLexicon *self);
+
+ uint32_t
+ Get_Num_Seg_Lexicons(PolyLexicon *self);
+
+ public void
+ Destroy(PolyLexicon *self);
+}
+
+class Lucy::Index::SegLexQueue cnick SegLexQ
+ inherits Lucy::Util::PriorityQueue {
+
+ inert incremented SegLexQueue*
+ new(uint32_t max_size);
+
+ bool_t
+ Less_Than(SegLexQueue *self, Obj *a, Obj *b);
+}
+
+
diff --git a/core/Lucy/Index/PolyReader.c b/core/Lucy/Index/PolyReader.c
new file mode 100644
index 0000000..08606b0
--- /dev/null
+++ b/core/Lucy/Index/PolyReader.c
@@ -0,0 +1,506 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POLYREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/IndexManager.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/StringHelper.h"
+
+// Obtain/release read locks and commit locks. If self->manager is
+// NULL, do nothing.
+static void
+S_obtain_read_lock(PolyReader *self, const CharBuf *snapshot_filename);
+static void
+S_obtain_deletion_lock(PolyReader *self);
+static void
+S_release_read_lock(PolyReader *self);
+static void
+S_release_deletion_lock(PolyReader *self);
+
+static Folder*
+S_derive_folder(Obj *index);
+
+PolyReader*
+PolyReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ IndexManager *manager, VArray *sub_readers) {
+ PolyReader *self = (PolyReader*)VTable_Make_Obj(POLYREADER);
+ return PolyReader_init(self, schema, folder, snapshot, manager,
+ sub_readers);
+}
+
+PolyReader*
+PolyReader_open(Obj *index, Snapshot *snapshot, IndexManager *manager) {
+ PolyReader *self = (PolyReader*)VTable_Make_Obj(POLYREADER);
+ return PolyReader_do_open(self, index, snapshot, manager);
+}
+
+static Obj*
+S_first_non_null(VArray *array) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(array); i < max; i++) {
+ Obj *thing = VA_Fetch(array, i);
+ if (thing) { return thing; }
+ }
+ return NULL;
+}
+
+static void
+S_init_sub_readers(PolyReader *self, VArray *sub_readers) {
+ uint32_t i;
+ uint32_t num_sub_readers = VA_Get_Size(sub_readers);
+ int32_t *starts = (int32_t*)MALLOCATE(num_sub_readers * sizeof(int32_t));
+ Hash *data_readers = Hash_new(0);
+
+ DECREF(self->sub_readers);
+ DECREF(self->offsets);
+ self->sub_readers = (VArray*)INCREF(sub_readers);
+
+ // Accumulate doc_max, subreader start offsets, and DataReaders.
+ self->doc_max = 0;
+ for (i = 0; i < num_sub_readers; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(sub_readers, i);
+ Hash *components = SegReader_Get_Components(seg_reader);
+ CharBuf *api;
+ DataReader *component;
+ starts[i] = self->doc_max;
+ self->doc_max += SegReader_Doc_Max(seg_reader);
+ Hash_Iterate(components);
+ while (Hash_Next(components, (Obj**)&api, (Obj**)&component)) {
+ VArray *readers = (VArray*)Hash_Fetch(data_readers, (Obj*)api);
+ if (!readers) {
+ readers = VA_new(num_sub_readers);
+ Hash_Store(data_readers, (Obj*)api, (Obj*)readers);
+ }
+ VA_Store(readers, i, INCREF(component));
+ }
+ }
+ self->offsets = I32Arr_new_steal(starts, num_sub_readers);
+
+ {
+ CharBuf *api;
+ VArray *readers;
+ Hash_Iterate(data_readers);
+ while (Hash_Next(data_readers, (Obj**)&api, (Obj**)&readers)) {
+ DataReader *datareader = (DataReader*)CERTIFY(
+ S_first_non_null(readers),
+ DATAREADER);
+ DataReader *aggregator
+ = DataReader_Aggregator(datareader, readers, self->offsets);
+ if (aggregator) {
+ CERTIFY(aggregator, DATAREADER);
+ Hash_Store(self->components, (Obj*)api, (Obj*)aggregator);
+ }
+ }
+ }
+ DECREF(data_readers);
+
+ {
+ DeletionsReader *del_reader
+ = (DeletionsReader*)Hash_Fetch(
+ self->components, (Obj*)VTable_Get_Name(DELETIONSREADER));
+ self->del_count = del_reader ? DelReader_Del_Count(del_reader) : 0;
+ }
+}
+
+PolyReader*
+PolyReader_init(PolyReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, IndexManager *manager,
+ VArray *sub_readers) {
+ self->doc_max = 0;
+ self->del_count = 0;
+
+ if (sub_readers) {
+ uint32_t num_segs = VA_Get_Size(sub_readers);
+ VArray *segments = VA_new(num_segs);
+ uint32_t i;
+ for (i = 0; i < num_segs; i++) {
+ SegReader *seg_reader
+ = (SegReader*)CERTIFY(VA_Fetch(sub_readers, i), SEGREADER);
+ VA_Push(segments, INCREF(SegReader_Get_Segment(seg_reader)));
+ }
+ IxReader_init((IndexReader*)self, schema, folder, snapshot,
+ segments, -1, manager);
+ DECREF(segments);
+ S_init_sub_readers(self, sub_readers);
+ }
+ else {
+ IxReader_init((IndexReader*)self, schema, folder, snapshot,
+ NULL, -1, manager);
+ self->sub_readers = VA_new(0);
+ self->offsets = I32Arr_new_steal(NULL, 0);
+ }
+
+ return self;
+}
+
+void
+PolyReader_close(PolyReader *self) {
+ PolyReader_close_t super_close
+ = (PolyReader_close_t)SUPER_METHOD(POLYREADER, PolyReader, Close);
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->sub_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(self->sub_readers, i);
+ SegReader_Close(seg_reader);
+ }
+ super_close(self);
+}
+
+void
+PolyReader_destroy(PolyReader *self) {
+ DECREF(self->sub_readers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, POLYREADER);
+}
+
+Obj*
+S_try_open_elements(PolyReader *self) {
+ VArray *files = Snapshot_List(self->snapshot);
+ Folder *folder = PolyReader_Get_Folder(self);
+ uint32_t num_segs = 0;
+ uint64_t latest_schema_gen = 0;
+ CharBuf *schema_file = NULL;
+ VArray *segments;
+ uint32_t i, max;
+
+ // Find schema file, count segments.
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
+
+ if (Seg_valid_seg_name(entry)) {
+ num_segs++;
+ }
+ else if (CB_Starts_With_Str(entry, "schema_", 7)
+ && CB_Ends_With_Str(entry, ".json", 5)
+ ) {
+ uint64_t gen = IxFileNames_extract_gen(entry);
+ if (gen > latest_schema_gen) {
+ latest_schema_gen = gen;
+ if (!schema_file) { schema_file = CB_Clone(entry); }
+ else { CB_Mimic(schema_file, (Obj*)entry); }
+ }
+ }
+ }
+
+ // Read Schema.
+ if (!schema_file) {
+ CharBuf *mess = MAKE_MESS("Can't find a schema file.");
+ DECREF(files);
+ return (Obj*)mess;
+ }
+ else {
+ Hash *dump = (Hash*)Json_slurp_json(folder, schema_file);
+ if (dump) { // read file successfully
+ DECREF(self->schema);
+ self->schema = (Schema*)CERTIFY(
+ VTable_Load_Obj(SCHEMA, (Obj*)dump), SCHEMA);
+ DECREF(dump);
+ DECREF(schema_file);
+ schema_file = NULL;
+ }
+ else {
+ CharBuf *mess = MAKE_MESS("Failed to parse %o", schema_file);
+ DECREF(schema_file);
+ DECREF(files);
+ return (Obj*)mess;
+ }
+ }
+
+ segments = VA_new(num_segs);
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(files, i);
+
+ // Create a Segment for each segmeta.
+ if (Seg_valid_seg_name(entry)) {
+ int64_t seg_num = IxFileNames_extract_gen(entry);
+ Segment *segment = Seg_new(seg_num);
+
+ // Bail if reading the file fails (probably because it's been
+ // deleted and a new snapshot file has been written so we need to
+ // retry).
+ if (Seg_Read_File(segment, folder)) {
+ VA_Push(segments, (Obj*)segment);
+ }
+ else {
+ CharBuf *mess = MAKE_MESS("Failed to read %o", entry);
+ DECREF(segment);
+ DECREF(segments);
+ DECREF(files);
+ return (Obj*)mess;
+ }
+ }
+ }
+
+ // Sort the segments by age.
+ VA_Sort(segments, NULL, NULL);
+
+ {
+ Obj *result = PolyReader_Try_Open_SegReaders(self, segments);
+ DECREF(segments);
+ DECREF(files);
+ return result;
+ }
+}
+
+// For test suite.
+CharBuf* PolyReader_race_condition_debug1 = NULL;
+int32_t PolyReader_debug1_num_passes = 0;
+
+PolyReader*
+PolyReader_do_open(PolyReader *self, Obj *index, Snapshot *snapshot,
+ IndexManager *manager) {
+ Folder *folder = S_derive_folder(index);
+ uint64_t last_gen = 0;
+
+ PolyReader_init(self, NULL, folder, snapshot, manager, NULL);
+ DECREF(folder);
+
+ if (manager) { S_obtain_deletion_lock(self); }
+
+ while (1) {
+ CharBuf *target_snap_file;
+ uint64_t gen;
+
+ // If a Snapshot was supplied, use its file.
+ if (snapshot) {
+ target_snap_file = Snapshot_Get_Path(snapshot);
+ if (!target_snap_file) {
+ THROW(ERR, "Supplied snapshot objects must not be empty");
+ }
+ else {
+ CB_Inc_RefCount(target_snap_file);
+ }
+ }
+ else {
+ // Otherwise, pick the most recent snap file.
+ target_snap_file = IxFileNames_latest_snapshot(folder);
+
+ // No snap file? Looks like the index is empty. We can stop now
+ // and return NULL.
+ if (!target_snap_file) { break; }
+ }
+
+ // Derive "generation" of this snapshot file from its name.
+ gen = IxFileNames_extract_gen(target_snap_file);
+
+ // Get a read lock on the most recent snapshot file if indicated.
+ if (manager) {
+ S_obtain_read_lock(self, target_snap_file);
+ }
+
+ // Testing only.
+ if (PolyReader_race_condition_debug1) {
+ ZombieCharBuf *temp = ZCB_WRAP_STR("temp", 4);
+ if (Folder_Exists(folder, (CharBuf*)temp)) {
+ bool_t success = Folder_Rename(folder, (CharBuf*)temp,
+ PolyReader_race_condition_debug1);
+ if (!success) { RETHROW(INCREF(Err_get_error())); }
+ }
+ PolyReader_debug1_num_passes++;
+ }
+
+ // If a Snapshot object was passed in, the file has already been read.
+ // If that's not the case, we must read the file we just picked.
+ if (!snapshot) {
+ CharBuf *error = PolyReader_try_read_snapshot(self->snapshot, folder,
+ target_snap_file);
+
+ if (error) {
+ S_release_read_lock(self);
+ DECREF(target_snap_file);
+ if (last_gen < gen) { // Index updated, so try again.
+ DECREF(error);
+ last_gen = gen;
+ continue;
+ }
+ else { // Real error.
+ if (manager) { S_release_deletion_lock(self); }
+ Err_throw_mess(ERR, error);
+ }
+ }
+ }
+
+ /* It's possible, though unlikely, for an Indexer to delete files
+ * out from underneath us after the snapshot file is read but before
+ * we've got SegReaders holding open all the required files. If we
+ * failed to open something, see if we can find a newer snapshot file.
+ * If we can, then the exception was due to the race condition. If
+ * not, we have a real exception, so throw an error. */
+ {
+ Obj *result = S_try_open_elements(self);
+ if (Obj_Is_A(result, CHARBUF)) { // Error occurred.
+ S_release_read_lock(self);
+ DECREF(target_snap_file);
+ if (last_gen < gen) { // Index updated, so try again.
+ DECREF(result);
+ last_gen = gen;
+ }
+ else { // Real error.
+ if (manager) { S_release_deletion_lock(self); }
+ Err_throw_mess(ERR, (CharBuf*)result);
+ }
+ }
+ else { // Succeeded.
+ S_init_sub_readers(self, (VArray*)result);
+ DECREF(result);
+ DECREF(target_snap_file);
+ break;
+ }
+ }
+ }
+
+ if (manager) { S_release_deletion_lock(self); }
+
+ return self;
+}
+
+static Folder*
+S_derive_folder(Obj *index) {
+ Folder *folder = NULL;
+ if (Obj_Is_A(index, FOLDER)) {
+ folder = (Folder*)INCREF(index);
+ }
+ else if (Obj_Is_A(index, CHARBUF)) {
+ folder = (Folder*)FSFolder_new((CharBuf*)index);
+ }
+ else {
+ THROW(ERR, "Invalid type for 'index': %o", Obj_Get_Class_Name(index));
+ }
+ return folder;
+}
+
+static void
+S_obtain_deletion_lock(PolyReader *self) {
+ self->deletion_lock = IxManager_Make_Deletion_Lock(self->manager);
+ Lock_Clear_Stale(self->deletion_lock);
+ if (!Lock_Obtain(self->deletion_lock)) {
+ DECREF(self->deletion_lock);
+ self->deletion_lock = NULL;
+ THROW(LOCKERR, "Couldn't get commit lock");
+ }
+}
+
+static void
+S_obtain_read_lock(PolyReader *self, const CharBuf *snapshot_file_name) {
+ if (!self->manager) { return; }
+ self->read_lock = IxManager_Make_Snapshot_Read_Lock(self->manager,
+ snapshot_file_name);
+
+ Lock_Clear_Stale(self->read_lock);
+ if (!Lock_Obtain(self->read_lock)) {
+ DECREF(self->read_lock);
+ THROW(LOCKERR, "Couldn't get read lock for %o", snapshot_file_name);
+ }
+}
+
+static void
+S_release_read_lock(PolyReader *self) {
+ if (self->read_lock) {
+ Lock_Release(self->read_lock);
+ DECREF(self->read_lock);
+ self->read_lock = NULL;
+ }
+}
+
+static void
+S_release_deletion_lock(PolyReader *self) {
+ if (self->deletion_lock) {
+ Lock_Release(self->deletion_lock);
+ DECREF(self->deletion_lock);
+ self->deletion_lock = NULL;
+ }
+}
+
+int32_t
+PolyReader_doc_max(PolyReader *self) {
+ return self->doc_max;
+}
+
+int32_t
+PolyReader_doc_count(PolyReader *self) {
+ return self->doc_max - self->del_count;
+}
+
+int32_t
+PolyReader_del_count(PolyReader *self) {
+ return self->del_count;
+}
+
+I32Array*
+PolyReader_offsets(PolyReader *self) {
+ return (I32Array*)INCREF(self->offsets);
+}
+
+VArray*
+PolyReader_seg_readers(PolyReader *self) {
+ return (VArray*)VA_Shallow_Copy(self->sub_readers);
+}
+
+VArray*
+PolyReader_get_seg_readers(PolyReader *self) {
+ return self->sub_readers;
+}
+
+uint32_t
+PolyReader_sub_tick(I32Array *offsets, int32_t doc_id) {
+ int32_t size = I32Arr_Get_Size(offsets);
+ if (size == 0) {
+ return 0;
+ }
+
+ int32_t lo = -1;
+ int32_t hi = size;
+ while (hi - lo > 1) {
+ int32_t mid = lo + ((hi - lo) / 2);
+ int32_t offset = I32Arr_Get(offsets, mid);
+ if (doc_id <= offset) {
+ hi = mid;
+ }
+ else {
+ lo = mid;
+ }
+ }
+ if (hi == size) {
+ hi--;
+ }
+
+ while (hi > 0) {
+ int32_t offset = I32Arr_Get(offsets, hi);
+ if (doc_id <= offset) {
+ hi--;
+ }
+ else {
+ break;
+ }
+ }
+
+ return hi;
+}
+
+
diff --git a/core/Lucy/Index/PolyReader.cfh b/core/Lucy/Index/PolyReader.cfh
new file mode 100644
index 0000000..703ac57
--- /dev/null
+++ b/core/Lucy/Index/PolyReader.cfh
@@ -0,0 +1,106 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Multi-segment implementation of IndexReader.
+ *
+ * PolyReader conflates index data from multiple segments. For instance, if
+ * an index contains three segments with 10 documents each, PolyReader's
+ * Doc_Max() method will return 30.
+ *
+ * Some of PolyReader's L<DataReader|Lucy::Index::DataReader> components
+ * may be less efficient or complete than the single-segment implementations
+ * accessed via L<SegReader|Lucy::Index::SegReader>.
+ */
+class Lucy::Index::PolyReader inherits Lucy::Index::IndexReader {
+
+ VArray *sub_readers;
+ int32_t doc_max;
+ int32_t del_count;
+ I32Array *offsets;
+
+ public inert incremented nullable PolyReader*
+ open(Obj *index, Snapshot *snapshot = NULL, IndexManager *manager = NULL);
+
+ /**
+ * @param index Either a string filepath or a L<Lucy::Folder>.
+ * @param snapshot A Snapshot. If not supplied, the most recent snapshot
+ * file will be used.
+ * @param manager An L<IndexManager|Lucy::Index::IndexManager>.
+ * Read-locking is off by default; supplying this argument turns it on.
+ */
+ public inert nullable PolyReader*
+ do_open(PolyReader *self, Obj *index, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL);
+
+ public inert incremented PolyReader*
+ new(Schema *schema = NULL, Folder *folder, Snapshot *snapshot = NULL,
+ IndexManager *manager = NULL, VArray *sub_readers = NULL);
+
+ public inert PolyReader*
+ init(PolyReader *self, Schema *schema = NULL, Folder *folder,
+ Snapshot *snapshot = NULL, IndexManager *manager = NULL,
+ VArray *sub_readers = NULL);
+
+ /** Attempt to open a SegReader for each Segment that the Snapshot knows
+ * about. If an exception occurs, catch it and return its error message.
+ * If the opening succeeds, return a VArray full of SegReaders.
+ */
+ incremented Obj*
+ Try_Open_SegReaders(PolyReader *self, VArray *segments);
+
+ /** Attempt to read a snapshot file. If the operation succeeds, return
+ * NULL. If an exception occurs, catch it and return its error message.
+ */
+ inert incremented nullable CharBuf*
+ try_read_snapshot(Snapshot *snapshot, Folder *folder,
+ const CharBuf *path);
+
+ inert CharBuf* race_condition_debug1;
+ inert int32_t debug1_num_passes;
+
+ /** Determine which sub-reader a document id belongs to.
+ */
+ inert uint32_t
+ sub_tick(I32Array *offsets, int32_t doc_id);
+
+ public int32_t
+ Doc_Max(PolyReader *self);
+
+ public int32_t
+ Doc_Count(PolyReader *self);
+
+ public int32_t
+ Del_Count(PolyReader *self);
+
+ public incremented I32Array*
+ Offsets(PolyReader *self);
+
+ public incremented VArray*
+ Seg_Readers(PolyReader *self);
+
+ VArray*
+ Get_Seg_Readers(PolyReader *self);
+
+ public void
+ Close(PolyReader *self);
+
+ public void
+ Destroy(PolyReader *self);
+}
+
+
diff --git a/core/Lucy/Index/Posting.c b/core/Lucy/Index/Posting.c
new file mode 100644
index 0000000..36cb1e9
--- /dev/null
+++ b/core/Lucy/Index/Posting.c
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POSTING
+#define C_LUCY_POSTINGWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/DataWriter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+
+Posting*
+Post_init(Posting *self) {
+ self->doc_id = 0;
+ return self;
+}
+
+void
+Post_set_doc_id(Posting *self, int32_t doc_id) {
+ self->doc_id = doc_id;
+}
+
+int32_t
+Post_get_doc_id(Posting *self) {
+ return self->doc_id;
+}
+
+PostingWriter*
+PostWriter_init(PostingWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, int32_t field_num) {
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment,
+ polyreader);
+ self->field_num = field_num;
+ return self;
+}
+
+
diff --git a/core/Lucy/Index/Posting.cfh b/core/Lucy/Index/Posting.cfh
new file mode 100644
index 0000000..4ef5724
--- /dev/null
+++ b/core/Lucy/Index/Posting.cfh
@@ -0,0 +1,96 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Vessel holding statistical data for a posting.
+ *
+ * A Posting, in Apache Lucy, is a vessel which stores information about a
+ * term-document match. (See L<Lucy::Docs::IRTheory> for the
+ * academic definition of "posting".)
+ *
+ * Subclasses include
+ * L<MatchPosting|Lucy::Index::Posting::MatchPosting>, the simplest
+ * posting format, and
+ * L<ScorePosting|Lucy::Index::Posting::ScorePosting>, the default.
+ */
+class Lucy::Index::Posting cnick Post inherits Lucy::Util::Stepper {
+
+ int32_t doc_id;
+
+ public inert Posting*
+ init(Posting *self);
+
+ /** Create a RawPosting object, suitable for index-time sorting.
+ *
+ * Updates the state of the document id, but nothing else.
+ */
+ abstract incremented RawPosting*
+ Read_Raw(Posting *self, InStream *instream, int32_t last_doc_id,
+ CharBuf *term_text, MemoryPool *mem_pool);
+
+ /** Process an Inversion into RawPosting objects and add them all to the
+ * supplied PostingPool.
+ */
+ abstract void
+ Add_Inversion_To_Pool(Posting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm);
+
+ public void
+ Set_Doc_ID(Posting *self, int32_t doc_id);
+
+ public int32_t
+ Get_Doc_ID(Posting *self);
+
+ /** Factory method for creating a Matcher.
+ */
+ abstract incremented Matcher*
+ Make_Matcher(Posting *self, Similarity *sim, PostingList *plist,
+ Compiler *compiler, bool_t need_score);
+}
+
+abstract class Lucy::Index::Posting::PostingWriter cnick PostWriter
+ inherits Lucy::Index::DataWriter {
+
+ int32_t field_num;
+
+ inert PostingWriter*
+ init(PostingWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, int32_t field_num);
+
+ /** Take a RawPosting that was flattened earlier and write it to the
+ * index. */
+ abstract void
+ Write_Posting(PostingWriter *self, RawPosting *posting);
+
+ /** Start a new term. Update the TermInfo to reflect the state of the
+ * PostingWriter.
+ */
+ abstract void
+ Start_Term(PostingWriter *self, TermInfo *tinfo);
+
+ /** Update the TermInfo to reflect the internal state of the
+ * PostingWriter so that skip information can be written.
+ *
+ * TODO: This is an ugly hack which needs refactoring.
+ */
+ abstract void
+ Update_Skip_Info(PostingWriter *self, TermInfo *tinfo);
+}
+
+
diff --git a/core/Lucy/Index/Posting/MatchPosting.c b/core/Lucy/Index/Posting/MatchPosting.c
new file mode 100644
index 0000000..5d91283
--- /dev/null
+++ b/core/Lucy/Index/Posting/MatchPosting.c
@@ -0,0 +1,328 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MATCHPOSTING
+#define C_LUCY_MATCHPOSTINGMATCHER
+#define C_LUCY_MATCHPOSTINGWRITER
+#define C_LUCY_MATCHTERMINFOSTEPPER
+#define C_LUCY_RAWPOSTING
+#define C_LUCY_TERMINFO
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Posting/MatchPosting.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingPool.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/MemoryPool.h"
+
+#define MAX_RAW_POSTING_LEN(_text_len) \
+ ( sizeof(RawPosting) \
+ + _text_len + 1 /* term text content */ \
+ )
+
+MatchPosting*
+MatchPost_new(Similarity *sim) {
+ MatchPosting *self = (MatchPosting*)VTable_Make_Obj(MATCHPOSTING);
+ return MatchPost_init(self, sim);
+}
+
+MatchPosting*
+MatchPost_init(MatchPosting *self, Similarity *sim) {
+ self->sim = (Similarity*)INCREF(sim);
+ return (MatchPosting*)Post_init((Posting*)self);
+}
+
+void
+MatchPost_destroy(MatchPosting *self) {
+ DECREF(self->sim);
+ SUPER_DESTROY(self, MATCHPOSTING);
+}
+
+int32_t
+MatchPost_get_freq(MatchPosting *self) {
+ return self->freq;
+}
+
+void
+MatchPost_reset(MatchPosting *self) {
+ self->doc_id = 0;
+}
+
+void
+MatchPost_read_record(MatchPosting *self, InStream *instream) {
+ const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t doc_delta = doc_code >> 1;
+
+ // Apply delta doc and retrieve freq.
+ self->doc_id += doc_delta;
+ if (doc_code & 1) {
+ self->freq = 1;
+ }
+ else {
+ self->freq = InStream_Read_C32(instream);
+ }
+}
+
+RawPosting*
+MatchPost_read_raw(MatchPosting *self, InStream *instream, int32_t last_doc_id,
+ CharBuf *term_text, MemoryPool *mem_pool) {
+ char *const text_buf = (char*)CB_Get_Ptr8(term_text);
+ const size_t text_size = CB_Get_Size(term_text);
+ const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t delta_doc = doc_code >> 1;
+ const int32_t doc_id = last_doc_id + delta_doc;
+ const uint32_t freq = (doc_code & 1)
+ ? 1
+ : InStream_Read_C32(instream);
+ size_t raw_post_bytes = MAX_RAW_POSTING_LEN(text_size);
+ void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
+ UNUSED_VAR(self);
+
+ return RawPost_new(allocation, doc_id, freq, text_buf, text_size);
+}
+
+void
+MatchPost_add_inversion_to_pool(MatchPosting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm) {
+ MemoryPool *mem_pool = PostPool_Get_Mem_Pool(post_pool);
+ Token **tokens;
+ uint32_t freq;
+
+ UNUSED_VAR(self);
+ UNUSED_VAR(type);
+ UNUSED_VAR(doc_boost);
+ UNUSED_VAR(length_norm);
+
+ Inversion_Reset(inversion);
+ while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
+ Token *token = *tokens;
+ uint32_t raw_post_bytes = MAX_RAW_POSTING_LEN(token->len);
+ RawPosting *raw_posting
+ = RawPost_new(MemPool_Grab(mem_pool, raw_post_bytes), doc_id,
+ freq, token->text, token->len);
+ PostPool_Feed(post_pool, &raw_posting);
+ }
+}
+
+MatchPostingMatcher*
+MatchPost_make_matcher(MatchPosting *self, Similarity *sim,
+ PostingList *plist, Compiler *compiler,
+ bool_t need_score) {
+ MatchPostingMatcher *matcher
+ = (MatchPostingMatcher*)VTable_Make_Obj(MATCHPOSTINGMATCHER);
+ UNUSED_VAR(self);
+ UNUSED_VAR(need_score);
+ return MatchPostMatcher_init(matcher, sim, plist, compiler);
+}
+
+/***************************************************************************/
+
+MatchPostingMatcher*
+MatchPostMatcher_init(MatchPostingMatcher *self, Similarity *sim,
+ PostingList *plist, Compiler *compiler) {
+ TermMatcher_init((TermMatcher*)self, sim, plist, compiler);
+ return self;
+}
+
+float
+MatchPostMatcher_score(MatchPostingMatcher* self) {
+ return self->weight;
+}
+
+/***************************************************************************/
+
+MatchPostingWriter*
+MatchPostWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, int32_t field_num) {
+ MatchPostingWriter *self
+ = (MatchPostingWriter*)VTable_Make_Obj(MATCHPOSTINGWRITER);
+ return MatchPostWriter_init(self, schema, snapshot, segment, polyreader,
+ field_num);
+}
+
+MatchPostingWriter*
+MatchPostWriter_init(MatchPostingWriter *self, Schema *schema,
+ Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, int32_t field_num) {
+ Folder *folder = PolyReader_Get_Folder(polyreader);
+ CharBuf *filename
+ = CB_newf("%o/postings-%i32.dat", Seg_Get_Name(segment), field_num);
+ PostWriter_init((PostingWriter*)self, schema, snapshot, segment,
+ polyreader, field_num);
+ self->outstream = Folder_Open_Out(folder, filename);
+ if (!self->outstream) { RETHROW(INCREF(Err_get_error())); }
+ DECREF(filename);
+ return self;
+}
+
+void
+MatchPostWriter_destroy(MatchPostingWriter *self) {
+ DECREF(self->outstream);
+ SUPER_DESTROY(self, MATCHPOSTINGWRITER);
+}
+
+void
+MatchPostWriter_write_posting(MatchPostingWriter *self, RawPosting *posting) {
+ OutStream *const outstream = self->outstream;
+ const int32_t doc_id = posting->doc_id;
+ const uint32_t delta_doc = doc_id - self->last_doc_id;
+ char *const aux_content = posting->blob + posting->content_len;
+ if (posting->freq == 1) {
+ const uint32_t doc_code = (delta_doc << 1) | 1;
+ OutStream_Write_C32(outstream, doc_code);
+ }
+ else {
+ const uint32_t doc_code = delta_doc << 1;
+ OutStream_Write_C32(outstream, doc_code);
+ OutStream_Write_C32(outstream, posting->freq);
+ }
+ OutStream_Write_Bytes(outstream, aux_content, posting->aux_len);
+ self->last_doc_id = doc_id;
+}
+
+void
+MatchPostWriter_start_term(MatchPostingWriter *self, TermInfo *tinfo) {
+ self->last_doc_id = 0;
+ tinfo->post_filepos = OutStream_Tell(self->outstream);
+}
+
+void
+MatchPostWriter_update_skip_info(MatchPostingWriter *self, TermInfo *tinfo) {
+ tinfo->post_filepos = OutStream_Tell(self->outstream);
+}
+
+/***************************************************************************/
+
+MatchTermInfoStepper*
+MatchTInfoStepper_new(Schema *schema) {
+ MatchTermInfoStepper *self
+ = (MatchTermInfoStepper*)VTable_Make_Obj(MATCHTERMINFOSTEPPER);
+ return MatchTInfoStepper_init(self, schema);
+}
+
+MatchTermInfoStepper*
+MatchTInfoStepper_init(MatchTermInfoStepper *self, Schema *schema) {
+ Architecture *arch = Schema_Get_Architecture(schema);
+ TermStepper_init((TermStepper*)self);
+ self->skip_interval = Arch_Skip_Interval(arch);
+ self->value = (Obj*)TInfo_new(0);
+ return self;
+}
+
+void
+MatchTInfoStepper_reset(MatchTermInfoStepper *self) {
+ TInfo_Reset((TermInfo*)self->value);
+}
+
+void
+MatchTInfoStepper_write_key_frame(MatchTermInfoStepper *self,
+ OutStream *outstream, Obj *value) {
+ TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO);
+ int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo);
+
+ // Write doc_freq.
+ OutStream_Write_C32(outstream, doc_freq);
+
+ // Write postings file pointer.
+ OutStream_Write_C64(outstream, tinfo->post_filepos);
+
+ // Write skip file pointer (maybe).
+ if (doc_freq >= self->skip_interval) {
+ OutStream_Write_C64(outstream, tinfo->skip_filepos);
+ }
+
+ TInfo_Mimic((TermInfo*)self->value, (Obj*)tinfo);
+}
+
+void
+MatchTInfoStepper_write_delta(MatchTermInfoStepper *self,
+ OutStream *outstream, Obj *value) {
+ TermInfo *tinfo = (TermInfo*)CERTIFY(value, TERMINFO);
+ TermInfo *last_tinfo = (TermInfo*)self->value;
+ int32_t doc_freq = TInfo_Get_Doc_Freq(tinfo);
+ int64_t post_delta = tinfo->post_filepos - last_tinfo->post_filepos;
+
+ // Write doc_freq.
+ OutStream_Write_C32(outstream, doc_freq);
+
+ // Write postings file pointer delta.
+ OutStream_Write_C64(outstream, post_delta);
+
+ // Write skip file pointer (maybe).
+ if (doc_freq >= self->skip_interval) {
+ OutStream_Write_C64(outstream, tinfo->skip_filepos);
+ }
+
+ TInfo_Mimic((TermInfo*)self->value, (Obj*)tinfo);
+}
+
+void
+MatchTInfoStepper_read_key_frame(MatchTermInfoStepper *self,
+ InStream *instream) {
+ TermInfo *const tinfo = (TermInfo*)self->value;
+
+ // Read doc freq.
+ tinfo->doc_freq = InStream_Read_C32(instream);
+
+ // Read postings file pointer.
+ tinfo->post_filepos = InStream_Read_C64(instream);
+
+ // Maybe read skip pointer.
+ if (tinfo->doc_freq >= self->skip_interval) {
+ tinfo->skip_filepos = InStream_Read_C64(instream);
+ }
+ else {
+ tinfo->skip_filepos = 0;
+ }
+}
+
+void
+MatchTInfoStepper_read_delta(MatchTermInfoStepper *self, InStream *instream) {
+ TermInfo *const tinfo = (TermInfo*)self->value;
+
+ // Read doc freq.
+ tinfo->doc_freq = InStream_Read_C32(instream);
+
+ // Adjust postings file pointer.
+ tinfo->post_filepos += InStream_Read_C64(instream);
+
+ // Maybe read skip pointer.
+ if (tinfo->doc_freq >= self->skip_interval) {
+ tinfo->skip_filepos = InStream_Read_C64(instream);
+ }
+ else {
+ tinfo->skip_filepos = 0;
+ }
+}
+
+
diff --git a/core/Lucy/Index/Posting/MatchPosting.cfh b/core/Lucy/Index/Posting/MatchPosting.cfh
new file mode 100644
index 0000000..645e291
--- /dev/null
+++ b/core/Lucy/Index/Posting/MatchPosting.cfh
@@ -0,0 +1,131 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Match but not score documents.
+ *
+ * Use MatchPosting for fields which only need to be matched, not scored. For
+ * instance, if you need to determine that that a query matches a particular
+ * category, but don't want the match to contribute to the document score, use
+ * MatchPosting for the field.
+ */
+class Lucy::Index::Posting::MatchPosting cnick MatchPost
+ inherits Lucy::Index::Posting {
+
+ Similarity *sim;
+ uint32_t freq;
+
+ inert incremented MatchPosting*
+ new(Similarity *similarity);
+
+ inert MatchPosting*
+ init(MatchPosting *self, Similarity *similarity);
+
+ public void
+ Destroy(MatchPosting *self);
+
+ int32_t
+ Get_Freq(MatchPosting *self);
+
+ void
+ Read_Record(MatchPosting *self, InStream *instream);
+
+ incremented RawPosting*
+ Read_Raw(MatchPosting *self, InStream *instream, int32_t last_doc_id,
+ CharBuf *term_text, MemoryPool *mem_pool);
+
+ void
+ Add_Inversion_To_Pool(MatchPosting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm);
+
+ public void
+ Reset(MatchPosting *self);
+
+ incremented MatchPostingMatcher*
+ Make_Matcher(MatchPosting *self, Similarity *sim, PostingList *plist,
+ Compiler *compiler, bool_t need_score);
+}
+
+class Lucy::Index::Posting::MatchPostingMatcher cnick MatchPostMatcher
+ inherits Lucy::Search::TermMatcher {
+
+ inert MatchPostingMatcher*
+ init(MatchPostingMatcher *self, Similarity *similarity,
+ PostingList *posting_list, Compiler *compiler);
+
+ public float
+ Score(MatchPostingMatcher *self);
+}
+
+class Lucy::Index::Posting::MatchPostingWriter cnick MatchPostWriter
+ inherits Lucy::Index::Posting::PostingWriter {
+
+ OutStream *outstream;
+ int32_t last_doc_id;
+
+ inert incremented MatchPostingWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, int32_t field_num);
+
+ inert MatchPostingWriter*
+ init(MatchPostingWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, int32_t field_num);
+
+ public void
+ Destroy(MatchPostingWriter *self);
+
+ void
+ Write_Posting(MatchPostingWriter *self, RawPosting *posting);
+
+ void
+ Start_Term(MatchPostingWriter *self, TermInfo *tinfo);
+
+ void
+ Update_Skip_Info(MatchPostingWriter *self, TermInfo *tinfo);
+}
+
+class Lucy::Index::Posting::MatchPosting::MatchTermInfoStepper
+ cnick MatchTInfoStepper inherits Lucy::Index::TermStepper {
+
+ int32_t skip_interval;
+
+ inert incremented MatchTermInfoStepper*
+ new(Schema *schema);
+
+ inert MatchTermInfoStepper*
+ init(MatchTermInfoStepper *self, Schema *schema);
+
+ public void
+ Reset(MatchTermInfoStepper *self);
+
+ public void
+ Write_Key_Frame(MatchTermInfoStepper *self, OutStream *outstream,
+ Obj *value);
+
+ public void
+ Write_Delta(MatchTermInfoStepper *self, OutStream *outstream, Obj *value);
+
+ public void
+ Read_Key_Frame(MatchTermInfoStepper *self, InStream *instream);
+
+ public void
+ Read_Delta(MatchTermInfoStepper *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Index/Posting/RawPosting.c b/core/Lucy/Index/Posting/RawPosting.c
new file mode 100644
index 0000000..10c37e3
--- /dev/null
+++ b/core/Lucy/Index/Posting/RawPosting.c
@@ -0,0 +1,141 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAWPOSTING
+#define C_LUCY_RAWPOSTINGWRITER
+#define C_LUCY_TERMINFO
+#include "Lucy/Util/ToolSet.h"
+
+#include <string.h>
+
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/StringHelper.h"
+
+RawPosting RAWPOSTING_BLANK = {
+ RAWPOSTING,
+ {1}, // ref.count
+ 0, // doc_id
+ 1, // freq
+ 0, // content_len
+ 0, // aux_len
+ { '\0' } // blob
+};
+
+
+RawPosting*
+RawPost_new(void *pre_allocated_memory, int32_t doc_id, uint32_t freq,
+ char *term_text, size_t term_text_len) {
+ RawPosting *self = (RawPosting*)pre_allocated_memory;
+ self->vtable = RAWPOSTING;
+ self->ref.count = 1; // never used
+ self->doc_id = doc_id;
+ self->freq = freq;
+ self->content_len = term_text_len;
+ self->aux_len = 0;
+ memcpy(&self->blob, term_text, term_text_len);
+
+ return self;
+}
+
+void
+RawPost_destroy(RawPosting *self) {
+ UNUSED_VAR(self);
+ THROW(ERR, "Illegal attempt to destroy RawPosting object");
+}
+
+uint32_t
+RawPost_get_refcount(RawPosting* self) {
+ UNUSED_VAR(self);
+ return 1;
+}
+
+RawPosting*
+RawPost_inc_refcount(RawPosting* self) {
+ return self;
+}
+
+uint32_t
+RawPost_dec_refcount(RawPosting* self) {
+ UNUSED_VAR(self);
+ return 1;
+}
+
+/***************************************************************************/
+
+RawPostingWriter*
+RawPostWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, OutStream *outstream) {
+ RawPostingWriter *self
+ = (RawPostingWriter*)VTable_Make_Obj(RAWPOSTINGWRITER);
+ return RawPostWriter_init(self, schema, snapshot, segment, polyreader,
+ outstream);
+}
+
+RawPostingWriter*
+RawPostWriter_init(RawPostingWriter *self, Schema *schema,
+ Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, OutStream *outstream) {
+ const int32_t invalid_field_num = 0;
+ PostWriter_init((PostingWriter*)self, schema, snapshot, segment,
+ polyreader, invalid_field_num);
+ self->outstream = (OutStream*)INCREF(outstream);
+ self->last_doc_id = 0;
+ return self;
+}
+
+void
+RawPostWriter_start_term(RawPostingWriter *self, TermInfo *tinfo) {
+ self->last_doc_id = 0;
+ tinfo->post_filepos = OutStream_Tell(self->outstream);
+}
+
+void
+RawPostWriter_update_skip_info(RawPostingWriter *self, TermInfo *tinfo) {
+ tinfo->post_filepos = OutStream_Tell(self->outstream);
+}
+
+void
+RawPostWriter_destroy(RawPostingWriter *self) {
+ DECREF(self->outstream);
+ SUPER_DESTROY(self, RAWPOSTINGWRITER);
+}
+
+void
+RawPostWriter_write_posting(RawPostingWriter *self, RawPosting *posting) {
+ OutStream *const outstream = self->outstream;
+ const int32_t doc_id = posting->doc_id;
+ const uint32_t delta_doc = doc_id - self->last_doc_id;
+ char *const aux_content = posting->blob + posting->content_len;
+ if (posting->freq == 1) {
+ const uint32_t doc_code = (delta_doc << 1) | 1;
+ OutStream_Write_C32(outstream, doc_code);
+ }
+ else {
+ const uint32_t doc_code = delta_doc << 1;
+ OutStream_Write_C32(outstream, doc_code);
+ OutStream_Write_C32(outstream, posting->freq);
+ }
+ OutStream_Write_Bytes(outstream, aux_content, posting->aux_len);
+ self->last_doc_id = doc_id;
+}
+
+
diff --git a/core/Lucy/Index/Posting/RawPosting.cfh b/core/Lucy/Index/Posting/RawPosting.cfh
new file mode 100644
index 0000000..6621d4f
--- /dev/null
+++ b/core/Lucy/Index/Posting/RawPosting.cfh
@@ -0,0 +1,103 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Sortable, serialized Posting.
+ *
+ * RawPosting is a specialized subclass of Posting for private use only. It
+ * is used at index-time for fast reading, writing, sorting and merging of
+ * index posting data by PostingPool.
+ *
+ * RawPosting's Destroy method throws an error. All RawPosting objects belong
+ * to a particular MemoryPool, which takes responsibility for freeing them.
+ *
+ * The last struct member, [blob], is a "flexible array" member. RawPosting
+ * objects are assigned one continuous memory block of variable size,
+ * depending on how much data needs to fit in blob.
+ *
+ * The first part of blob is the term's text content, the length of which is
+ * indicated by [content_len]. At the end of the content, encoded auxilliary
+ * posting information begins, ready to be blasted out verbatim to a postings
+ * file once the after the doc id is written.
+ */
+
+class Lucy::Index::RawPosting cnick RawPost
+ inherits Lucy::Index::Posting {
+
+ uint32_t freq;
+ uint32_t content_len;
+ uint32_t aux_len;
+ char[1] blob; /* flexible array */
+
+ /** Constructor. Uses pre-allocated memory.
+ */
+ inert incremented RawPosting*
+ new(void *pre_allocated_memory, int32_t doc_id, uint32_t freq,
+ char *term_text, size_t term_text_len);
+
+ uint32_t
+ Get_RefCount(RawPosting* self);
+
+ incremented RawPosting*
+ Inc_RefCount(RawPosting* self);
+
+ uint32_t
+ Dec_RefCount(RawPosting* self);
+
+ /** Throws an error.
+ */
+ public void
+ Destroy(RawPosting *self);
+}
+
+class Lucy::Index::Posting::RawPostingWriter cnick RawPostWriter
+ inherits Lucy::Index::Posting::PostingWriter {
+
+ OutStream *outstream;
+ int32_t last_doc_id;
+
+ inert incremented RawPostingWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, OutStream *outstream);
+
+ inert RawPostingWriter*
+ init(RawPostingWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, OutStream *outstream);
+
+ public void
+ Destroy(RawPostingWriter *self);
+
+ void
+ Start_Term(RawPostingWriter *self, TermInfo *tinfo);
+
+ void
+ Update_Skip_Info(RawPostingWriter *self, TermInfo *tinfo);
+
+ void
+ Write_Posting(RawPostingWriter *self, RawPosting *posting);
+}
+
+__C__
+extern lucy_RawPosting LUCY_RAWPOSTING_BLANK;
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define RAWPOSTING_BLANK LUCY_RAWPOSTING_BLANK
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Index/Posting/RichPosting.c b/core/Lucy/Index/Posting/RichPosting.c
new file mode 100644
index 0000000..85c4c0c
--- /dev/null
+++ b/core/Lucy/Index/Posting/RichPosting.c
@@ -0,0 +1,204 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RICHPOSTING
+#define C_LUCY_RICHPOSTINGMATCHER
+#define C_LUCY_RAWPOSTING
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Posting/RichPosting.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingPool.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Util/MemoryPool.h"
+
+#define FREQ_MAX_LEN C32_MAX_BYTES
+#define MAX_RAW_POSTING_LEN(_text_len, _freq) \
+ ( sizeof(RawPosting) \
+ + _text_len /* term text content */ \
+ + FREQ_MAX_LEN /* freq c32 */ \
+ + (C32_MAX_BYTES * _freq) /* positions deltas */ \
+ + _freq /* per-pos boost byte */ \
+ )
+
+RichPosting*
+RichPost_new(Similarity *sim) {
+ RichPosting *self = (RichPosting*)VTable_Make_Obj(RICHPOSTING);
+ return RichPost_init(self, sim);
+}
+
+RichPosting*
+RichPost_init(RichPosting *self, Similarity *sim) {
+ ScorePost_init((ScorePosting*)self, sim);
+ self->prox_boosts = NULL;
+ return self;
+}
+
+void
+RichPost_destroy(RichPosting *self) {
+ FREEMEM(self->prox_boosts);
+ SUPER_DESTROY(self, RICHPOSTING);
+}
+
+void
+RichPost_read_record(RichPosting *self, InStream *instream) {
+ float *const norm_decoder = self->norm_decoder;
+ uint32_t doc_code;
+ uint32_t num_prox = 0;
+ uint32_t position = 0;
+ uint32_t *positions;
+ float *prox_boosts;
+ float aggregate_weight = 0.0;
+
+ // Decode delta doc.
+ doc_code = InStream_Read_C32(instream);
+ self->doc_id += doc_code >> 1;
+
+ // If the stored num was odd, the freq is 1.
+ if (doc_code & 1) {
+ self->freq = 1;
+ }
+ // Otherwise, freq was stored as a C32.
+ else {
+ self->freq = InStream_Read_C32(instream);
+ }
+
+ // Read positions, aggregate per-position boost byte into weight.
+ num_prox = self->freq;
+ if (num_prox > self->prox_cap) {
+ self->prox
+ = (uint32_t*)REALLOCATE(self->prox, num_prox * sizeof(uint32_t));
+ self->prox_boosts
+ = (float*)REALLOCATE(self->prox_boosts, num_prox * sizeof(float));
+ }
+ positions = self->prox;
+ prox_boosts = self->prox_boosts;
+
+ while (num_prox--) {
+ position += InStream_Read_C32(instream);
+ *positions++ = position;
+ *prox_boosts = norm_decoder[InStream_Read_U8(instream)];
+ aggregate_weight += *prox_boosts;
+ prox_boosts++;
+ }
+ self->weight = aggregate_weight / self->freq;
+}
+
+void
+RichPost_add_inversion_to_pool(RichPosting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm) {
+ MemoryPool *mem_pool = PostPool_Get_Mem_Pool(post_pool);
+ Similarity *sim = self->sim;
+ float field_boost = doc_boost * FType_Get_Boost(type) * length_norm;
+ Token **tokens;
+ uint32_t freq;
+
+ Inversion_Reset(inversion);
+ while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
+ Token *token = *tokens;
+ uint32_t raw_post_bytes = MAX_RAW_POSTING_LEN(token->len, freq);
+ RawPosting *raw_posting
+ = RawPost_new(MemPool_Grab(mem_pool, raw_post_bytes), doc_id,
+ freq, token->text, token->len);
+ char *const start = raw_posting->blob + token->len;
+ char *dest = start;
+ uint32_t last_prox = 0;
+ uint32_t i;
+
+ // Positions and boosts.
+ for (i = 0; i < freq; i++) {
+ Token *const t = tokens[i];
+ const uint32_t prox_delta = t->pos - last_prox;
+ const float boost = field_boost * t->boost;
+
+ NumUtil_encode_c32(prox_delta, &dest);
+ last_prox = t->pos;
+
+ *((uint8_t*)dest) = Sim_Encode_Norm(sim, boost);
+ dest++;
+ }
+
+ // Resize raw posting memory allocation.
+ raw_posting->aux_len = dest - start;
+ raw_post_bytes = dest - (char*)raw_posting;
+ MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
+ PostPool_Feed(post_pool, &raw_posting);
+ }
+}
+
+RawPosting*
+RichPost_read_raw(RichPosting *self, InStream *instream, int32_t last_doc_id,
+ CharBuf *term_text, MemoryPool *mem_pool) {
+ char *const text_buf = (char*)CB_Get_Ptr8(term_text);
+ const size_t text_size = CB_Get_Size(term_text);
+ const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t delta_doc = doc_code >> 1;
+ const int32_t doc_id = last_doc_id + delta_doc;
+ const uint32_t freq = (doc_code & 1)
+ ? 1
+ : InStream_Read_C32(instream);
+ size_t raw_post_bytes = MAX_RAW_POSTING_LEN(text_size, freq);
+ void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
+ RawPosting *const raw_posting
+ = RawPost_new(allocation, doc_id, freq, text_buf, text_size);
+ uint32_t num_prox = freq;
+ char *const start = raw_posting->blob + text_size;
+ char * dest = start;
+ UNUSED_VAR(self);
+
+ // Read positions and per-position boosts.
+ while (num_prox--) {
+ dest += InStream_Read_Raw_C64(instream, dest);
+ *((uint8_t*)dest) = InStream_Read_U8(instream);
+ dest++;
+ }
+
+ // Resize raw posting memory allocation.
+ raw_posting->aux_len = dest - start;
+ raw_post_bytes = dest - (char*)raw_posting;
+ MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
+
+ return raw_posting;
+}
+
+RichPostingMatcher*
+RichPost_make_matcher(RichPosting *self, Similarity *sim,
+ PostingList *plist, Compiler *compiler,
+ bool_t need_score) {
+ RichPostingMatcher* matcher
+ = (RichPostingMatcher*)VTable_Make_Obj(RICHPOSTINGMATCHER);
+ UNUSED_VAR(self);
+ UNUSED_VAR(need_score);
+ return RichPostMatcher_init(matcher, sim, plist, compiler);
+}
+
+RichPostingMatcher*
+RichPostMatcher_init(RichPostingMatcher *self, Similarity *sim,
+ PostingList *plist, Compiler *compiler) {
+ return (RichPostingMatcher*)ScorePostMatcher_init((ScorePostingMatcher*)self,
+ sim, plist, compiler);
+}
+
+
diff --git a/core/Lucy/Index/Posting/RichPosting.cfh b/core/Lucy/Index/Posting/RichPosting.cfh
new file mode 100644
index 0000000..53c7d1d
--- /dev/null
+++ b/core/Lucy/Index/Posting/RichPosting.cfh
@@ -0,0 +1,71 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Posting with per-position boost.
+ *
+ * RichPosting is similar to
+ * L<ScorePosting|Lucy::Index::Posting::ScorePosting>, but weighting is
+ * per-position rather than per-field. To exploit this, you need a custom
+ * L<Analyzer|Lucy::Analysis::Analyzer> which assigns varying boosts to
+ * individual L<Token|Lucy::Analysis::Token> objects.
+ *
+ * A typical application for RichPosting is an HTMLAnalyzer which assigns
+ * boost based on the visual size and weight of the marked up text: H1
+ * blocks get the greatest weight, H2 blocks almost as much, etc.
+ */
+class Lucy::Index::Posting::RichPosting cnick RichPost
+ inherits Lucy::Index::Posting::ScorePosting {
+
+ float *prox_boosts;
+
+ inert incremented RichPosting*
+ new(Similarity *similarity);
+
+ inert RichPosting*
+ init(RichPosting *self, Similarity *similarity);
+
+ public void
+ Destroy(RichPosting *self);
+
+ void
+ Read_Record(RichPosting *self, InStream *instream);
+
+ incremented RawPosting*
+ Read_Raw(RichPosting *self, InStream *instream, int32_t last_doc_id,
+ CharBuf *term_text, MemoryPool *mem_pool);
+
+ void
+ Add_Inversion_To_Pool(RichPosting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm);
+
+ incremented RichPostingMatcher*
+ Make_Matcher(RichPosting *self, Similarity *sim, PostingList *plist,
+ Compiler *compiler, bool_t need_score);
+}
+
+class Lucy::Index::Posting::RichPostingMatcher cnick RichPostMatcher
+ inherits Lucy::Index::Posting::ScorePostingMatcher {
+
+ inert RichPostingMatcher*
+ init(RichPostingMatcher *self, Similarity *similarity,
+ PostingList *posting_list, Compiler *compiler);
+}
+
+
diff --git a/core/Lucy/Index/Posting/ScorePosting.c b/core/Lucy/Index/Posting/ScorePosting.c
new file mode 100644
index 0000000..fbb0625
--- /dev/null
+++ b/core/Lucy/Index/Posting/ScorePosting.c
@@ -0,0 +1,255 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SCOREPOSTING
+#define C_LUCY_SCOREPOSTINGMATCHER
+#define C_LUCY_RAWPOSTING
+#define C_LUCY_TOKEN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingPool.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Util/MemoryPool.h"
+
+#define FIELD_BOOST_LEN 1
+#define FREQ_MAX_LEN C32_MAX_BYTES
+#define MAX_RAW_POSTING_LEN(_text_len, _freq) \
+ ( sizeof(RawPosting) \
+ + _text_len /* term text content */ \
+ + FIELD_BOOST_LEN /* field boost byte */ \
+ + FREQ_MAX_LEN /* freq c32 */ \
+ + (C32_MAX_BYTES * _freq) /* positions deltas */ \
+ )
+
+ScorePosting*
+ScorePost_new(Similarity *sim) {
+ ScorePosting *self = (ScorePosting*)VTable_Make_Obj(SCOREPOSTING);
+ return ScorePost_init(self, sim);
+}
+
+ScorePosting*
+ScorePost_init(ScorePosting *self, Similarity *sim) {
+ MatchPost_init((MatchPosting*)self, sim);
+ self->norm_decoder = Sim_Get_Norm_Decoder(sim);
+ self->freq = 0;
+ self->weight = 0.0;
+ self->prox = NULL;
+ self->prox_cap = 0;
+ return self;
+}
+
+void
+ScorePost_destroy(ScorePosting *self) {
+ FREEMEM(self->prox);
+ SUPER_DESTROY(self, SCOREPOSTING);
+}
+
+uint32_t*
+ScorePost_get_prox(ScorePosting *self) {
+ return self->prox;
+}
+
+void
+ScorePost_add_inversion_to_pool(ScorePosting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm) {
+ MemoryPool *mem_pool = PostPool_Get_Mem_Pool(post_pool);
+ Similarity *sim = self->sim;
+ float field_boost = doc_boost * FType_Get_Boost(type) * length_norm;
+ const uint8_t field_boost_byte = Sim_Encode_Norm(sim, field_boost);
+ Token **tokens;
+ uint32_t freq;
+
+ Inversion_Reset(inversion);
+ while ((tokens = Inversion_Next_Cluster(inversion, &freq)) != NULL) {
+ Token *token = *tokens;
+ uint32_t raw_post_bytes = MAX_RAW_POSTING_LEN(token->len, freq);
+ RawPosting *raw_posting
+ = RawPost_new(MemPool_Grab(mem_pool, raw_post_bytes), doc_id,
+ freq, token->text, token->len);
+ char *const start = raw_posting->blob + token->len;
+ char *dest = start;
+ uint32_t last_prox = 0;
+ uint32_t i;
+
+ // Field_boost.
+ *((uint8_t*)dest) = field_boost_byte;
+ dest++;
+
+ // Positions.
+ for (i = 0; i < freq; i++) {
+ Token *const t = tokens[i];
+ const uint32_t prox_delta = t->pos - last_prox;
+ NumUtil_encode_c32(prox_delta, &dest);
+ last_prox = t->pos;
+ }
+
+ // Resize raw posting memory allocation.
+ raw_posting->aux_len = dest - start;
+ raw_post_bytes = dest - (char*)raw_posting;
+ MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
+ PostPool_Feed(post_pool, &raw_posting);
+ }
+}
+
+void
+ScorePost_reset(ScorePosting *self) {
+ self->doc_id = 0;
+ self->freq = 0;
+ self->weight = 0.0;
+}
+
+void
+ScorePost_read_record(ScorePosting *self, InStream *instream) {
+ uint32_t num_prox;
+ uint32_t position = 0;
+ uint32_t *positions;
+ const size_t max_start_bytes = (C32_MAX_BYTES * 2) + 1;
+ char *buf = InStream_Buf(instream, max_start_bytes);
+ const uint32_t doc_code = NumUtil_decode_c32(&buf);
+ const uint32_t doc_delta = doc_code >> 1;
+
+ // Apply delta doc and retrieve freq.
+ self->doc_id += doc_delta;
+ if (doc_code & 1) {
+ self->freq = 1;
+ }
+ else {
+ self->freq = NumUtil_decode_c32(&buf);
+ }
+
+ // Decode boost/norm byte.
+ self->weight = self->norm_decoder[*(uint8_t*)buf];
+ buf++;
+
+ // Read positions.
+ num_prox = self->freq;
+ if (num_prox > self->prox_cap) {
+ self->prox = (uint32_t*)REALLOCATE(
+ self->prox, num_prox * sizeof(uint32_t));
+ self->prox_cap = num_prox;
+ }
+ positions = self->prox;
+
+ InStream_Advance_Buf(instream, buf);
+ buf = InStream_Buf(instream, num_prox * C32_MAX_BYTES);
+ while (num_prox--) {
+ position += NumUtil_decode_c32(&buf);
+ *positions++ = position;
+ }
+
+ InStream_Advance_Buf(instream, buf);
+}
+
+RawPosting*
+ScorePost_read_raw(ScorePosting *self, InStream *instream,
+ int32_t last_doc_id, CharBuf *term_text,
+ MemoryPool *mem_pool) {
+ char *const text_buf = (char*)CB_Get_Ptr8(term_text);
+ const size_t text_size = CB_Get_Size(term_text);
+ const uint32_t doc_code = InStream_Read_C32(instream);
+ const uint32_t delta_doc = doc_code >> 1;
+ const int32_t doc_id = last_doc_id + delta_doc;
+ const uint32_t freq = (doc_code & 1)
+ ? 1
+ : InStream_Read_C32(instream);
+ size_t raw_post_bytes = MAX_RAW_POSTING_LEN(text_size, freq);
+ void *const allocation = MemPool_Grab(mem_pool, raw_post_bytes);
+ RawPosting *const raw_posting
+ = RawPost_new(allocation, doc_id, freq, text_buf, text_size);
+ uint32_t num_prox = freq;
+ char *const start = raw_posting->blob + text_size;
+ char *dest = start;
+ UNUSED_VAR(self);
+
+ // Field_boost.
+ *((uint8_t*)dest) = InStream_Read_U8(instream);
+ dest++;
+
+ // Read positions.
+ while (num_prox--) {
+ dest += InStream_Read_Raw_C64(instream, dest);
+ }
+
+ // Resize raw posting memory allocation.
+ raw_posting->aux_len = dest - start;
+ raw_post_bytes = dest - (char*)raw_posting;
+ MemPool_Resize(mem_pool, raw_posting, raw_post_bytes);
+
+ return raw_posting;
+}
+
+ScorePostingMatcher*
+ScorePost_make_matcher(ScorePosting *self, Similarity *sim,
+ PostingList *plist, Compiler *compiler,
+ bool_t need_score) {
+ ScorePostingMatcher *matcher
+ = (ScorePostingMatcher*)VTable_Make_Obj(SCOREPOSTINGMATCHER);
+ UNUSED_VAR(self);
+ UNUSED_VAR(need_score);
+ return ScorePostMatcher_init(matcher, sim, plist, compiler);
+}
+
+ScorePostingMatcher*
+ScorePostMatcher_init(ScorePostingMatcher *self, Similarity *sim,
+ PostingList *plist, Compiler *compiler) {
+ uint32_t i;
+
+ // Init.
+ TermMatcher_init((TermMatcher*)self, sim, plist, compiler);
+
+ // Fill score cache.
+ self->score_cache = (float*)MALLOCATE(TERMMATCHER_SCORE_CACHE_SIZE * sizeof(float));
+ for (i = 0; i < TERMMATCHER_SCORE_CACHE_SIZE; i++) {
+ self->score_cache[i] = Sim_TF(sim, (float)i) * self->weight;
+ }
+
+ return self;
+}
+
+float
+ScorePostMatcher_score(ScorePostingMatcher* self) {
+ ScorePosting *const posting = (ScorePosting*)self->posting;
+ const uint32_t freq = posting->freq;
+
+ // Calculate initial score based on frequency of term.
+ float score = (freq < TERMMATCHER_SCORE_CACHE_SIZE)
+ ? self->score_cache[freq] // cache hit
+ : Sim_TF(self->sim, (float)freq) * self->weight;
+
+ // Factor in field-length normalization and doc/field/prox boost.
+ score *= posting->weight;
+
+ return score;
+}
+
+void
+ScorePostMatcher_destroy(ScorePostingMatcher *self) {
+ FREEMEM(self->score_cache);
+ SUPER_DESTROY(self, SCOREPOSTINGMATCHER);
+}
+
+
diff --git a/core/Lucy/Index/Posting/ScorePosting.cfh b/core/Lucy/Index/Posting/ScorePosting.cfh
new file mode 100644
index 0000000..3076be7
--- /dev/null
+++ b/core/Lucy/Index/Posting/ScorePosting.cfh
@@ -0,0 +1,82 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Default posting type.
+ *
+ * ScorePosting is the default posting format in Apache Lucy. The
+ * term-document pairing used by MatchPosting is supplemented by additional
+ * frequency, position, and weighting information.
+ */
+class Lucy::Index::Posting::ScorePosting cnick ScorePost
+ inherits Lucy::Index::Posting::MatchPosting {
+
+ float weight;
+ float *norm_decoder;
+ uint32_t *prox;
+ uint32_t prox_cap;
+
+ inert incremented ScorePosting*
+ new(Similarity *similarity);
+
+ inert ScorePosting*
+ init(ScorePosting *self, Similarity *similarity);
+
+ public void
+ Destroy(ScorePosting *self);
+
+ void
+ Read_Record(ScorePosting *self, InStream *instream);
+
+ incremented RawPosting*
+ Read_Raw(ScorePosting *self, InStream *instream, int32_t last_doc_id,
+ CharBuf *term_text, MemoryPool *mem_pool);
+
+ void
+ Add_Inversion_To_Pool(ScorePosting *self, PostingPool *post_pool,
+ Inversion *inversion, FieldType *type,
+ int32_t doc_id, float doc_boost,
+ float length_norm);
+
+ public void
+ Reset(ScorePosting *self);
+
+ incremented ScorePostingMatcher*
+ Make_Matcher(ScorePosting *self, Similarity *sim, PostingList *plist,
+ Compiler *compiler, bool_t need_score);
+
+ nullable uint32_t*
+ Get_Prox(ScorePosting *self);
+}
+
+class Lucy::Index::Posting::ScorePostingMatcher cnick ScorePostMatcher
+ inherits Lucy::Search::TermMatcher {
+
+ float *score_cache;
+
+ inert ScorePostingMatcher*
+ init(ScorePostingMatcher *self, Similarity *sim, PostingList *plist,
+ Compiler *compiler);
+
+ public float
+ Score(ScorePostingMatcher* self);
+
+ public void
+ Destroy(ScorePostingMatcher *self);
+}
+
+
diff --git a/core/Lucy/Index/PostingList.c b/core/Lucy/Index/PostingList.c
new file mode 100644
index 0000000..4c173fa
--- /dev/null
+++ b/core/Lucy/Index/PostingList.c
@@ -0,0 +1,33 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POSTINGLIST
+#include <string.h>
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/Lexicon.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Util/Memory.h"
+
+PostingList*
+PList_init(PostingList *self) {
+ ABSTRACT_CLASS_CHECK(self, POSTINGLIST);
+ return self;
+}
+
+
diff --git a/core/Lucy/Index/PostingList.cfh b/core/Lucy/Index/PostingList.cfh
new file mode 100644
index 0000000..c545a18
--- /dev/null
+++ b/core/Lucy/Index/PostingList.cfh
@@ -0,0 +1,70 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Term-Document pairings.
+ *
+ * PostingList is an iterator which supplies a list of document ids that match
+ * a given term.
+ *
+ * See L<Lucy::Docs::IRTheory> for definitions of "posting" and "posting
+ * list".
+ */
+class Lucy::Index::PostingList cnick PList
+ inherits Lucy::Search::Matcher {
+
+ public inert PostingList*
+ init(PostingList *self);
+
+ /** Return the iterator's current Posting. Should not be called before
+ * the iterator is initialized or after it empties.
+ */
+ abstract Posting*
+ Get_Posting(PostingList *self);
+
+ /** Return the number of documents that the PostingList contains. (This
+ * number will include any documents which have been marked as deleted but
+ * not yet purged.)
+ */
+ public abstract uint32_t
+ Get_Doc_Freq(PostingList *self);
+
+ /** Prepare the PostingList object to iterate over matches for documents
+ * that match <code>target</code>.
+ *
+ * @param target The term to match. If NULL, the iterator will be empty.
+ */
+ public abstract void
+ Seek(PostingList *self, Obj *target = NULL);
+
+ abstract void
+ Seek_Lex(PostingList *self, Lexicon *lexicon);
+
+ /** Invoke Post_Make_Matcher() for this PostingList's posting.
+ */
+ abstract Matcher*
+ Make_Matcher(PostingList *self, Similarity *similarity,
+ Compiler *compiler, bool_t need_score);
+
+ /** Indexing helper function.
+ */
+ abstract RawPosting*
+ Read_Raw(PostingList *self, int32_t last_doc_id, CharBuf *term_text,
+ MemoryPool *mem_pool);
+}
+
+
diff --git a/core/Lucy/Index/PostingListReader.c b/core/Lucy/Index/PostingListReader.c
new file mode 100644
index 0000000..72fb8b1
--- /dev/null
+++ b/core/Lucy/Index/PostingListReader.c
@@ -0,0 +1,130 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POSTINGLISTREADER
+#define C_LUCY_POLYPOSTINGLISTREADER
+#define C_LUCY_DEFAULTPOSTINGLISTREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Index/PostingListWriter.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegPostingList.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+
+PostingListReader*
+PListReader_init(PostingListReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ ABSTRACT_CLASS_CHECK(self, POSTINGLISTREADER);
+ return self;
+}
+
+PostingListReader*
+PListReader_aggregator(PostingListReader *self, VArray *readers,
+ I32Array *offsets) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(readers);
+ UNUSED_VAR(offsets);
+ return NULL;
+}
+
+DefaultPostingListReader*
+DefPListReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick,
+ LexiconReader *lex_reader) {
+ DefaultPostingListReader *self
+ = (DefaultPostingListReader*)VTable_Make_Obj(DEFAULTPOSTINGLISTREADER);
+ return DefPListReader_init(self, schema, folder, snapshot, segments,
+ seg_tick, lex_reader);
+}
+
+DefaultPostingListReader*
+DefPListReader_init(DefaultPostingListReader *self, Schema *schema,
+ Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick, LexiconReader *lex_reader) {
+ PListReader_init((PostingListReader*)self, schema, folder, snapshot,
+ segments, seg_tick);
+ Segment *segment = DefPListReader_Get_Segment(self);
+
+ // Derive.
+ self->lex_reader = (LexiconReader*)INCREF(lex_reader);
+
+ // Check format.
+ {
+ Hash *my_meta = (Hash*)Seg_Fetch_Metadata_Str(segment, "postings", 8);
+ if (!my_meta) {
+ my_meta = (Hash*)Seg_Fetch_Metadata_Str(segment,
+ "posting_list", 12);
+ }
+
+ if (my_meta) {
+ Obj *format = Hash_Fetch_Str(my_meta, "format", 6);
+ if (!format) { THROW(ERR, "Missing 'format' var"); }
+ else {
+ if (Obj_To_I64(format) != PListWriter_current_file_format) {
+ THROW(ERR, "Unsupported postings format: %i64",
+ Obj_To_I64(format));
+ }
+ }
+ }
+ }
+
+ return self;
+}
+
+void
+DefPListReader_close(DefaultPostingListReader *self) {
+ if (self->lex_reader) {
+ LexReader_Close(self->lex_reader);
+ DECREF(self->lex_reader);
+ self->lex_reader = NULL;
+ }
+}
+
+void
+DefPListReader_destroy(DefaultPostingListReader *self) {
+ DECREF(self->lex_reader);
+ SUPER_DESTROY(self, DEFAULTPOSTINGLISTREADER);
+}
+
+SegPostingList*
+DefPListReader_posting_list(DefaultPostingListReader *self,
+ const CharBuf *field, Obj *target) {
+ FieldType *type = Schema_Fetch_Type(self->schema, field);
+
+ // Only return an object if we've got an indexed field.
+ if (type != NULL && FType_Indexed(type)) {
+ SegPostingList *plist = SegPList_new((PostingListReader*)self, field);
+ if (target) { SegPList_Seek(plist, target); }
+ return plist;
+ }
+ else {
+ return NULL;
+ }
+}
+
+LexiconReader*
+DefPListReader_get_lex_reader(DefaultPostingListReader *self) {
+ return self->lex_reader;
+}
+
diff --git a/core/Lucy/Index/PostingListReader.cfh b/core/Lucy/Index/PostingListReader.cfh
new file mode 100644
index 0000000..ec1a923
--- /dev/null
+++ b/core/Lucy/Index/PostingListReader.cfh
@@ -0,0 +1,81 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read postings data.
+ *
+ * PostingListReaders produce L<PostingList|Lucy::Index::PostingList>
+ * objects which convey document matching information.
+ */
+class Lucy::Index::PostingListReader cnick PListReader
+ inherits Lucy::Index::DataReader {
+
+ inert PostingListReader*
+ init(PostingListReader *self, Schema *schema = NULL,
+ Folder *folder = NULL, Snapshot *snapshot = NULL,
+ VArray *segments = NULL, int32_t seg_tick = -1);
+
+ /** Returns a PostingList, or NULL if either <code>field</code> is NULL or
+ * <code>field</code> is not present in any documents.
+ *
+ * @param field A field name.
+ * @param term If supplied, the PostingList will be pre-located to this
+ * term using Seek().
+ */
+ public abstract incremented nullable PostingList*
+ Posting_List(PostingListReader *self, const CharBuf *field = NULL,
+ Obj *term = NULL);
+
+ abstract LexiconReader*
+ Get_Lex_Reader(PostingListReader *self);
+
+ /** Returns NULL since PostingLists may only be iterated at the segment
+ * level.
+ */
+ public incremented nullable PostingListReader*
+ Aggregator(PostingListReader *self, VArray *readers, I32Array *offsets);
+}
+
+class Lucy::Index::DefaultPostingListReader cnick DefPListReader
+ inherits Lucy::Index::PostingListReader {
+
+ LexiconReader *lex_reader;
+
+ inert incremented DefaultPostingListReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick, LexiconReader *lex_reader);
+
+ inert DefaultPostingListReader*
+ init(DefaultPostingListReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick,
+ LexiconReader *lex_reader);
+
+ public incremented nullable SegPostingList*
+ Posting_List(DefaultPostingListReader *self, const CharBuf *field = NULL,
+ Obj *term = NULL);
+
+ LexiconReader*
+ Get_Lex_Reader(DefaultPostingListReader *self);
+
+ public void
+ Close(DefaultPostingListReader *self);
+
+ public void
+ Destroy(DefaultPostingListReader *self);
+}
+
+
diff --git a/core/Lucy/Index/PostingListWriter.c b/core/Lucy/Index/PostingListWriter.c
new file mode 100644
index 0000000..f56e8f8
--- /dev/null
+++ b/core/Lucy/Index/PostingListWriter.c
@@ -0,0 +1,258 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POSTINGLISTWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PostingListWriter.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/PostingPool.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/LexiconWriter.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/MemoryPool.h"
+
+static size_t default_mem_thresh = 0x1000000;
+
+int32_t PListWriter_current_file_format = 1;
+
+// Open streams only if content gets added.
+static void
+S_lazy_init(PostingListWriter *self);
+
+// Return the PostingPool for this field, creating one if necessary.
+static PostingPool*
+S_lazy_init_posting_pool(PostingListWriter *self, int32_t field_num);
+
+PostingListWriter*
+PListWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, LexiconWriter *lex_writer) {
+ PostingListWriter *self
+ = (PostingListWriter*)VTable_Make_Obj(POSTINGLISTWRITER);
+ return PListWriter_init(self, schema, snapshot, segment, polyreader,
+ lex_writer);
+}
+
+PostingListWriter*
+PListWriter_init(PostingListWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader,
+ LexiconWriter *lex_writer) {
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+
+ // Assign.
+ self->lex_writer = (LexiconWriter*)INCREF(lex_writer);
+
+ // Init.
+ self->pools = VA_new(Schema_Num_Fields(schema));
+ self->mem_thresh = default_mem_thresh;
+ self->mem_pool = MemPool_new(0);
+ self->lex_temp_out = NULL;
+ self->post_temp_out = NULL;
+
+ return self;
+}
+
+static void
+S_lazy_init(PostingListWriter *self) {
+ if (!self->lex_temp_out) {
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *lex_temp_path = CB_newf("%o/lextemp", seg_name);
+ CharBuf *post_temp_path = CB_newf("%o/ptemp", seg_name);
+ CharBuf *skip_path = CB_newf("%o/postings.skip", seg_name);
+
+ // Open temp streams and final skip stream.
+ self->lex_temp_out = Folder_Open_Out(folder, lex_temp_path);
+ if (!self->lex_temp_out) { RETHROW(INCREF(Err_get_error())); }
+ self->post_temp_out = Folder_Open_Out(folder, post_temp_path);
+ if (!self->post_temp_out) { RETHROW(INCREF(Err_get_error())); }
+ self->skip_out = Folder_Open_Out(folder, skip_path);
+ if (!self->skip_out) { RETHROW(INCREF(Err_get_error())); }
+
+ DECREF(skip_path);
+ DECREF(post_temp_path);
+ DECREF(lex_temp_path);
+ }
+}
+
+static PostingPool*
+S_lazy_init_posting_pool(PostingListWriter *self, int32_t field_num) {
+ PostingPool *pool = (PostingPool*)VA_Fetch(self->pools, field_num);
+ if (!pool && field_num != 0) {
+ CharBuf *field = Seg_Field_Name(self->segment, field_num);
+ pool = PostPool_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, field, self->lex_writer,
+ self->mem_pool, self->lex_temp_out,
+ self->post_temp_out, self->skip_out);
+ VA_Store(self->pools, field_num, (Obj*)pool);
+ }
+ return pool;
+}
+
+void
+PListWriter_destroy(PostingListWriter *self) {
+ DECREF(self->lex_writer);
+ DECREF(self->mem_pool);
+ DECREF(self->pools);
+ DECREF(self->lex_temp_out);
+ DECREF(self->post_temp_out);
+ DECREF(self->skip_out);
+ SUPER_DESTROY(self, POSTINGLISTWRITER);
+}
+
+void
+PListWriter_set_default_mem_thresh(size_t mem_thresh) {
+ default_mem_thresh = mem_thresh;
+}
+
+int32_t
+PListWriter_format(PostingListWriter *self) {
+ UNUSED_VAR(self);
+ return PListWriter_current_file_format;
+}
+
+void
+PListWriter_add_inverted_doc(PostingListWriter *self, Inverter *inverter,
+ int32_t doc_id) {
+ S_lazy_init(self);
+
+ // Iterate over fields in document, adding the content of indexed fields
+ // to their respective PostingPools.
+ float doc_boost = Inverter_Get_Boost(inverter);
+ Inverter_Iterate(inverter);
+ int32_t field_num;
+ while (0 != (field_num = Inverter_Next(inverter))) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Indexed(type)) {
+ Inversion *inversion = Inverter_Get_Inversion(inverter);
+ Similarity *sim = Inverter_Get_Similarity(inverter);
+ PostingPool *pool = S_lazy_init_posting_pool(self, field_num);
+ float length_norm
+ = Sim_Length_Norm(sim, Inversion_Get_Size(inversion));
+ PostPool_Add_Inversion(pool, inversion, doc_id, doc_boost,
+ length_norm);
+ }
+ }
+
+ // If our PostingPools have collectively passed the memory threshold,
+ // flush all of them, then release all the RawPostings with a single
+ // action.
+ if (MemPool_Get_Consumed(self->mem_pool) > self->mem_thresh) {
+ for (uint32_t i = 0, max = VA_Get_Size(self->pools); i < max; i++) {
+ PostingPool *const pool = (PostingPool*)VA_Fetch(self->pools, i);
+ if (pool) { PostPool_Flush(pool); }
+ }
+ MemPool_Release_All(self->mem_pool);
+ }
+}
+
+void
+PListWriter_add_segment(PostingListWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ Segment *other_segment = SegReader_Get_Segment(reader);
+ Schema *schema = self->schema;
+ Segment *segment = self->segment;
+ VArray *all_fields = Schema_All_Fields(schema);
+ S_lazy_init(self);
+
+ for (uint32_t i = 0, max = VA_Get_Size(all_fields); i < max; i++) {
+ CharBuf *field = (CharBuf*)VA_Fetch(all_fields, i);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ int32_t old_field_num = Seg_Field_Num(other_segment, field);
+ int32_t new_field_num = Seg_Field_Num(segment, field);
+
+ if (!FType_Indexed(type)) { continue; }
+ if (!old_field_num) { continue; } // not in old segment
+ if (!new_field_num) {
+ THROW(ERR, "Unrecognized field: %o", field);
+ }
+
+ PostingPool *pool = S_lazy_init_posting_pool(self, new_field_num);
+ PostPool_Add_Segment(pool, reader, doc_map,
+ (int32_t)Seg_Get_Count(segment));
+ }
+
+ // Clean up.
+ DECREF(all_fields);
+}
+
+void
+PListWriter_finish(PostingListWriter *self) {
+ // If S_lazy_init was never called, we have no data, so bail out.
+ if (!self->lex_temp_out) { return; }
+
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *lex_temp_path = CB_newf("%o/lextemp", seg_name);
+ CharBuf *post_temp_path = CB_newf("%o/ptemp", seg_name);
+
+ // Close temp streams.
+ OutStream_Close(self->lex_temp_out);
+ OutStream_Close(self->post_temp_out);
+
+ // Try to free up some memory.
+ for (uint32_t i = 0, max = VA_Get_Size(self->pools); i < max; i++) {
+ PostingPool *pool = (PostingPool*)VA_Fetch(self->pools, i);
+ if (pool) { PostPool_Shrink(pool); }
+ }
+
+ // Write postings for each field.
+ for (uint32_t i = 0, max = VA_Get_Size(self->pools); i < max; i++) {
+ PostingPool *pool = (PostingPool*)VA_Delete(self->pools, i);
+ if (pool) {
+ // Write out content for each PostingPool. Let each PostingPool
+ // use more RAM while finishing. (This is a little dicy, because if
+ // Shrink() was ineffective, we may double the RAM footprint.)
+ PostPool_Set_Mem_Thresh(pool, self->mem_thresh);
+ PostPool_Flip(pool);
+ PostPool_Finish(pool);
+ DECREF(pool);
+ }
+ }
+
+ // Store metadata.
+ Seg_Store_Metadata_Str(self->segment, "postings", 8,
+ (Obj*)PListWriter_Metadata(self));
+
+ // Close down and clean up.
+ OutStream_Close(self->skip_out);
+ if (!Folder_Delete(folder, lex_temp_path)) {
+ THROW(ERR, "Couldn't delete %o", lex_temp_path);
+ }
+ if (!Folder_Delete(folder, post_temp_path)) {
+ THROW(ERR, "Couldn't delete %o", post_temp_path);
+ }
+ DECREF(self->skip_out);
+ self->skip_out = NULL;
+ DECREF(post_temp_path);
+ DECREF(lex_temp_path);
+
+ // Dispatch the LexiconWriter.
+ LexWriter_Finish(self->lex_writer);
+}
+
+
diff --git a/core/Lucy/Index/PostingListWriter.cfh b/core/Lucy/Index/PostingListWriter.cfh
new file mode 100644
index 0000000..f967259
--- /dev/null
+++ b/core/Lucy/Index/PostingListWriter.cfh
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Write postings data to an index.
+ *
+ * PostingListWriter writes frequency and positional data files, plus feeds
+ * data to LexiconWriter.
+ */
+
+class Lucy::Index::PostingListWriter cnick PListWriter
+ inherits Lucy::Index::DataWriter {
+
+ LexiconWriter *lex_writer;
+ VArray *pools;
+ MemoryPool *mem_pool;
+ OutStream *lex_temp_out;
+ OutStream *post_temp_out;
+ OutStream *skip_out;
+ uint32_t mem_thresh;
+
+ inert int32_t current_file_format;
+
+ inert incremented PostingListWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, LexiconWriter *lex_writer);
+
+ inert PostingListWriter*
+ init(PostingListWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, LexiconWriter *lex_writer);
+
+ /** Test only. */
+ inert void
+ set_default_mem_thresh(size_t mem_thresh);
+
+ public void
+ Add_Inverted_Doc(PostingListWriter *self, Inverter *inverter,
+ int32_t doc_id);
+
+ public void
+ Add_Segment(PostingListWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Finish(PostingListWriter *self);
+
+ public int32_t
+ Format(PostingListWriter *self);
+
+ public void
+ Destroy(PostingListWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/PostingPool.c b/core/Lucy/Index/PostingPool.c
new file mode 100644
index 0000000..b0f6de5
--- /dev/null
+++ b/core/Lucy/Index/PostingPool.c
@@ -0,0 +1,594 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POSTINGPOOL
+#define C_LUCY_RAWPOSTING
+#define C_LUCY_MEMORYPOOL
+#define C_LUCY_TERMINFO
+#define C_LUCY_SKIPSTEPPER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PostingPool.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Index/LexiconWriter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/RawLexicon.h"
+#include "Lucy/Index/RawPostingList.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SkipStepper.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Index/TermStepper.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/MemoryPool.h"
+
+// Prepare to read back postings from disk.
+static void
+S_fresh_flip(PostingPool *self, InStream *lex_temp_in,
+ InStream *post_temp_in);
+
+// Main loop.
+static void
+S_write_terms_and_postings(PostingPool *self, PostingWriter *post_writer,
+ OutStream *skip_stream);
+
+PostingPool*
+PostPool_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, const CharBuf *field,
+ LexiconWriter *lex_writer, MemoryPool *mem_pool,
+ OutStream *lex_temp_out, OutStream *post_temp_out,
+ OutStream *skip_out) {
+ PostingPool *self = (PostingPool*)VTable_Make_Obj(POSTINGPOOL);
+ return PostPool_init(self, schema, snapshot, segment, polyreader, field,
+ lex_writer, mem_pool, lex_temp_out, post_temp_out,
+ skip_out);
+}
+
+PostingPool*
+PostPool_init(PostingPool *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, const CharBuf *field,
+ LexiconWriter *lex_writer, MemoryPool *mem_pool,
+ OutStream *lex_temp_out, OutStream *post_temp_out,
+ OutStream *skip_out) {
+ // Init.
+ SortEx_init((SortExternal*)self, sizeof(Obj*));
+ self->doc_base = 0;
+ self->last_doc_id = 0;
+ self->doc_map = NULL;
+ self->post_count = 0;
+ self->lexicon = NULL;
+ self->plist = NULL;
+ self->lex_temp_in = NULL;
+ self->post_temp_in = NULL;
+ self->lex_start = I64_MAX;
+ self->post_start = I64_MAX;
+ self->lex_end = 0;
+ self->post_end = 0;
+ self->skip_stepper = SkipStepper_new();
+
+ // Assign.
+ self->schema = (Schema*)INCREF(schema);
+ self->snapshot = (Snapshot*)INCREF(snapshot);
+ self->segment = (Segment*)INCREF(segment);
+ self->polyreader = (PolyReader*)INCREF(polyreader);
+ self->lex_writer = (LexiconWriter*)INCREF(lex_writer);
+ self->mem_pool = (MemoryPool*)INCREF(mem_pool);
+ self->field = CB_Clone(field);
+ self->lex_temp_out = (OutStream*)INCREF(lex_temp_out);
+ self->post_temp_out = (OutStream*)INCREF(post_temp_out);
+ self->skip_out = (OutStream*)INCREF(skip_out);
+
+ // Derive.
+ Similarity *sim = Schema_Fetch_Sim(schema, field);
+ self->posting = Sim_Make_Posting(sim);
+ self->type = (FieldType*)INCREF(Schema_Fetch_Type(schema, field));
+ self->field_num = Seg_Field_Num(segment, field);
+
+ return self;
+}
+
+void
+PostPool_destroy(PostingPool *self) {
+ DECREF(self->schema);
+ DECREF(self->snapshot);
+ DECREF(self->segment);
+ DECREF(self->polyreader);
+ DECREF(self->lex_writer);
+ DECREF(self->mem_pool);
+ DECREF(self->field);
+ DECREF(self->doc_map);
+ DECREF(self->lexicon);
+ DECREF(self->plist);
+ DECREF(self->lex_temp_out);
+ DECREF(self->post_temp_out);
+ DECREF(self->skip_out);
+ DECREF(self->lex_temp_in);
+ DECREF(self->post_temp_in);
+ DECREF(self->posting);
+ DECREF(self->skip_stepper);
+ DECREF(self->type);
+ SUPER_DESTROY(self, POSTINGPOOL);
+}
+
+int
+PostPool_compare(PostingPool *self, void *va, void *vb) {
+ RawPosting *const a = *(RawPosting**)va;
+ RawPosting *const b = *(RawPosting**)vb;
+ const size_t a_len = a->content_len;
+ const size_t b_len = b->content_len;
+ const size_t len = a_len < b_len ? a_len : b_len;
+ int comparison = memcmp(a->blob, b->blob, len);
+ UNUSED_VAR(self);
+
+ if (comparison == 0) {
+ // If a is a substring of b, it's less than b, so return a neg num.
+ comparison = a_len - b_len;
+
+ // Break ties by doc id.
+ if (comparison == 0) {
+ comparison = a->doc_id - b->doc_id;
+ }
+ }
+
+ return comparison;
+}
+
+MemoryPool*
+PostPool_get_mem_pool(PostingPool *self) {
+ return self->mem_pool;
+}
+
+void
+PostPool_flip(PostingPool *self) {
+ uint32_t i;
+ uint32_t num_runs = VA_Get_Size(self->runs);
+ uint32_t sub_thresh = num_runs > 0
+ ? self->mem_thresh / num_runs
+ : self->mem_thresh;
+
+ if (num_runs) {
+ Folder *folder = PolyReader_Get_Folder(self->polyreader);
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *lex_temp_path = CB_newf("%o/lextemp", seg_name);
+ CharBuf *post_temp_path = CB_newf("%o/ptemp", seg_name);
+ self->lex_temp_in = Folder_Open_In(folder, lex_temp_path);
+ if (!self->lex_temp_in) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ self->post_temp_in = Folder_Open_In(folder, post_temp_path);
+ if (!self->post_temp_in) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ DECREF(lex_temp_path);
+ DECREF(post_temp_path);
+ }
+
+ PostPool_Sort_Cache(self);
+ if (num_runs && (self->cache_max - self->cache_tick) > 0) {
+ uint32_t num_items = PostPool_Cache_Count(self);
+ // Cheap imitation of flush. FIXME.
+ PostingPool *run
+ = PostPool_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, self->field, self->lex_writer,
+ self->mem_pool, self->lex_temp_out,
+ self->post_temp_out, self->skip_out);
+ PostPool_Grow_Cache(run, num_items);
+ memcpy(run->cache, ((Obj**)self->cache) + self->cache_tick,
+ num_items * sizeof(Obj*));
+ run->cache_max = num_items;
+ PostPool_Add_Run(self, (SortExternal*)run);
+ self->cache_tick = 0;
+ self->cache_max = 0;
+ }
+
+ // Assign.
+ for (i = 0; i < num_runs; i++) {
+ PostingPool *run = (PostingPool*)VA_Fetch(self->runs, i);
+ if (run != NULL) {
+ PostPool_Set_Mem_Thresh(run, sub_thresh);
+ if (!run->lexicon) {
+ S_fresh_flip(run, self->lex_temp_in, self->post_temp_in);
+ }
+ }
+ }
+
+ self->flipped = true;
+}
+
+void
+PostPool_add_segment(PostingPool *self, SegReader *reader, I32Array *doc_map,
+ int32_t doc_base) {
+ LexiconReader *lex_reader = (LexiconReader*)SegReader_Fetch(
+ reader, VTable_Get_Name(LEXICONREADER));
+ Lexicon *lexicon = lex_reader
+ ? LexReader_Lexicon(lex_reader, self->field, NULL)
+ : NULL;
+
+ if (lexicon) {
+ PostingListReader *plist_reader
+ = (PostingListReader*)SegReader_Fetch(
+ reader, VTable_Get_Name(POSTINGLISTREADER));
+ PostingList *plist = plist_reader
+ ? PListReader_Posting_List(plist_reader, self->field, NULL)
+ : NULL;
+ if (!plist) {
+ THROW(ERR, "Got a Lexicon but no PostingList for '%o' in '%o'",
+ self->field, SegReader_Get_Seg_Name(reader));
+ }
+ PostingPool *run
+ = PostPool_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, self->field, self->lex_writer,
+ self->mem_pool, self->lex_temp_out,
+ self->post_temp_out, self->skip_out);
+ run->lexicon = lexicon;
+ run->plist = plist;
+ run->doc_base = doc_base;
+ run->doc_map = (I32Array*)INCREF(doc_map);
+ PostPool_Add_Run(self, (SortExternal*)run);
+ }
+}
+
+void
+PostPool_shrink(PostingPool *self) {
+ if (self->cache_max - self->cache_tick > 0) {
+ size_t cache_count = PostPool_Cache_Count(self);
+ size_t size = cache_count * sizeof(Obj*);
+ if (self->cache_tick > 0) {
+ Obj **start = ((Obj**)self->cache) + self->cache_tick;
+ memmove(self->cache, start, size);
+ }
+ self->cache = (uint8_t*)REALLOCATE(self->cache, size);
+ self->cache_tick = 0;
+ self->cache_max = cache_count;
+ self->cache_cap = cache_count;
+ }
+ else {
+ FREEMEM(self->cache);
+ self->cache = NULL;
+ self->cache_tick = 0;
+ self->cache_max = 0;
+ self->cache_cap = 0;
+ }
+ self->scratch_cap = 0;
+ FREEMEM(self->scratch);
+ self->scratch = NULL;
+
+ // It's not necessary to iterate over the runs, because they don't have
+ // any cache costs until Refill() gets called.
+}
+
+void
+PostPool_flush(PostingPool *self) {
+ // Don't add a run unless we have data to put in it.
+ if (PostPool_Cache_Count(self) == 0) { return; }
+
+ PostingPool *run
+ = PostPool_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, self->field, self->lex_writer,
+ self->mem_pool, self->lex_temp_out,
+ self->post_temp_out, self->skip_out);
+ PostingWriter *post_writer
+ = (PostingWriter*)RawPostWriter_new(self->schema, self->snapshot,
+ self->segment, self->polyreader,
+ self->post_temp_out);
+
+ // Borrow the cache.
+ run->cache = self->cache;
+ run->cache_tick = self->cache_tick;
+ run->cache_max = self->cache_max;
+ run->cache_cap = self->cache_cap;
+
+ // Write to temp files.
+ LexWriter_Enter_Temp_Mode(self->lex_writer, self->field,
+ self->lex_temp_out);
+ run->lex_start = OutStream_Tell(self->lex_temp_out);
+ run->post_start = OutStream_Tell(self->post_temp_out);
+ PostPool_Sort_Cache(self);
+ S_write_terms_and_postings(run, post_writer, NULL);
+
+ run->lex_end = OutStream_Tell(self->lex_temp_out);
+ run->post_end = OutStream_Tell(self->post_temp_out);
+ LexWriter_Leave_Temp_Mode(self->lex_writer);
+
+ // Return the cache and empty it.
+ run->cache = NULL;
+ run->cache_tick = 0;
+ run->cache_max = 0;
+ run->cache_cap = 0;
+ PostPool_Clear_Cache(self);
+
+ // Add the run to the array.
+ PostPool_Add_Run(self, (SortExternal*)run);
+
+ DECREF(post_writer);
+}
+
+void
+PostPool_finish(PostingPool *self) {
+ // Bail if there's no data.
+ if (!PostPool_Peek(self)) { return; }
+
+ Similarity *sim = Schema_Fetch_Sim(self->schema, self->field);
+ PostingWriter *post_writer
+ = Sim_Make_Posting_Writer(sim, self->schema, self->snapshot,
+ self->segment, self->polyreader,
+ self->field_num);
+ LexWriter_Start_Field(self->lex_writer, self->field_num);
+ S_write_terms_and_postings(self, post_writer, self->skip_out);
+ LexWriter_Finish_Field(self->lex_writer, self->field_num);
+ DECREF(post_writer);
+}
+
+static void
+S_write_terms_and_postings(PostingPool *self, PostingWriter *post_writer,
+ OutStream *skip_stream) {
+ TermInfo *const tinfo = TInfo_new(0);
+ TermInfo *const skip_tinfo = TInfo_new(0);
+ CharBuf *const last_term_text = CB_new(0);
+ LexiconWriter *const lex_writer = self->lex_writer;
+ SkipStepper *const skip_stepper = self->skip_stepper;
+ int32_t last_doc_id = 0;
+ int32_t last_skip_doc = 0;
+ int64_t last_skip_filepos = 0;
+ const int32_t skip_interval
+ = Arch_Skip_Interval(Schema_Get_Architecture(self->schema));
+
+ // Prime heldover variables.
+ RawPosting *posting = (RawPosting*)CERTIFY(
+ (*(RawPosting**)PostPool_Fetch(self)),
+ RAWPOSTING);
+ CB_Mimic_Str(last_term_text, posting->blob, posting->content_len);
+ char *last_text_buf = (char*)CB_Get_Ptr8(last_term_text);
+ uint32_t last_text_size = CB_Get_Size(last_term_text);
+ SkipStepper_Set_ID_And_Filepos(skip_stepper, 0, 0);
+
+ while (1) {
+ bool_t same_text_as_last = true;
+
+ if (posting == NULL) {
+ // On the last iter, use an empty string to make LexiconWriter
+ // DTRT.
+ posting = &RAWPOSTING_BLANK;
+ same_text_as_last = false;
+ }
+ else {
+ // Compare once.
+ if (posting->content_len != last_text_size
+ || memcmp(&posting->blob, last_text_buf, last_text_size) != 0
+ ) {
+ same_text_as_last = false;
+ }
+ }
+
+ // If the term text changes, process the last term.
+ if (!same_text_as_last) {
+ // Hand off to LexiconWriter.
+ LexWriter_Add_Term(lex_writer, last_term_text, tinfo);
+
+ // Start each term afresh.
+ TInfo_Reset(tinfo);
+ PostWriter_Start_Term(post_writer, tinfo);
+
+ // Init skip data in preparation for the next term.
+ skip_stepper->doc_id = 0;
+ skip_stepper->filepos = tinfo->post_filepos;
+ last_skip_doc = 0;
+ last_skip_filepos = tinfo->post_filepos;
+
+ // Remember the term_text so we can write string diffs.
+ CB_Mimic_Str(last_term_text, posting->blob,
+ posting->content_len);
+ last_text_buf = (char*)CB_Get_Ptr8(last_term_text);
+ last_text_size = CB_Get_Size(last_term_text);
+
+ // Starting a new term, thus a new delta doc sequence at 0.
+ last_doc_id = 0;
+ }
+
+ // Bail on last iter before writing invalid posting data.
+ if (posting == &RAWPOSTING_BLANK) { break; }
+
+ // Write posting data.
+ PostWriter_Write_Posting(post_writer, posting);
+
+ // Doc freq lags by one iter.
+ tinfo->doc_freq++;
+
+ // Write skip data.
+ if (skip_stream != NULL
+ && same_text_as_last
+ && tinfo->doc_freq % skip_interval == 0
+ && tinfo->doc_freq != 0
+ ) {
+ // If first skip group, save skip stream pos for term info.
+ if (tinfo->doc_freq == skip_interval) {
+ tinfo->skip_filepos = OutStream_Tell(skip_stream);
+ }
+ // Write deltas.
+ last_skip_doc = skip_stepper->doc_id;
+ last_skip_filepos = skip_stepper->filepos;
+ skip_stepper->doc_id = posting->doc_id;
+ PostWriter_Update_Skip_Info(post_writer, skip_tinfo);
+ skip_stepper->filepos = skip_tinfo->post_filepos;
+ SkipStepper_Write_Record(skip_stepper, skip_stream,
+ last_skip_doc, last_skip_filepos);
+ }
+
+ // Remember last doc id because we need it for delta encoding.
+ last_doc_id = posting->doc_id;
+
+ // Retrieve the next posting from the sort pool.
+ // DECREF(posting); // No!! DON'T destroy!!!
+
+ void *address = PostPool_Fetch(self);
+ posting = address
+ ? *(RawPosting**)address
+ : NULL;
+ }
+
+ // Clean up.
+ DECREF(last_term_text);
+ DECREF(skip_tinfo);
+ DECREF(tinfo);
+}
+
+uint32_t
+PostPool_refill(PostingPool *self) {
+ Lexicon *const lexicon = self->lexicon;
+ PostingList *const plist = self->plist;
+ I32Array *const doc_map = self->doc_map;
+ const uint32_t mem_thresh = self->mem_thresh;
+ const int32_t doc_base = self->doc_base;
+ uint32_t num_elems = 0; // number of items recovered
+ MemoryPool *mem_pool;
+ CharBuf *term_text = NULL;
+
+ if (self->lexicon == NULL) { return 0; }
+ else { term_text = (CharBuf*)Lex_Get_Term(lexicon); }
+
+ // Make sure cache is empty.
+ if (self->cache_max - self->cache_tick > 0) {
+ THROW(ERR, "Refill called but cache contains %u32 items",
+ self->cache_max - self->cache_tick);
+ }
+ self->cache_max = 0;
+ self->cache_tick = 0;
+
+ // Ditch old MemoryPool and get another.
+ DECREF(self->mem_pool);
+ self->mem_pool = MemPool_new(0);
+ mem_pool = self->mem_pool;
+
+ while (1) {
+ RawPosting *raw_posting;
+
+ if (self->post_count == 0) {
+ // Read a term.
+ if (Lex_Next(lexicon)) {
+ self->post_count = Lex_Doc_Freq(lexicon);
+ term_text = (CharBuf*)Lex_Get_Term(lexicon);
+ if (term_text && !Obj_Is_A((Obj*)term_text, CHARBUF)) {
+ THROW(ERR, "Only CharBuf terms are supported for now");
+ }
+ {
+ Posting *posting = PList_Get_Posting(plist);
+ Post_Set_Doc_ID(posting, doc_base);
+ self->last_doc_id = doc_base;
+ }
+ }
+ // Bail if we've read everything in this run.
+ else {
+ break;
+ }
+ }
+
+ // Bail if we've hit the ceiling for this run's cache.
+ if (mem_pool->consumed >= mem_thresh && num_elems > 0) {
+ break;
+ }
+
+ // Read a posting from the input stream.
+ raw_posting = PList_Read_Raw(plist, self->last_doc_id, term_text,
+ mem_pool);
+ self->last_doc_id = raw_posting->doc_id;
+ self->post_count--;
+
+ // Skip deletions.
+ if (doc_map != NULL) {
+ const int32_t remapped
+ = I32Arr_Get(doc_map, raw_posting->doc_id - doc_base);
+ if (!remapped) {
+ continue;
+ }
+ raw_posting->doc_id = remapped;
+ }
+
+ // Add to the run's cache.
+ if (num_elems >= self->cache_cap) {
+ size_t new_cap = Memory_oversize(num_elems + 1, sizeof(Obj*));
+ PostPool_Grow_Cache(self, new_cap);
+ }
+ Obj **cache = (Obj**)self->cache;
+ cache[num_elems] = (Obj*)raw_posting;
+ num_elems++;
+ }
+
+ // Reset the cache array position and length; remember file pos.
+ self->cache_max = num_elems;
+ self->cache_tick = 0;
+
+ return num_elems;
+}
+
+void
+PostPool_add_inversion(PostingPool *self, Inversion *inversion, int32_t doc_id,
+ float doc_boost, float length_norm) {
+ Post_Add_Inversion_To_Pool(self->posting, self, inversion, self->type,
+ doc_id, doc_boost, length_norm);
+}
+
+static void
+S_fresh_flip(PostingPool *self, InStream *lex_temp_in,
+ InStream *post_temp_in) {
+ if (self->flipped) { THROW(ERR, "Can't Flip twice"); }
+ self->flipped = true;
+
+ // Sort RawPostings in cache, if any.
+ PostPool_Sort_Cache(self);
+
+ // Bail if never flushed.
+ if (self->lex_end == 0) { return; }
+
+ // Get a Lexicon.
+ CharBuf *lex_alias = CB_newf("%o-%i64-to-%i64",
+ InStream_Get_Filename(lex_temp_in),
+ self->lex_start, self->lex_end);
+ InStream *lex_temp_in_dupe = InStream_Reopen(
+ lex_temp_in, lex_alias, self->lex_start,
+ self->lex_end - self->lex_start);
+ self->lexicon = (Lexicon*)RawLex_new(
+ self->schema, self->field, lex_temp_in_dupe, 0,
+ self->lex_end - self->lex_start);
+ DECREF(lex_alias);
+ DECREF(lex_temp_in_dupe);
+
+ // Get a PostingList.
+ CharBuf *post_alias
+ = CB_newf("%o-%i64-to-%i64", InStream_Get_Filename(post_temp_in),
+ self->post_start, self->post_end);
+ InStream *post_temp_in_dupe
+ = InStream_Reopen(post_temp_in, post_alias, self->post_start,
+ self->post_end - self->post_start);
+ self->plist
+ = (PostingList*)RawPList_new(self->schema, self->field,
+ post_temp_in_dupe, 0,
+ self->post_end - self->post_start);
+ DECREF(post_alias);
+ DECREF(post_temp_in_dupe);
+}
+
+
diff --git a/core/Lucy/Index/PostingPool.cfh b/core/Lucy/Index/PostingPool.cfh
new file mode 100644
index 0000000..b4f2614
--- /dev/null
+++ b/core/Lucy/Index/PostingPool.cfh
@@ -0,0 +1,105 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * External sorter for raw postings.
+ */
+class Lucy::Index::PostingPool cnick PostPool
+ inherits Lucy::Util::SortExternal {
+
+ Schema *schema;
+ Snapshot *snapshot;
+ Segment *segment;
+ PolyReader *polyreader;
+ CharBuf *field;
+ LexiconWriter *lex_writer;
+ Lexicon *lexicon;
+ PostingList *plist;
+ MemoryPool *mem_pool;
+ I32Array *doc_map;
+ int32_t field_num;
+ int32_t doc_base;
+ int32_t last_doc_id;
+ uint32_t post_count;
+ OutStream *lex_temp_out;
+ OutStream *post_temp_out;
+ OutStream *skip_out;
+ InStream *lex_temp_in;
+ InStream *post_temp_in;
+ FieldType *type;
+ Posting *posting;
+ SkipStepper *skip_stepper;
+ int64_t lex_start;
+ int64_t post_start;
+ int64_t lex_end;
+ int64_t post_end;
+
+ inert incremented PostingPool*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, const CharBuf *field,
+ LexiconWriter *lex_writer, MemoryPool *mem_pool,
+ OutStream *lex_temp_out, OutStream *post_temp_out,
+ OutStream *skip_out);
+
+ inert PostingPool*
+ init(PostingPool *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, const CharBuf *field,
+ LexiconWriter *lex_writer, MemoryPool *mem_pool,
+ OutStream *lex_temp_out, OutStream *post_temp_out,
+ OutStream *skip_out);
+
+ /** Add a field's inverted content.
+ */
+ void
+ Add_Inversion(PostingPool *self, Inversion *inversion, int32_t doc_id,
+ float doc_boost, float length_norm);
+
+ /** Reduce RAM footprint as much as possible.
+ */
+ void
+ Shrink(PostingPool *self);
+
+ MemoryPool*
+ Get_Mem_Pool(PostingPool *self);
+
+ void
+ Add_Segment(PostingPool *self, SegReader *reader, I32Array *doc_map,
+ int32_t doc_base);
+
+ void
+ Flip(PostingPool *self);
+
+ uint32_t
+ Refill(PostingPool *self);
+
+ /** Compares two non-NULL RawPosting objects.
+ */
+ int
+ Compare(PostingPool *self, void *va, void *vb);
+
+ void
+ Finish(PostingPool *self);
+
+ void
+ Flush(PostingPool *self);
+
+ public void
+ Destroy(PostingPool *self);
+}
+
+
diff --git a/core/Lucy/Index/RawLexicon.c b/core/Lucy/Index/RawLexicon.c
new file mode 100644
index 0000000..9b6a6d8
--- /dev/null
+++ b/core/Lucy/Index/RawLexicon.c
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAWLEXICON
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/RawLexicon.h"
+#include "Lucy/Index/Posting/MatchPosting.h"
+#include "Lucy/Index/TermStepper.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+
+RawLexicon*
+RawLex_new(Schema *schema, const CharBuf *field, InStream *instream,
+ int64_t start, int64_t end) {
+ RawLexicon *self = (RawLexicon*)VTable_Make_Obj(RAWLEXICON);
+ return RawLex_init(self, schema, field, instream, start, end);
+}
+
+RawLexicon*
+RawLex_init(RawLexicon *self, Schema *schema, const CharBuf *field,
+ InStream *instream, int64_t start, int64_t end) {
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ Lex_init((Lexicon*)self, field);
+
+ // Assign
+ self->start = start;
+ self->end = end;
+ self->len = end - start;
+ self->instream = (InStream*)INCREF(instream);
+
+ // Get ready to begin.
+ InStream_Seek(self->instream, self->start);
+
+ // Get steppers.
+ self->term_stepper = FType_Make_Term_Stepper(type);
+ self->tinfo_stepper = (TermStepper*)MatchTInfoStepper_new(schema);
+
+ return self;
+}
+
+void
+RawLex_destroy(RawLexicon *self) {
+ DECREF(self->instream);
+ DECREF(self->term_stepper);
+ DECREF(self->tinfo_stepper);
+ SUPER_DESTROY(self, RAWLEXICON);
+}
+
+bool_t
+RawLex_next(RawLexicon *self) {
+ if (InStream_Tell(self->instream) >= self->len) { return false; }
+ TermStepper_Read_Delta(self->term_stepper, self->instream);
+ TermStepper_Read_Delta(self->tinfo_stepper, self->instream);
+ return true;
+}
+
+Obj*
+RawLex_get_term(RawLexicon *self) {
+ return TermStepper_Get_Value(self->term_stepper);
+}
+
+int32_t
+RawLex_doc_freq(RawLexicon *self) {
+ TermInfo *tinfo = (TermInfo*)TermStepper_Get_Value(self->tinfo_stepper);
+ return tinfo ? TInfo_Get_Doc_Freq(tinfo) : 0;
+}
+
+
diff --git a/core/Lucy/Index/RawLexicon.cfh b/core/Lucy/Index/RawLexicon.cfh
new file mode 100644
index 0000000..06d579f
--- /dev/null
+++ b/core/Lucy/Index/RawLexicon.cfh
@@ -0,0 +1,52 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Private scan-only Lexicon helper class.
+ */
+class Lucy::Index::RawLexicon cnick RawLex
+ inherits Lucy::Index::Lexicon {
+
+ InStream *instream;
+ TermStepper *term_stepper;
+ TermStepper *tinfo_stepper;
+ int64_t start;
+ int64_t end;
+ int64_t len;
+
+ inert incremented RawLexicon*
+ new(Schema *schema, const CharBuf *field, InStream *instream,
+ int64_t start, int64_t end);
+
+ inert RawLexicon*
+ init(RawLexicon *self, Schema *schema, const CharBuf *field,
+ InStream *instream, int64_t start, int64_t end);
+
+ public void
+ Destroy(RawLexicon *self);
+
+ public bool_t
+ Next(RawLexicon *self);
+
+ public nullable Obj*
+ Get_Term(RawLexicon *self);
+
+ public int32_t
+ Doc_Freq(RawLexicon *self);
+}
+
+
diff --git a/core/Lucy/Index/RawPostingList.c b/core/Lucy/Index/RawPostingList.c
new file mode 100644
index 0000000..a22a015
--- /dev/null
+++ b/core/Lucy/Index/RawPostingList.c
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAWPOSTINGLIST
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/RawPostingList.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Util/MemoryPool.h"
+
+RawPostingList*
+RawPList_new(Schema *schema, const CharBuf *field, InStream *instream,
+ int64_t start, int64_t end) {
+ RawPostingList *self = (RawPostingList*)VTable_Make_Obj(RAWPOSTINGLIST);
+ return RawPList_init(self, schema, field, instream, start, end);
+}
+
+RawPostingList*
+RawPList_init(RawPostingList *self, Schema *schema, const CharBuf *field,
+ InStream *instream, int64_t start, int64_t end) {
+ PList_init((PostingList*)self);
+ self->start = start;
+ self->end = end;
+ self->len = end - start;
+ self->instream = (InStream*)INCREF(instream);
+ Similarity *sim = Schema_Fetch_Sim(schema, field);
+ self->posting = Sim_Make_Posting(sim);
+ InStream_Seek(self->instream, self->start);
+ return self;
+}
+
+void
+RawPList_destroy(RawPostingList *self) {
+ DECREF(self->instream);
+ DECREF(self->posting);
+ SUPER_DESTROY(self, RAWPOSTINGLIST);
+}
+
+Posting*
+RawPList_get_posting(RawPostingList *self) {
+ return self->posting;
+}
+
+RawPosting*
+RawPList_read_raw(RawPostingList *self, int32_t last_doc_id, CharBuf *term_text,
+ MemoryPool *mem_pool) {
+ return Post_Read_Raw(self->posting, self->instream,
+ last_doc_id, term_text, mem_pool);
+}
+
+
diff --git a/core/Lucy/Index/RawPostingList.cfh b/core/Lucy/Index/RawPostingList.cfh
new file mode 100644
index 0000000..423a36c
--- /dev/null
+++ b/core/Lucy/Index/RawPostingList.cfh
@@ -0,0 +1,47 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::RawPostingList cnick RawPList
+ inherits Lucy::Index::PostingList {
+
+ Posting *posting;
+ InStream *instream;
+ int64_t start;
+ int64_t end;
+ int64_t len;
+
+ inert incremented RawPostingList*
+ new(Schema *schema, const CharBuf *field, InStream *instream,
+ int64_t start, int64_t end);
+
+ inert RawPostingList*
+ init(RawPostingList *self, Schema *schema, const CharBuf *field,
+ InStream *instream, int64_t lex_start, int64_t lex_end);
+
+ public void
+ Destroy(RawPostingList *self);
+
+ RawPosting*
+ Read_Raw(RawPostingList *self, int32_t last_doc_id, CharBuf *term_text,
+ MemoryPool *mem_pool);
+
+ Posting*
+ Get_Posting(RawPostingList *self);
+}
+
+
diff --git a/core/Lucy/Index/SegLexicon.c b/core/Lucy/Index/SegLexicon.c
new file mode 100644
index 0000000..43087aa
--- /dev/null
+++ b/core/Lucy/Index/SegLexicon.c
@@ -0,0 +1,210 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SEGLEXICON
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SegLexicon.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Index/LexIndex.h"
+#include "Lucy/Index/LexiconWriter.h"
+#include "Lucy/Index/Posting/MatchPosting.h"
+#include "Lucy/Index/SegPostingList.h"
+#include "Lucy/Index/TermStepper.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+
+// Iterate until the state is greater than or equal to the target.
+static void
+S_scan_to(SegLexicon *self, Obj *target);
+
+SegLexicon*
+SegLex_new(Schema *schema, Folder *folder, Segment *segment,
+ const CharBuf *field) {
+ SegLexicon *self = (SegLexicon*)VTable_Make_Obj(SEGLEXICON);
+ return SegLex_init(self, schema, folder, segment, field);
+}
+
+SegLexicon*
+SegLex_init(SegLexicon *self, Schema *schema, Folder *folder,
+ Segment *segment, const CharBuf *field) {
+ Hash *metadata = (Hash*)CERTIFY(
+ Seg_Fetch_Metadata_Str(segment, "lexicon", 7),
+ HASH);
+ Architecture *arch = Schema_Get_Architecture(schema);
+ Hash *counts = (Hash*)Hash_Fetch_Str(metadata, "counts", 6);
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ int32_t field_num = Seg_Field_Num(segment, field);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ CharBuf *filename = CB_newf("%o/lexicon-%i32.dat", seg_name, field_num);
+
+ Lex_init((Lexicon*)self, field);
+
+ // Check format.
+ if (!format) { THROW(ERR, "Missing 'format'"); }
+ else {
+ if (Obj_To_I64(format) > LexWriter_current_file_format) {
+ THROW(ERR, "Unsupported lexicon format: %i64",
+ Obj_To_I64(format));
+ }
+ }
+
+ // Extract count from metadata.
+ if (!counts) { THROW(ERR, "Failed to extract 'counts'"); }
+ else {
+ Obj *count = CERTIFY(Hash_Fetch(counts, (Obj*)field), OBJ);
+ self->size = (int32_t)Obj_To_I64(count);
+ }
+
+ // Assign.
+ self->segment = (Segment*)INCREF(segment);
+
+ // Derive.
+ self->lex_index = LexIndex_new(schema, folder, segment, field);
+ self->field_num = field_num;
+ self->index_interval = Arch_Index_Interval(arch);
+ self->skip_interval = Arch_Skip_Interval(arch);
+ self->instream = Folder_Open_In(folder, filename);
+ if (!self->instream) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(filename);
+ DECREF(self);
+ RETHROW(error);
+ }
+ DECREF(filename);
+
+ // Define the term_num as "not yet started".
+ self->term_num = -1;
+
+ // Get steppers.
+ self->term_stepper = FType_Make_Term_Stepper(type);
+ self->tinfo_stepper = (TermStepper*)MatchTInfoStepper_new(schema);
+
+ return self;
+}
+
+void
+SegLex_destroy(SegLexicon *self) {
+ DECREF(self->segment);
+ DECREF(self->term_stepper);
+ DECREF(self->tinfo_stepper);
+ DECREF(self->lex_index);
+ DECREF(self->instream);
+ SUPER_DESTROY(self, SEGLEXICON);
+}
+
+void
+SegLex_seek(SegLexicon *self, Obj *target) {
+ LexIndex *const lex_index = self->lex_index;
+
+ // Reset upon null term.
+ if (target == NULL) {
+ SegLex_Reset(self);
+ return;
+ }
+
+ // Use the LexIndex to get in the ballpark.
+ LexIndex_Seek(lex_index, target);
+ {
+ TermInfo *target_tinfo = LexIndex_Get_Term_Info(lex_index);
+ TermInfo *my_tinfo
+ = (TermInfo*)TermStepper_Get_Value(self->tinfo_stepper);
+ Obj *lex_index_term = Obj_Clone(LexIndex_Get_Term(lex_index));
+ TInfo_Mimic(my_tinfo, (Obj*)target_tinfo);
+ TermStepper_Set_Value(self->term_stepper, lex_index_term);
+ DECREF(lex_index_term);
+ InStream_Seek(self->instream, TInfo_Get_Lex_FilePos(target_tinfo));
+ }
+ self->term_num = LexIndex_Get_Term_Num(lex_index);
+
+ // Scan to the precise location.
+ S_scan_to(self, target);
+}
+
+void
+SegLex_reset(SegLexicon* self) {
+ self->term_num = -1;
+ InStream_Seek(self->instream, 0);
+ TermStepper_Reset(self->term_stepper);
+ TermStepper_Reset(self->tinfo_stepper);
+}
+
+int32_t
+SegLex_get_field_num(SegLexicon *self) {
+ return self->field_num;
+}
+
+Obj*
+SegLex_get_term(SegLexicon *self) {
+ return TermStepper_Get_Value(self->term_stepper);
+}
+
+int32_t
+SegLex_doc_freq(SegLexicon *self) {
+ TermInfo *tinfo = (TermInfo*)TermStepper_Get_Value(self->tinfo_stepper);
+ return tinfo ? TInfo_Get_Doc_Freq(tinfo) : 0;
+}
+
+TermInfo*
+SegLex_get_term_info(SegLexicon *self) {
+ return (TermInfo*)TermStepper_Get_Value(self->tinfo_stepper);
+}
+
+Segment*
+SegLex_get_segment(SegLexicon *self) {
+ return self->segment;
+}
+
+bool_t
+SegLex_next(SegLexicon *self) {
+ // If we've run out of terms, null out and return.
+ if (++self->term_num >= self->size) {
+ self->term_num = self->size; // don't keep growing
+ TermStepper_Reset(self->term_stepper);
+ TermStepper_Reset(self->tinfo_stepper);
+ return false;
+ }
+
+ // Read next term/terminfo.
+ TermStepper_Read_Delta(self->term_stepper, self->instream);
+ TermStepper_Read_Delta(self->tinfo_stepper, self->instream);
+
+ return true;
+}
+
+static void
+S_scan_to(SegLexicon *self, Obj *target) {
+ // (mildly evil encapsulation violation, since value can be null)
+ Obj *current = TermStepper_Get_Value(self->term_stepper);
+ if (!Obj_Is_A(target, Obj_Get_VTable(current))) {
+ THROW(ERR, "Target is a %o, and not comparable to a %o",
+ Obj_Get_Class_Name(target), Obj_Get_Class_Name(current));
+ }
+
+ // Keep looping until the term text is ge target.
+ do {
+ const int32_t comparison = Obj_Compare_To(current, target);
+ if (comparison >= 0 && self->term_num != -1) { break; }
+ } while (SegLex_Next(self));
+}
+
+
diff --git a/core/Lucy/Index/SegLexicon.cfh b/core/Lucy/Index/SegLexicon.cfh
new file mode 100644
index 0000000..3e28ab0
--- /dev/null
+++ b/core/Lucy/Index/SegLexicon.cfh
@@ -0,0 +1,78 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Single-segment Lexicon.
+ */
+
+class Lucy::Index::SegLexicon cnick SegLex
+ inherits Lucy::Index::Lexicon {
+
+ Segment *segment;
+ TermStepper *term_stepper;
+ TermStepper *tinfo_stepper;
+ InStream *instream;
+ LexIndex *lex_index;
+ int32_t field_num;
+ int32_t size;
+ int32_t term_num;
+ int32_t skip_interval;
+ int32_t index_interval;
+
+ /**
+ * @param schema A Schema.
+ * @param folder A Folder.
+ * @param segment A Segment.
+ * @param field The field whose terms the Lexicon will iterate over.
+ */
+ inert incremented SegLexicon*
+ new(Schema *schema, Folder *folder, Segment *segment,
+ const CharBuf *field);
+
+ inert SegLexicon*
+ init(SegLexicon *self, Schema *schema, Folder *folder, Segment *segment,
+ const CharBuf *field);
+
+ nullable TermInfo*
+ Get_Term_Info(SegLexicon *self);
+
+ int32_t
+ Get_Field_Num(SegLexicon *self);
+
+ Segment*
+ Get_Segment(SegLexicon *self);
+
+ public void
+ Destroy(SegLexicon *self);
+
+ public void
+ Seek(SegLexicon*self, Obj *target = NULL);
+
+ public void
+ Reset(SegLexicon* self);
+
+ public nullable Obj*
+ Get_Term(SegLexicon *self);
+
+ public int32_t
+ Doc_Freq(SegLexicon *self);
+
+ public bool_t
+ Next(SegLexicon *self);
+}
+
+
diff --git a/core/Lucy/Index/SegPostingList.c b/core/Lucy/Index/SegPostingList.c
new file mode 100644
index 0000000..d0195c8
--- /dev/null
+++ b/core/Lucy/Index/SegPostingList.c
@@ -0,0 +1,307 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SEGPOSTINGLIST
+#define C_LUCY_POSTING
+#define C_LUCY_SKIPSTEPPER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SegPostingList.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/Posting/RawPosting.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SkipStepper.h"
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Index/SegLexicon.h"
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Util/MemoryPool.h"
+
+// Low level seek call.
+static void
+S_seek_tinfo(SegPostingList *self, TermInfo *tinfo);
+
+SegPostingList*
+SegPList_new(PostingListReader *plist_reader, const CharBuf *field) {
+ SegPostingList *self = (SegPostingList*)VTable_Make_Obj(SEGPOSTINGLIST);
+ return SegPList_init(self, plist_reader, field);
+}
+
+SegPostingList*
+SegPList_init(SegPostingList *self, PostingListReader *plist_reader,
+ const CharBuf *field) {
+ Schema *const schema = PListReader_Get_Schema(plist_reader);
+ Folder *const folder = PListReader_Get_Folder(plist_reader);
+ Segment *const segment = PListReader_Get_Segment(plist_reader);
+ Architecture *const arch = Schema_Get_Architecture(schema);
+ CharBuf *const seg_name = Seg_Get_Name(segment);
+ int32_t field_num = Seg_Field_Num(segment, field);
+ CharBuf *post_file = CB_newf("%o/postings-%i32.dat",
+ seg_name, field_num);
+ CharBuf *skip_file = CB_newf("%o/postings.skip", seg_name);
+
+ // Init.
+ self->doc_freq = 0;
+ self->count = 0;
+
+ // Init skipping vars.
+ self->skip_stepper = SkipStepper_new();
+ self->skip_count = 0;
+ self->num_skips = 0;
+
+ // Assign.
+ self->plist_reader = (PostingListReader*)INCREF(plist_reader);
+ self->field = CB_Clone(field);
+ self->skip_interval = Arch_Skip_Interval(arch);
+
+ // Derive.
+ Similarity *sim = Schema_Fetch_Sim(schema, field);
+ self->posting = Sim_Make_Posting(sim);
+ self->field_num = field_num;
+
+ // Open both a main stream and a skip stream if the field exists.
+ if (Folder_Exists(folder, post_file)) {
+ self->post_stream = Folder_Open_In(folder, post_file);
+ if (!self->post_stream) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(post_file);
+ DECREF(skip_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ self->skip_stream = Folder_Open_In(folder, skip_file);
+ if (!self->skip_stream) {
+ Err *error = (Err*)INCREF(Err_get_error());
+ DECREF(post_file);
+ DECREF(skip_file);
+ DECREF(self);
+ RETHROW(error);
+ }
+ }
+ else {
+ // Empty, so don't bother with these.
+ self->post_stream = NULL;
+ self->skip_stream = NULL;
+ }
+ DECREF(post_file);
+ DECREF(skip_file);
+
+ return self;
+}
+
+void
+SegPList_destroy(SegPostingList *self) {
+ DECREF(self->plist_reader);
+ DECREF(self->posting);
+ DECREF(self->skip_stepper);
+ DECREF(self->field);
+
+ if (self->post_stream != NULL) {
+ InStream_Close(self->post_stream);
+ InStream_Close(self->skip_stream);
+ DECREF(self->post_stream);
+ DECREF(self->skip_stream);
+ }
+
+ SUPER_DESTROY(self, SEGPOSTINGLIST);
+}
+
+Posting*
+SegPList_get_posting(SegPostingList *self) {
+ return self->posting;
+}
+
+uint32_t
+SegPList_get_doc_freq(SegPostingList *self) {
+ return self->doc_freq;
+}
+
+int32_t
+SegPList_get_doc_id(SegPostingList *self) {
+ return self->posting->doc_id;
+}
+
+uint32_t
+SegPList_get_count(SegPostingList *self) {
+ return self->count;
+}
+
+InStream*
+SegPList_get_post_stream(SegPostingList *self) {
+ return self->post_stream;
+}
+
+int32_t
+SegPList_next(SegPostingList *self) {
+ InStream *const post_stream = self->post_stream;
+ Posting *const posting = self->posting;
+
+ // Bail if we're out of docs.
+ if (self->count >= self->doc_freq) {
+ Post_Reset(posting);
+ return 0;
+ }
+ self->count++;
+
+ Post_Read_Record(posting, post_stream);
+
+ return posting->doc_id;
+}
+
+int32_t
+SegPList_advance(SegPostingList *self, int32_t target) {
+ Posting *posting = self->posting;
+ const uint32_t skip_interval = self->skip_interval;
+
+ if (self->doc_freq >= skip_interval) {
+ InStream *post_stream = self->post_stream;
+ InStream *skip_stream = self->skip_stream;
+ SkipStepper *const skip_stepper = self->skip_stepper;
+ uint32_t new_doc_id = skip_stepper->doc_id;
+ int64_t new_filepos = InStream_Tell(post_stream);
+
+ /* Assuming the default skip_interval of 16...
+ *
+ * Say we're currently on the 5th doc matching this term, and we get a
+ * request to skip to the 18th doc matching it. We won't have skipped
+ * yet, but we'll have already gone past 5 of the 16 skip docs --
+ * ergo, the modulus in the following formula.
+ */
+ int32_t num_skipped = 0 - (self->count % skip_interval);
+ if (num_skipped == 0 && self->count != 0) {
+ num_skipped = 0 - skip_interval;
+ }
+
+ // See if there's anything to skip.
+ while (target > skip_stepper->doc_id) {
+ new_doc_id = skip_stepper->doc_id;
+ new_filepos = skip_stepper->filepos;
+
+ if (skip_stepper->doc_id != 0
+ && skip_stepper->doc_id >= posting->doc_id
+ ) {
+ num_skipped += skip_interval;
+ }
+
+ if (self->skip_count >= self->num_skips) {
+ break;
+ }
+
+ SkipStepper_Read_Record(skip_stepper, skip_stream);
+ self->skip_count++;
+ }
+
+ // If we found something to skip, skip it.
+ if (new_filepos > InStream_Tell(post_stream)) {
+
+ // Move the postings filepointer up.
+ InStream_Seek(post_stream, new_filepos);
+
+ // Jump to the new doc id.
+ posting->doc_id = new_doc_id;
+
+ // Increase count by the number of docs we skipped over.
+ self->count += num_skipped;
+ }
+ }
+
+ // Done skipping, so scan.
+ while (1) {
+ int32_t doc_id = SegPList_Next(self);
+ if (doc_id == 0 || doc_id >= target) {
+ return doc_id;
+ }
+ }
+}
+
+void
+SegPList_seek(SegPostingList *self, Obj *target) {
+ LexiconReader *lex_reader = PListReader_Get_Lex_Reader(self->plist_reader);
+ TermInfo *tinfo = LexReader_Fetch_Term_Info(lex_reader,
+ self->field, target);
+ S_seek_tinfo(self, tinfo);
+ DECREF(tinfo);
+}
+
+void
+SegPList_seek_lex(SegPostingList *self, Lexicon *lexicon) {
+ // Maybe true, maybe not.
+ SegLexicon *const seg_lexicon = (SegLexicon*)lexicon;
+
+ // Optimized case.
+ if (Obj_Is_A((Obj*)lexicon, SEGLEXICON)
+ && (SegLex_Get_Segment(seg_lexicon)
+ == PListReader_Get_Segment(self->plist_reader)) // i.e. same segment
+ ) {
+ S_seek_tinfo(self, SegLex_Get_Term_Info(seg_lexicon));
+ }
+ // Punt case. This is more expensive because of the call to
+ // LexReader_Fetch_Term_Info() in Seek().
+ else {
+ Obj *term = Lex_Get_Term(lexicon);
+ SegPList_Seek(self, term);
+ }
+}
+
+static void
+S_seek_tinfo(SegPostingList *self, TermInfo *tinfo) {
+ self->count = 0;
+
+ if (tinfo == NULL) {
+ // Next will return false; other methods invalid now.
+ self->doc_freq = 0;
+ }
+ else {
+ // Transfer doc_freq, seek main stream.
+ int64_t post_filepos = TInfo_Get_Post_FilePos(tinfo);
+ self->doc_freq = TInfo_Get_Doc_Freq(tinfo);
+ InStream_Seek(self->post_stream, post_filepos);
+
+ // Prepare posting.
+ Post_Reset(self->posting);
+
+ // Prepare to skip.
+ self->skip_count = 0;
+ self->num_skips = self->doc_freq / self->skip_interval;
+ SkipStepper_Set_ID_And_Filepos(self->skip_stepper, 0, post_filepos);
+ InStream_Seek(self->skip_stream, TInfo_Get_Skip_FilePos(tinfo));
+ }
+}
+
+Matcher*
+SegPList_make_matcher(SegPostingList *self, Similarity *sim,
+ Compiler *compiler, bool_t need_score) {
+ return Post_Make_Matcher(self->posting, sim, (PostingList*)self, compiler,
+ need_score);
+}
+
+RawPosting*
+SegPList_read_raw(SegPostingList *self, int32_t last_doc_id, CharBuf *term_text,
+ MemoryPool *mem_pool) {
+ return Post_Read_Raw(self->posting, self->post_stream,
+ last_doc_id, term_text, mem_pool);
+}
+
+
+
diff --git a/core/Lucy/Index/SegPostingList.cfh b/core/Lucy/Index/SegPostingList.cfh
new file mode 100644
index 0000000..6b45108
--- /dev/null
+++ b/core/Lucy/Index/SegPostingList.cfh
@@ -0,0 +1,86 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Single-segment PostingList.
+ */
+
+class Lucy::Index::SegPostingList cnick SegPList
+ inherits Lucy::Index::PostingList {
+
+ PostingListReader *plist_reader;
+ CharBuf *field;
+ Posting *posting;
+ InStream *post_stream;
+ InStream *skip_stream;
+ SkipStepper *skip_stepper;
+ int32_t skip_interval;
+ uint32_t count;
+ uint32_t doc_freq;
+ uint32_t skip_count;
+ uint32_t num_skips;
+ int32_t field_num;
+
+ inert incremented SegPostingList*
+ new(PostingListReader *plist_reader, const CharBuf *field);
+
+ inert SegPostingList*
+ init(SegPostingList *self, PostingListReader *plist_reader,
+ const CharBuf *field);
+
+ InStream*
+ Get_Post_Stream(SegPostingList *self);
+
+ uint32_t
+ Get_Count(SegPostingList *self);
+
+ public void
+ Destroy(SegPostingList *self);
+
+ public uint32_t
+ Get_Doc_Freq(SegPostingList *self);
+
+ public int32_t
+ Get_Doc_ID(SegPostingList *self);
+
+ Posting*
+ Get_Posting(SegPostingList *self);
+
+ public int32_t
+ Next(SegPostingList *self);
+
+ public int32_t
+ Advance(SegPostingList *self, int32_t target);
+
+ public void
+ Seek(SegPostingList *self, Obj *target = NULL);
+
+ /** Optimized version of Seek(), designed to speed sequential access.
+ */
+ void
+ Seek_Lex(SegPostingList *self, Lexicon *lexicon);
+
+ Matcher*
+ Make_Matcher(SegPostingList *self, Similarity *similarity,
+ Compiler *compiler, bool_t need_score);
+
+ RawPosting*
+ Read_Raw(SegPostingList *self, int32_t last_doc_id, CharBuf *term_text,
+ MemoryPool *mem_pool);
+}
+
+
diff --git a/core/Lucy/Index/SegReader.c b/core/Lucy/Index/SegReader.c
new file mode 100644
index 0000000..2aa4fee
--- /dev/null
+++ b/core/Lucy/Index/SegReader.c
@@ -0,0 +1,122 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SEGREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/DocReader.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Store/Folder.h"
+
+SegReader*
+SegReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick) {
+ SegReader *self = (SegReader*)VTable_Make_Obj(SEGREADER);
+ return SegReader_init(self, schema, folder, snapshot, segments, seg_tick);
+}
+
+SegReader*
+SegReader_init(SegReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ CharBuf *mess;
+ Segment *segment;
+
+ IxReader_init((IndexReader*)self, schema, folder, snapshot, segments,
+ seg_tick, NULL);
+ segment = SegReader_Get_Segment(self);
+
+ self->doc_max = (int32_t)Seg_Get_Count(segment);
+ self->seg_name = (CharBuf*)INCREF(Seg_Get_Name(segment));
+ self->seg_num = Seg_Get_Number(segment);
+ mess = SegReader_Try_Init_Components(self);
+ if (mess) {
+ // An error occurred, so clean up self and throw an exception.
+ DECREF(self);
+ Err_throw_mess(ERR, mess);
+ }
+ {
+ DeletionsReader *del_reader
+ = (DeletionsReader*)Hash_Fetch(
+ self->components, (Obj*)VTable_Get_Name(DELETIONSREADER));
+ self->del_count = del_reader ? DelReader_Del_Count(del_reader) : 0;
+ }
+ return self;
+}
+
+void
+SegReader_destroy(SegReader *self) {
+ DECREF(self->seg_name);
+ SUPER_DESTROY(self, SEGREADER);
+}
+
+void
+SegReader_register(SegReader *self, const CharBuf *api,
+ DataReader *component) {
+ if (Hash_Fetch(self->components, (Obj*)api)) {
+ THROW(ERR, "Interface '%o' already registered");
+ }
+ CERTIFY(component, DATAREADER);
+ Hash_Store(self->components, (Obj*)api, (Obj*)component);
+}
+
+CharBuf*
+SegReader_get_seg_name(SegReader *self) {
+ return self->seg_name;
+}
+
+int64_t
+SegReader_get_seg_num(SegReader *self) {
+ return self->seg_num;
+}
+
+int32_t
+SegReader_del_count(SegReader *self) {
+ return self->del_count;
+}
+
+int32_t
+SegReader_doc_max(SegReader *self) {
+ return self->doc_max;
+}
+
+int32_t
+SegReader_doc_count(SegReader *self) {
+ return self->doc_max - self->del_count;
+}
+
+I32Array*
+SegReader_offsets(SegReader *self) {
+ int32_t *ints = (int32_t*)CALLOCATE(1, sizeof(int32_t));
+ UNUSED_VAR(self);
+ return I32Arr_new_steal(ints, 1);
+}
+
+VArray*
+SegReader_seg_readers(SegReader *self) {
+ VArray *seg_readers = VA_new(1);
+ VA_Push(seg_readers, INCREF(self));
+ return seg_readers;
+}
+
+
diff --git a/core/Lucy/Index/SegReader.cfh b/core/Lucy/Index/SegReader.cfh
new file mode 100644
index 0000000..5a0fef9
--- /dev/null
+++ b/core/Lucy/Index/SegReader.cfh
@@ -0,0 +1,103 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Single-segment IndexReader.
+ *
+ * SegReader interprets the data within a single segment of an index.
+ *
+ * Generally speaking, only advanced users writing subclasses which manipulate
+ * data at the segment level need to deal with the SegReader API directly.
+ *
+ * Nearly all of SegReader's functionality is implemented by pluggable
+ * components spawned by L<Architecture|Lucy::Plan::Architecture>'s
+ * factory methods.
+ */
+
+class Lucy::Index::SegReader inherits Lucy::Index::IndexReader {
+
+ int32_t doc_max;
+ int32_t del_count;
+ int64_t seg_num;
+ CharBuf *seg_name;
+
+ inert incremented SegReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot = NULL,
+ VArray *segments, int32_t seg_tick);
+
+ /**
+ * @param schema A Schema.
+ * @param folder A Folder.
+ * @param snapshot A Snapshot, which must contain the files needed by the
+ * Segment.
+ * @param segments An array of Segment objects.
+ * @param seg_tick The array index of the Segment object within
+ * <code>segments</code> that this particular SegReader is assigned to.
+ */
+ inert SegReader*
+ init(SegReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot = NULL, VArray *segments, int32_t seg_tick);
+
+ public void
+ Destroy(SegReader *self);
+
+ /** Constructor helper.
+ *
+ * @return either NULL indicating success, or a CharBuf with an error
+ * message.
+ */
+ incremented CharBuf*
+ Try_Init_Components(SegReader *self);
+
+ /** Add a component to the SegReader. Using the same <code>api</code> key
+ * twice is an error.
+ *
+ * @param api The name of the DataReader subclass that defines the
+ * interface implemented by <code>component</code>.
+ * @param component A DataReader.
+ */
+ public void
+ Register(SegReader *self, const CharBuf *api,
+ decremented DataReader *component);
+
+ /** Return the name of the segment.
+ */
+ public CharBuf*
+ Get_Seg_Name(SegReader *self);
+
+ /** Return the number of the segment.
+ */
+ public int64_t
+ Get_Seg_Num(SegReader *self);
+
+ public int32_t
+ Del_Count(SegReader *self);
+
+ public int32_t
+ Doc_Max(SegReader *self);
+
+ public int32_t
+ Doc_Count(SegReader *self);
+
+ public incremented I32Array*
+ Offsets(SegReader *self);
+
+ public incremented VArray*
+ Seg_Readers(SegReader *self);
+}
+
+
diff --git a/core/Lucy/Index/SegWriter.c b/core/Lucy/Index/SegWriter.c
new file mode 100644
index 0000000..e444260
--- /dev/null
+++ b/core/Lucy/Index/SegWriter.c
@@ -0,0 +1,228 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SEGWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SegWriter.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Plan/Architecture.h"
+
+SegWriter*
+SegWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ SegWriter *self = (SegWriter*)VTable_Make_Obj(SEGWRITER);
+ return SegWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+SegWriter*
+SegWriter_init(SegWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ Architecture *arch = Schema_Get_Architecture(schema);
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+ self->by_api = Hash_new(0);
+ self->inverter = Inverter_new(schema, segment);
+ self->writers = VA_new(16);
+ Arch_Init_Seg_Writer(arch, self);
+ return self;
+}
+
+void
+SegWriter_destroy(SegWriter *self) {
+ DECREF(self->inverter);
+ DECREF(self->writers);
+ DECREF(self->by_api);
+ DECREF(self->del_writer);
+ SUPER_DESTROY(self, SEGWRITER);
+}
+
+void
+SegWriter_register(SegWriter *self, const CharBuf *api,
+ DataWriter *component) {
+ CERTIFY(component, DATAWRITER);
+ if (Hash_Fetch(self->by_api, (Obj*)api)) {
+ THROW(ERR, "API %o already registered", api);
+ }
+ Hash_Store(self->by_api, (Obj*)api, (Obj*)component);
+}
+
+Obj*
+SegWriter_fetch(SegWriter *self, const CharBuf *api) {
+ return Hash_Fetch(self->by_api, (Obj*)api);
+}
+
+void
+SegWriter_add_writer(SegWriter *self, DataWriter *writer) {
+ VA_Push(self->writers, (Obj*)writer);
+}
+
+void
+SegWriter_prep_seg_dir(SegWriter *self) {
+ Folder *folder = SegWriter_Get_Folder(self);
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+
+ // Clear stale segment files from crashed indexing sessions.
+ if (Folder_Exists(folder, seg_name)) {
+ bool_t result = Folder_Delete_Tree(folder, seg_name);
+ if (!result) {
+ THROW(ERR, "Couldn't completely remove '%o'", seg_name);
+ }
+ }
+
+ {
+ // Create the segment directory.
+ bool_t result = Folder_MkDir(folder, seg_name);
+ if (!result) { RETHROW(INCREF(Err_get_error())); }
+ }
+}
+
+void
+SegWriter_add_doc(SegWriter *self, Doc *doc, float boost) {
+ int32_t doc_id = (int32_t)Seg_Increment_Count(self->segment, 1);
+ Inverter_Invert_Doc(self->inverter, doc);
+ Inverter_Set_Boost(self->inverter, boost);
+ SegWriter_Add_Inverted_Doc(self, self->inverter, doc_id);
+}
+
+void
+SegWriter_add_inverted_doc(SegWriter *self, Inverter *inverter,
+ int32_t doc_id) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->writers); i < max; i++) {
+ DataWriter *writer = (DataWriter*)VA_Fetch(self->writers, i);
+ DataWriter_Add_Inverted_Doc(writer, inverter, doc_id);
+ }
+}
+
+// Adjust current doc id. We create our own doc_count rather than rely on
+// SegReader's number because the DeletionsWriter and the SegReader are
+// probably out of sync.
+static void
+S_adjust_doc_id(SegWriter *self, SegReader *reader, I32Array *doc_map) {
+ uint32_t doc_count = SegReader_Doc_Max(reader);
+ uint32_t i, max;
+ for (i = 1, max = I32Arr_Get_Size(doc_map); i < max; i++) {
+ if (I32Arr_Get(doc_map, i) == 0) { doc_count--; }
+ }
+ Seg_Increment_Count(self->segment, doc_count);
+}
+
+void
+SegWriter_add_segment(SegWriter *self, SegReader *reader, I32Array *doc_map) {
+ uint32_t i, max;
+
+ // Bulk add the slab of documents to the various writers.
+ for (i = 0, max = VA_Get_Size(self->writers); i < max; i++) {
+ DataWriter *writer = (DataWriter*)VA_Fetch(self->writers, i);
+ DataWriter_Add_Segment(writer, reader, doc_map);
+ }
+
+ // Bulk add the segment to the DeletionsWriter, so that it can merge
+ // previous segment files as necessary.
+ DelWriter_Add_Segment(self->del_writer, reader, doc_map);
+
+ // Adust the document id.
+ S_adjust_doc_id(self, reader, doc_map);
+}
+
+void
+SegWriter_merge_segment(SegWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ Snapshot *snapshot = SegWriter_Get_Snapshot(self);
+ CharBuf *seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
+ uint32_t i, max;
+
+ // Have all the sub-writers merge the segment.
+ for (i = 0, max = VA_Get_Size(self->writers); i < max; i++) {
+ DataWriter *writer = (DataWriter*)VA_Fetch(self->writers, i);
+ DataWriter_Merge_Segment(writer, reader, doc_map);
+ }
+ DelWriter_Merge_Segment(self->del_writer, reader, doc_map);
+
+ // Remove seg directory from snapshot.
+ Snapshot_Delete_Entry(snapshot, seg_name);
+
+ // Adust the document id.
+ S_adjust_doc_id(self, reader, doc_map);
+}
+
+void
+SegWriter_delete_segment(SegWriter *self, SegReader *reader) {
+ Snapshot *snapshot = SegWriter_Get_Snapshot(self);
+ CharBuf *seg_name = Seg_Get_Name(SegReader_Get_Segment(reader));
+ uint32_t i, max;
+
+ // Have all the sub-writers delete the segment.
+ for (i = 0, max = VA_Get_Size(self->writers); i < max; i++) {
+ DataWriter *writer = (DataWriter*)VA_Fetch(self->writers, i);
+ DataWriter_Delete_Segment(writer, reader);
+ }
+ DelWriter_Delete_Segment(self->del_writer, reader);
+
+ // Remove seg directory from snapshot.
+ Snapshot_Delete_Entry(snapshot, seg_name);
+}
+
+void
+SegWriter_finish(SegWriter *self) {
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ uint32_t i, max;
+
+ // Finish off children.
+ for (i = 0, max = VA_Get_Size(self->writers); i < max; i++) {
+ DataWriter *writer = (DataWriter*)VA_Fetch(self->writers, i);
+ DataWriter_Finish(writer);
+ }
+
+ // Write segment metadata and add the segment directory to the snapshot.
+ {
+ Snapshot *snapshot = SegWriter_Get_Snapshot(self);
+ CharBuf *segmeta_filename = CB_newf("%o/segmeta.json", seg_name);
+ Seg_Write_File(self->segment, self->folder);
+ Snapshot_Add_Entry(snapshot, seg_name);
+ DECREF(segmeta_filename);
+ }
+
+ // Collapse segment files into compound file.
+ Folder_Consolidate(self->folder, seg_name);
+}
+
+void
+SegWriter_add_data_writer(SegWriter *self, DataWriter *writer) {
+ VA_Push(self->writers, (Obj*)writer);
+}
+
+void
+SegWriter_set_del_writer(SegWriter *self, DeletionsWriter *del_writer) {
+ DECREF(self->del_writer);
+ self->del_writer = (DeletionsWriter*)INCREF(del_writer);
+}
+
+DeletionsWriter*
+SegWriter_get_del_writer(SegWriter *self) {
+ return self->del_writer;
+}
+
+
diff --git a/core/Lucy/Index/SegWriter.cfh b/core/Lucy/Index/SegWriter.cfh
new file mode 100644
index 0000000..cc42feb
--- /dev/null
+++ b/core/Lucy/Index/SegWriter.cfh
@@ -0,0 +1,127 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+
+/** Write one segment of an index.
+ *
+ * SegWriter is a conduit through which information fed to Indexer passes. It
+ * manages L<Segment|Lucy::Index::Segment> and Inverter, invokes the
+ * L<Analyzer|Lucy::Analysis::Analyzer> chain, and feeds low
+ * level L<DataWriters|Lucy::Index::DataWriter> such as
+ * PostingListWriter and DocWriter.
+ *
+ * The sub-components of a SegWriter are determined by
+ * L<Architecture|Lucy::Plan::Architecture>. DataWriter components
+ * which are added to the stack of writers via Add_Writer() have
+ * Add_Inverted_Doc() invoked for each document supplied to SegWriter's
+ * Add_Doc().
+ */
+class Lucy::Index::SegWriter inherits Lucy::Index::DataWriter {
+
+ Inverter *inverter;
+ VArray *writers;
+ Hash *by_api;
+ DeletionsWriter *del_writer;
+
+ inert incremented SegWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ /**
+ * @param schema A Schema.
+ * @param snapshot The Snapshot that will be committed at the end of the
+ * indexing session.
+ * @param segment The Segment in progress.
+ * @param polyreader A PolyReader representing all existing data in the
+ * index. (If the index is brand new, the PolyReader will have no
+ * sub-readers).
+ */
+ inert SegWriter*
+ init(SegWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ /**
+ * Register a DataWriter component with the SegWriter. (Note that
+ * registration simply makes the writer available via Fetch(), so you may
+ * also want to call Add_Writer()).
+ *
+ * @param api The name of the DataWriter api which <code>writer</code>
+ * implements.
+ * @param component A DataWriter.
+ */
+ public void
+ Register(SegWriter *self, const CharBuf *api,
+ decremented DataWriter *component);
+
+ /** Retrieve a registered component.
+ *
+ * @param api The name of the DataWriter api which the component
+ * implements.
+ */
+ public nullable Obj*
+ Fetch(SegWriter *self, const CharBuf *api);
+
+ /** Add a DataWriter to the SegWriter's stack of writers.
+ */
+ public void
+ Add_Writer(SegWriter *self, decremented DataWriter *writer);
+
+ /** Create the segment directory. If it already exists, delete any files
+ * within it (which are presumed to have been left over from a crashed
+ * indexing session).
+ */
+ void
+ Prep_Seg_Dir(SegWriter *self);
+
+ /** Add a document to the segment. Inverts <code>doc</code>, increments
+ * the Segment's internal document id, then calls Add_Inverted_Doc(),
+ * feeding all sub-writers.
+ */
+ public void
+ Add_Doc(SegWriter *self, Doc *doc, float boost = 1.0);
+
+ void
+ Set_Del_Writer(SegWriter *self, DeletionsWriter *del_writer = NULL);
+
+ /** Accessor for DeletionsWriter member.
+ */
+ DeletionsWriter*
+ Get_Del_Writer(SegWriter *self);
+
+ public void
+ Add_Inverted_Doc(SegWriter *self, Inverter *inverter, int32_t doc_id);
+
+ public void
+ Add_Segment(SegWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Merge_Segment(SegWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public void
+ Delete_Segment(SegWriter *self, SegReader *reader);
+
+ public void
+ Finish(SegWriter *self);
+
+ public void
+ Destroy(SegWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/Segment.c b/core/Lucy/Index/Segment.c
new file mode 100644
index 0000000..d14edd1
--- /dev/null
+++ b/core/Lucy/Index/Segment.c
@@ -0,0 +1,255 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <ctype.h>
+
+#define C_LUCY_SEGMENT
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/StringHelper.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+Segment*
+Seg_new(int64_t number) {
+ Segment *self = (Segment*)VTable_Make_Obj(SEGMENT);
+ return Seg_init(self, number);
+}
+
+Segment*
+Seg_init(Segment *self, int64_t number) {
+ // Validate.
+ if (number < 0) { THROW(ERR, "Segment number %i64 less than 0", number); }
+
+ // Init.
+ self->metadata = Hash_new(0);
+ self->count = 0;
+ self->by_num = VA_new(2);
+ self->by_name = Hash_new(0);
+
+ // Start field numbers at 1, not 0.
+ VA_Push(self->by_num, INCREF(&EMPTY));
+
+ // Assign.
+ self->number = number;
+
+ // Derive.
+ self->name = Seg_num_to_name(number);
+
+ return self;
+}
+
+CharBuf*
+Seg_num_to_name(int64_t number) {
+ char base36[StrHelp_MAX_BASE36_BYTES];
+ StrHelp_to_base36(number, &base36);
+ return CB_newf("seg_%s", &base36);
+}
+
+bool_t
+Seg_valid_seg_name(const CharBuf *name) {
+ if (CB_Starts_With_Str(name, "seg_", 4)) {
+ ZombieCharBuf *scratch = ZCB_WRAP(name);
+ ZCB_Nip(scratch, 4);
+ uint32_t code_point;
+ while (0 != (code_point = ZCB_Nip_One(scratch))) {
+ if (!isalnum(code_point)) { return false; }
+ }
+ if (ZCB_Get_Size(scratch) == 0) { return true; } // Success!
+ }
+ return false;
+}
+
+void
+Seg_destroy(Segment *self) {
+ DECREF(self->name);
+ DECREF(self->metadata);
+ DECREF(self->by_name);
+ DECREF(self->by_num);
+ SUPER_DESTROY(self, SEGMENT);
+}
+
+bool_t
+Seg_read_file(Segment *self, Folder *folder) {
+ CharBuf *filename = CB_newf("%o/segmeta.json", self->name);
+ Hash *metadata = (Hash*)Json_slurp_json(folder, filename);
+ Hash *my_metadata;
+
+ // Bail unless the segmeta file was read successfully.
+ DECREF(filename);
+ if (!metadata) { return false; }
+ CERTIFY(metadata, HASH);
+
+ // Grab metadata for the Segment object itself.
+ DECREF(self->metadata);
+ self->metadata = metadata;
+ my_metadata
+ = (Hash*)CERTIFY(Hash_Fetch_Str(self->metadata, "segmeta", 7), HASH);
+
+ // Assign.
+ {
+ Obj *count = Hash_Fetch_Str(my_metadata, "count", 5);
+ if (!count) { count = Hash_Fetch_Str(my_metadata, "doc_count", 9); }
+ if (!count) { THROW(ERR, "Missing 'count'"); }
+ else { self->count = Obj_To_I64(count); }
+ }
+
+ // Get list of field nums.
+ {
+ uint32_t i;
+ VArray *source_by_num = (VArray*)Hash_Fetch_Str(my_metadata,
+ "field_names", 11);
+ uint32_t num_fields = source_by_num ? VA_Get_Size(source_by_num) : 0;
+ if (source_by_num == NULL) {
+ THROW(ERR, "Failed to extract 'field_names' from metadata");
+ }
+
+ // Init.
+ DECREF(self->by_num);
+ DECREF(self->by_name);
+ self->by_num = VA_new(num_fields);
+ self->by_name = Hash_new(num_fields);
+
+ // Copy the list of fields from the source.
+ for (i = 0; i < num_fields; i++) {
+ CharBuf *name = (CharBuf*)VA_Fetch(source_by_num, i);
+ Seg_Add_Field(self, name);
+ }
+ }
+
+ return true;
+}
+
+void
+Seg_write_file(Segment *self, Folder *folder) {
+ Hash *my_metadata = Hash_new(16);
+
+ // Store metadata specific to this Segment object.
+ Hash_Store_Str(my_metadata, "count", 5,
+ (Obj*)CB_newf("%i64", self->count));
+ Hash_Store_Str(my_metadata, "name", 4, (Obj*)CB_Clone(self->name));
+ Hash_Store_Str(my_metadata, "field_names", 11, INCREF(self->by_num));
+ Hash_Store_Str(my_metadata, "format", 6, (Obj*)CB_newf("%i32", 1));
+ Hash_Store_Str(self->metadata, "segmeta", 7, (Obj*)my_metadata);
+
+ {
+ CharBuf *filename = CB_newf("%o/segmeta.json", self->name);
+ bool_t result
+ = Json_spew_json((Obj*)self->metadata, folder, filename);
+ DECREF(filename);
+ if (!result) { RETHROW(INCREF(Err_get_error())); }
+ }
+}
+
+int32_t
+Seg_add_field(Segment *self, const CharBuf *field) {
+ Integer32 *num = (Integer32*)Hash_Fetch(self->by_name, (Obj*)field);
+ if (num) {
+ return Int32_Get_Value(num);
+ }
+ else {
+ int32_t field_num = VA_Get_Size(self->by_num);
+ Hash_Store(self->by_name, (Obj*)field, (Obj*)Int32_new(field_num));
+ VA_Push(self->by_num, (Obj*)CB_Clone(field));
+ return field_num;
+ }
+}
+
+CharBuf*
+Seg_get_name(Segment *self) {
+ return self->name;
+}
+
+int64_t
+Seg_get_number(Segment *self) {
+ return self->number;
+}
+
+void
+Seg_set_count(Segment *self, int64_t count) {
+ self->count = count;
+}
+
+int64_t
+Seg_get_count(Segment *self) {
+ return self->count;
+}
+
+int64_t
+Seg_increment_count(Segment *self, int64_t increment) {
+ self->count += increment;
+ return self->count;
+}
+
+void
+Seg_store_metadata(Segment *self, const CharBuf *key, Obj *value) {
+ if (Hash_Fetch(self->metadata, (Obj*)key)) {
+ THROW(ERR, "Metadata key '%o' already registered", key);
+ }
+ Hash_Store(self->metadata, (Obj*)key, value);
+}
+
+void
+Seg_store_metadata_str(Segment *self, const char *key, size_t key_len,
+ Obj *value) {
+ ZombieCharBuf *k = ZCB_WRAP_STR((char*)key, key_len);
+ Seg_Store_Metadata(self, (CharBuf*)k, value);
+}
+
+Obj*
+Seg_fetch_metadata(Segment *self, const CharBuf *key) {
+ return Hash_Fetch(self->metadata, (Obj*)key);
+}
+
+Obj*
+Seg_fetch_metadata_str(Segment *self, const char *key, size_t len) {
+ return Hash_Fetch_Str(self->metadata, key, len);
+}
+
+Hash*
+Seg_get_metadata(Segment *self) {
+ return self->metadata;
+}
+
+int32_t
+Seg_compare_to(Segment *self, Obj *other) {
+ Segment *other_seg = (Segment*)CERTIFY(other, SEGMENT);
+ if (self->number < other_seg->number) { return -1; }
+ else if (self->number == other_seg->number) { return 0; }
+ else { return 1; }
+}
+
+CharBuf*
+Seg_field_name(Segment *self, int32_t field_num) {
+ return field_num
+ ? (CharBuf*)VA_Fetch(self->by_num, field_num)
+ : NULL;
+}
+
+int32_t
+Seg_field_num(Segment *self, const CharBuf *field) {
+ if (field == NULL) {
+ return 0;
+ }
+ else {
+ Integer32 *num = (Integer32*)Hash_Fetch(self->by_name, (Obj*)field);
+ return num ? Int32_Get_Value(num) : 0;
+ }
+}
+
+
diff --git a/core/Lucy/Index/Segment.cfh b/core/Lucy/Index/Segment.cfh
new file mode 100644
index 0000000..2ee7ace
--- /dev/null
+++ b/core/Lucy/Index/Segment.cfh
@@ -0,0 +1,159 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Warehouse for information about one segment of an inverted index.
+ *
+ * Apache Lucy's indexes are made up of individual "segments", each of which is
+ * is an independent inverted index. On the file system, each segment is a
+ * directory within the main index directory whose name starts with "seg_":
+ * "seg_2", "seg_5a", etc.
+ *
+ * Each Segment object keeps track of information about an index segment: its
+ * fields, document count, and so on. The Segment object itself writes one
+ * file, <code>segmeta.json</code>; besides storing info needed by Segment
+ * itself, the "segmeta" file serves as a central repository for metadata
+ * generated by other index components -- relieving them of the burden of
+ * storing metadata themselves.
+ */
+
+class Lucy::Index::Segment cnick Seg inherits Lucy::Object::Obj {
+
+ CharBuf *name;
+ int64_t count;
+ int64_t number;
+ Hash *by_name; /* field numbers by name */
+ VArray *by_num; /* field names by num */
+ Hash *metadata;
+
+ inert incremented Segment*
+ new(int64_t number);
+
+ public inert Segment*
+ init(Segment *self, int64_t number);
+
+ /** Return a segment name with a base-36-encoded segment number.
+ */
+ inert incremented CharBuf*
+ num_to_name(int64_t number);
+
+ /** Return true if the CharBuf is a segment name, i.e. matches this
+ * pattern: /^seg_\w+$/
+ */
+ inert bool_t
+ valid_seg_name(const CharBuf *name);
+
+ /** Register a new field and assign it a field number. If the field was
+ * already known, nothing happens.
+ *
+ * @param field Field name.
+ * @return the field's field number, which is a positive integer.
+ */
+ public int32_t
+ Add_Field(Segment *self, const CharBuf *field);
+
+ /** Store arbitrary information in the segment's metadata Hash, to be
+ * serialized later. Throws an error if <code>key</code> is used twice.
+ *
+ * @param key String identifying an index component.
+ * @param metadata JSON-izable data structure.
+ */
+ public void
+ Store_Metadata(Segment *self, const CharBuf *key,
+ decremented Obj *metadata);
+
+ void
+ Store_Metadata_Str(Segment *self, const char *key, size_t len,
+ decremented Obj *value);
+
+ /** Fetch a value from the Segment's metadata hash.
+ */
+ public nullable Obj*
+ Fetch_Metadata(Segment *self, const CharBuf *key);
+
+ nullable Obj*
+ Fetch_Metadata_Str(Segment *self, const char *key, size_t len);
+
+ /** Given a field name, return its field number for this segment (which
+ * may differ from its number in other segments). Return 0 (an invalid
+ * field number) if the field name can't be found.
+ *
+ * @param field Field name.
+ */
+ public int32_t
+ Field_Num(Segment *self, const CharBuf *field);
+
+ /** Given a field number, return the name of its field, or NULL if the
+ * field name can't be found.
+ */
+ public nullable CharBuf*
+ Field_Name(Segment *self, int32_t field_num);
+
+ /** Getter for the object's seg name.
+ */
+ public CharBuf*
+ Get_Name(Segment *self);
+
+ /** Getter for the segment number.
+ */
+ public int64_t
+ Get_Number(Segment *self);
+
+ /** Setter for the object's document count.
+ */
+ public void
+ Set_Count(Segment *self, int64_t count);
+
+ /** Getter for the object's document count.
+ */
+ public int64_t
+ Get_Count(Segment *self);
+
+ /** Add <code>increment</code> to the object's document count, then return
+ * the new, modified total.
+ */
+ int64_t
+ Increment_Count(Segment *self, int64_t increment);
+
+ /** Get the segment metadata.
+ */
+ Hash*
+ Get_Metadata(Segment *self);
+
+ /** Write the segdata file.
+ */
+ public void
+ Write_File(Segment *self, Folder *folder);
+
+ /** Read the segmeta file for this segment.
+ *
+ * @return true if the file is read and decoded successfully, false
+ * otherwise.
+ */
+ public bool_t
+ Read_File(Segment *self, Folder *folder);
+
+ /** Compare by segment number.
+ */
+ public int32_t
+ Compare_To(Segment *self, Obj *other);
+
+ public void
+ Destroy(Segment *self);
+}
+
+
diff --git a/core/Lucy/Index/Similarity.c b/core/Lucy/Index/Similarity.c
new file mode 100644
index 0000000..bf7e7ef
--- /dev/null
+++ b/core/Lucy/Index/Similarity.c
@@ -0,0 +1,226 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SIMILARITY
+#include "Lucy/Util/ToolSet.h"
+
+#include "math.h"
+
+#include "Lucy/Index/Similarity.h"
+
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Posting/MatchPosting.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+Similarity*
+Sim_new() {
+ Similarity *self = (Similarity*)VTable_Make_Obj(SIMILARITY);
+ return Sim_init(self);
+}
+
+Similarity*
+Sim_init(Similarity *self) {
+ self->norm_decoder = NULL;
+ return self;
+}
+
+void
+Sim_destroy(Similarity *self) {
+ if (self->norm_decoder) {
+ FREEMEM(self->norm_decoder);
+ }
+ SUPER_DESTROY(self, SIMILARITY);
+}
+
+Posting*
+Sim_make_posting(Similarity *self) {
+ return (Posting*)ScorePost_new(self);
+}
+
+PostingWriter*
+Sim_make_posting_writer(Similarity *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader,
+ int32_t field_num) {
+ UNUSED_VAR(self);
+ return (PostingWriter*)MatchPostWriter_new(schema, snapshot, segment,
+ polyreader, field_num);
+}
+
+float*
+Sim_get_norm_decoder(Similarity *self) {
+ if (!self->norm_decoder) {
+ // Cache decoded boost bytes.
+ self->norm_decoder = (float*)MALLOCATE(256 * sizeof(float));
+ for (uint32_t i = 0; i < 256; i++) {
+ self->norm_decoder[i] = Sim_Decode_Norm(self, i);
+ }
+ }
+ return self->norm_decoder;
+}
+
+Obj*
+Sim_dump(Similarity *self) {
+ Hash *dump = Hash_new(0);
+ Hash_Store_Str(dump, "_class", 6,
+ (Obj*)CB_Clone(Sim_Get_Class_Name(self)));
+ return (Obj*)dump;
+}
+
+Similarity*
+Sim_load(Similarity *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *class_name
+ = (CharBuf*)CERTIFY(Hash_Fetch_Str(source, "_class", 6), CHARBUF);
+ VTable *vtable = VTable_singleton(class_name, NULL);
+ Similarity *loaded = (Similarity*)VTable_Make_Obj(vtable);
+ UNUSED_VAR(self);
+ return Sim_init(loaded);
+}
+
+void
+Sim_serialize(Similarity *self, OutStream *target) {
+ // Only the class name.
+ CB_Serialize(Sim_Get_Class_Name(self), target);
+}
+
+Similarity*
+Sim_deserialize(Similarity *self, InStream *instream) {
+ CharBuf *class_name = CB_deserialize(NULL, instream);
+ if (!self) {
+ VTable *vtable = VTable_singleton(class_name, SIMILARITY);
+ self = (Similarity*)VTable_Make_Obj(vtable);
+ }
+ else if (!CB_Equals(class_name, (Obj*)Sim_Get_Class_Name(self))) {
+ THROW(ERR, "Class name mismatch: '%o' '%o'", Sim_Get_Class_Name(self),
+ class_name);
+ }
+ DECREF(class_name);
+
+ Sim_init(self);
+ return self;
+}
+
+bool_t
+Sim_equals(Similarity *self, Obj *other) {
+ if (Sim_Get_VTable(self) != Obj_Get_VTable(other)) { return false; }
+ return true;
+}
+
+float
+Sim_idf(Similarity *self, int64_t doc_freq, int64_t total_docs) {
+ UNUSED_VAR(self);
+ if (total_docs == 0) {
+ // Guard against log of zero error, return meaningless number.
+ return 1;
+ }
+ else {
+ double total_documents = (double)total_docs;
+ double document_freq = (double)doc_freq;
+ return (float)(1 + log(total_documents / (1 + document_freq)));
+ }
+}
+
+float
+Sim_tf(Similarity *self, float freq) {
+ UNUSED_VAR(self);
+ return (float)sqrt(freq);
+}
+
+uint32_t
+Sim_encode_norm(Similarity *self, float f) {
+ uint32_t norm;
+ UNUSED_VAR(self);
+
+ if (f < 0.0) {
+ f = 0.0;
+ }
+
+ if (f == 0.0) {
+ norm = 0;
+ }
+ else {
+ const uint32_t bits = *(uint32_t*)&f;
+ uint32_t mantissa = (bits & 0xffffff) >> 21;
+ uint32_t exponent = (((bits >> 24) & 0x7f) - 63) + 15;
+
+ if (exponent > 31) {
+ exponent = 31;
+ mantissa = 7;
+ }
+
+ norm = (exponent << 3) | mantissa;
+ }
+
+ return norm;
+}
+
+float
+Sim_decode_norm(Similarity *self, uint32_t input) {
+ uint8_t byte = input & 0xFF;
+ uint32_t result;
+ UNUSED_VAR(self);
+
+ if (byte == 0) {
+ result = 0;
+ }
+ else {
+ const uint32_t mantissa = byte & 7;
+ const uint32_t exponent = (byte >> 3) & 31;
+ result = ((exponent + (63 - 15)) << 24) | (mantissa << 21);
+ }
+
+ return *(float*)&result;
+}
+
+float
+Sim_length_norm(Similarity *self, uint32_t num_tokens) {
+ UNUSED_VAR(self);
+ if (num_tokens == 0) { // guard against div by zero
+ return 0;
+ }
+ else {
+ return (float)(1.0 / sqrt((double)num_tokens));
+ }
+}
+
+float
+Sim_query_norm(Similarity *self, float sum_of_squared_weights) {
+ UNUSED_VAR(self);
+ if (sum_of_squared_weights == 0.0f) { // guard against div by zero
+ return 0;
+ }
+ else {
+ return (float)(1.0 / sqrt(sum_of_squared_weights));
+ }
+}
+
+float
+Sim_coord(Similarity *self, uint32_t overlap, uint32_t max_overlap) {
+ UNUSED_VAR(self);
+ if (max_overlap == 0) {
+ return 1;
+ }
+ else {
+ return (float)overlap / (float)max_overlap;
+ }
+}
+
+
diff --git a/core/Lucy/Index/Similarity.cfh b/core/Lucy/Index/Similarity.cfh
new file mode 100644
index 0000000..e49021c
--- /dev/null
+++ b/core/Lucy/Index/Similarity.cfh
@@ -0,0 +1,143 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Judge how well a document matches a query.
+ *
+ * After determining whether a document matches a given query, a score must be
+ * calculated which indicates how I<well> the document matches the query. The
+ * Similarity class is used to judge how "similar" the query and the document
+ * are to each other; the closer the resemblance, they higher the document
+ * scores.
+ *
+ * The default implementation uses Lucene's modified cosine similarity
+ * measure. Subclasses might tweak the existing algorithms, or might be used
+ * in conjunction with custom Query subclasses to implement arbitrary scoring
+ * schemes.
+ *
+ * Most of the methods operate on single fields, but some are used to combine
+ * scores from multiple fields.
+ */
+
+class Lucy::Index::Similarity cnick Sim
+ inherits Lucy::Object::Obj : dumpable {
+
+ float *norm_decoder;
+
+ inert incremented Similarity*
+ new();
+
+ /** Constructor. Takes no arguments.
+ */
+ public inert Similarity*
+ init(Similarity *self);
+
+ /** Factory method for creating a Posting.
+ */
+ public incremented Posting*
+ Make_Posting(Similarity *self);
+
+ /** Factory method for creating a PostingWriter.
+ */
+ incremented PostingWriter*
+ Make_Posting_Writer(Similarity *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader,
+ int32_t field_num);
+
+ /** Return a score factor based on the frequency of a term in a given
+ * document. The default implementation is sqrt(freq). Other
+ * implementations typically produce ascending scores with ascending
+ * freqs, since the more times a doc matches, the more relevant it is
+ * likely to be.
+ */
+ public float
+ TF(Similarity *self, float freq);
+
+ /** Calculate the Inverse Document Frequecy for a term in a given
+ * collection.
+ *
+ * @param doc_freq The number of documents that the term appears in.
+ * @param total_docs The number of documents in the collection.
+ */
+ public float
+ IDF(Similarity *self, int64_t doc_freq, int64_t total_docs);
+
+ /** Calculate a score factor based on the number of terms which match.
+ */
+ public float
+ Coord(Similarity *self, uint32_t overlap, uint32_t max_overlap);
+
+ /** Dampen the scores of long documents.
+ *
+ * After a field is broken up into terms at index-time, each term must be
+ * assigned a weight. One of the factors in calculating this weight is
+ * the number of tokens that the original field was broken into.
+ *
+ * Typically, we assume that the more tokens in a field, the less
+ * important any one of them is -- so that, e.g. 5 mentions of "Kafka" in
+ * a short article are given more heft than 5 mentions of "Kafka" in an
+ * entire book. The default implementation of length_norm expresses this
+ * using an inverted square root.
+ *
+ * However, the inverted square root has a tendency to reward very short
+ * fields highly, which isn't always appropriate for fields you expect to
+ * have a lot of tokens on average.
+ */
+ public float
+ Length_Norm(Similarity *self, uint32_t num_tokens);
+
+ /** Normalize a Query's weight so that it is comparable to other Queries.
+ */
+ public float
+ Query_Norm(Similarity *self, float sum_of_squared_weights);
+
+ /** encode_norm and decode_norm encode and decode between 32-bit IEEE
+ * floating point numbers and a 5-bit exponent, 3-bit mantissa float. The
+ * range covered by the single-byte encoding is 7x10^9 to 2x10^-9. The
+ * accuracy is about one significant decimal digit.
+ */
+ uint32_t
+ Encode_Norm(Similarity *self, float f);
+
+ /** See encode_norm.
+ */
+ float
+ Decode_Norm(Similarity *self, uint32_t input);
+
+ float*
+ Get_Norm_Decoder(Similarity *self);
+
+ public void
+ Destroy(Similarity *self);
+
+ public incremented Obj*
+ Dump(Similarity *self);
+
+ public incremented Similarity*
+ Load(Similarity *self, Obj *dump);
+
+ public bool_t
+ Equals(Similarity *self, Obj *other);
+
+ public void
+ Serialize(Similarity *self, OutStream *outstream);
+
+ public incremented Similarity*
+ Deserialize(Similarity *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Index/SkipStepper.c b/core/Lucy/Index/SkipStepper.c
new file mode 100644
index 0000000..0af6b29
--- /dev/null
+++ b/core/Lucy/Index/SkipStepper.c
@@ -0,0 +1,73 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SKIPSTEPPER
+#include <stdio.h>
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SkipStepper.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+SkipStepper*
+SkipStepper_new() {
+ SkipStepper *self = (SkipStepper*)VTable_Make_Obj(SKIPSTEPPER);
+
+ // Init.
+ self->doc_id = 0;
+ self->filepos = 0;
+
+ return self;
+}
+
+void
+SkipStepper_set_id_and_filepos(SkipStepper *self, int32_t doc_id,
+ int64_t filepos) {
+ self->doc_id = doc_id;
+ self->filepos = filepos;
+}
+
+void
+SkipStepper_read_record(SkipStepper *self, InStream *instream) {
+ self->doc_id += InStream_Read_C32(instream);
+ self->filepos += InStream_Read_C64(instream);
+}
+
+CharBuf*
+SkipStepper_to_string(SkipStepper *self) {
+ char *ptr = (char*)MALLOCATE(60);
+ size_t len = sprintf(ptr, "skip doc: %u file pointer: %" I64P,
+ self->doc_id, self->filepos);
+ return CB_new_steal_from_trusted_str(ptr, len, 60);
+}
+
+void
+SkipStepper_write_record(SkipStepper *self, OutStream *outstream,
+ int32_t last_doc_id, int64_t last_filepos) {
+ const int32_t delta_doc_id = self->doc_id - last_doc_id;
+ const int64_t delta_filepos = self->filepos - last_filepos;
+
+ // Write delta doc id.
+ OutStream_Write_C32(outstream, delta_doc_id);
+
+ // Write delta file pointer.
+ OutStream_Write_C64(outstream, delta_filepos);
+}
+
+
+
diff --git a/core/Lucy/Index/SkipStepper.cfh b/core/Lucy/Index/SkipStepper.cfh
new file mode 100644
index 0000000..6dceba9
--- /dev/null
+++ b/core/Lucy/Index/SkipStepper.cfh
@@ -0,0 +1,44 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::SkipStepper inherits Lucy::Util::Stepper {
+
+ int32_t doc_id;
+ int64_t filepos;
+
+ inert incremented SkipStepper*
+ new();
+
+ void
+ Read_Record(SkipStepper *self, InStream *instream);
+
+ void
+ Write_Record(SkipStepper *self, OutStream *outstream,
+ int32_t last_doc_id, int64_t last_filepos);
+
+ /** Set a base document id and a base file position which Read_Record
+ * will add onto with its deltas.
+ */
+ void
+ Set_ID_And_Filepos(SkipStepper *self, int32_t doc_id, int64_t filepos);
+
+ public incremented CharBuf*
+ To_String(SkipStepper *self);
+}
+
+
diff --git a/core/Lucy/Index/Snapshot.c b/core/Lucy/Index/Snapshot.c
new file mode 100644
index 0000000..f6d4742
--- /dev/null
+++ b/core/Lucy/Index/Snapshot.c
@@ -0,0 +1,204 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SNAPSHOT
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Util/StringHelper.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/Json.h"
+
+static VArray*
+S_clean_segment_contents(VArray *orig);
+
+int32_t Snapshot_current_file_format = 2;
+static int32_t Snapshot_current_file_subformat = 1;
+
+Snapshot*
+Snapshot_new() {
+ Snapshot *self = (Snapshot*)VTable_Make_Obj(SNAPSHOT);
+ return Snapshot_init(self);
+}
+
+static void
+S_zero_out(Snapshot *self) {
+ DECREF(self->entries);
+ DECREF(self->path);
+ self->entries = Hash_new(0);
+ self->path = NULL;
+}
+
+Snapshot*
+Snapshot_init(Snapshot *self) {
+ S_zero_out(self);
+ return self;
+}
+
+void
+Snapshot_destroy(Snapshot *self) {
+ DECREF(self->entries);
+ DECREF(self->path);
+ SUPER_DESTROY(self, SNAPSHOT);
+}
+
+void
+Snapshot_add_entry(Snapshot *self, const CharBuf *entry) {
+ Hash_Store(self->entries, (Obj*)entry, INCREF(&EMPTY));
+}
+
+bool_t
+Snapshot_delete_entry(Snapshot *self, const CharBuf *entry) {
+ Obj *val = Hash_Delete(self->entries, (Obj*)entry);
+ if (val) {
+ Obj_Dec_RefCount(val);
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+VArray*
+Snapshot_list(Snapshot *self) {
+ return Hash_Keys(self->entries);
+}
+
+uint32_t
+Snapshot_num_entries(Snapshot *self) {
+ return Hash_Get_Size(self->entries);
+}
+
+void
+Snapshot_set_path(Snapshot *self, const CharBuf *path) {
+ DECREF(self->path);
+ self->path = path ? CB_Clone(path) : NULL;
+}
+
+CharBuf*
+Snapshot_get_path(Snapshot *self) {
+ return self->path;
+}
+
+Snapshot*
+Snapshot_read_file(Snapshot *self, Folder *folder, const CharBuf *path) {
+ // Eliminate all prior data. Pick a snapshot file.
+ S_zero_out(self);
+ self->path = path ? CB_Clone(path) : IxFileNames_latest_snapshot(folder);
+
+ if (self->path) {
+ Hash *snap_data
+ = (Hash*)CERTIFY(Json_slurp_json(folder, self->path), HASH);
+ Obj *format_obj
+ = CERTIFY(Hash_Fetch_Str(snap_data, "format", 6), OBJ);
+ int32_t format = (int32_t)Obj_To_I64(format_obj);
+ Obj *subformat_obj = Hash_Fetch_Str(snap_data, "subformat", 9);
+ int32_t subformat = subformat_obj
+ ? (int32_t)Obj_To_I64(subformat_obj)
+ : 0;
+
+ // Verify that we can read the index properly.
+ if (format > Snapshot_current_file_format) {
+ THROW(ERR, "Snapshot format too recent: %i32, %i32",
+ format, Snapshot_current_file_format);
+ }
+
+ // Build up list of entries.
+ VArray *list = (VArray*)CERTIFY(
+ Hash_Fetch_Str(snap_data, "entries", 7),
+ VARRAY);
+ INCREF(list);
+ if (format == 1 || (format == 2 && subformat < 1)) {
+ VArray *cleaned = S_clean_segment_contents(list);
+ DECREF(list);
+ list = cleaned;
+ }
+ Hash_Clear(self->entries);
+ for (uint32_t i = 0, max = VA_Get_Size(list); i < max; i++) {
+ CharBuf *entry
+ = (CharBuf*)CERTIFY(VA_Fetch(list, i), CHARBUF);
+ Hash_Store(self->entries, (Obj*)entry, INCREF(&EMPTY));
+ }
+
+ DECREF(list);
+ DECREF(snap_data);
+ }
+
+ return self;
+}
+
+static VArray*
+S_clean_segment_contents(VArray *orig) {
+ // Since Snapshot format 2, no DataReader has depended on individual files
+ // within segment directories being listed. Filter these files because
+ // they cause a problem with FilePurger.
+ VArray *cleaned = VA_new(VA_Get_Size(orig));
+ for (uint32_t i = 0, max = VA_Get_Size(orig); i < max; i++) {
+ CharBuf *name = (CharBuf*)VA_Fetch(orig, i);
+ if (!Seg_valid_seg_name(name)) {
+ if (CB_Starts_With_Str(name, "seg_", 4)) {
+ continue; // Skip this file.
+ }
+ }
+ VA_Push(cleaned, INCREF(name));
+ }
+ return cleaned;
+}
+
+
+void
+Snapshot_write_file(Snapshot *self, Folder *folder, const CharBuf *path) {
+ Hash *all_data = Hash_new(0);
+ VArray *list = Snapshot_List(self);
+
+ // Update path.
+ DECREF(self->path);
+ if (path) {
+ self->path = CB_Clone(path);
+ }
+ else {
+ CharBuf *latest = IxFileNames_latest_snapshot(folder);
+ uint64_t gen = latest ? IxFileNames_extract_gen(latest) + 1 : 1;
+ char base36[StrHelp_MAX_BASE36_BYTES];
+ StrHelp_to_base36(gen, &base36);
+ self->path = CB_newf("snapshot_%s.json", &base36);
+ DECREF(latest);
+ }
+
+ // Don't overwrite.
+ if (Folder_Exists(folder, self->path)) {
+ THROW(ERR, "Snapshot file '%o' already exists", self->path);
+ }
+
+ // Sort, then store file names.
+ VA_Sort(list, NULL, NULL);
+ Hash_Store_Str(all_data, "entries", 7, (Obj*)list);
+
+ // Create a JSON-izable data structure.
+ Hash_Store_Str(all_data, "format", 6,
+ (Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_format));
+ Hash_Store_Str(all_data, "subformat", 9,
+ (Obj*)CB_newf("%i32", (int32_t)Snapshot_current_file_subformat));
+
+ // Write out JSON-ized data to the new file.
+ Json_spew_json((Obj*)all_data, folder, self->path);
+
+ DECREF(all_data);
+}
+
+
diff --git a/core/Lucy/Index/Snapshot.cfh b/core/Lucy/Index/Snapshot.cfh
new file mode 100644
index 0000000..d28c0b1
--- /dev/null
+++ b/core/Lucy/Index/Snapshot.cfh
@@ -0,0 +1,107 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Point-in-time index file list.
+ *
+ * A Snapshot is list of index files and folders. Because index files, once
+ * written, are never modified, a Snapshot defines a point-in-time view of the
+ * data in an index.
+ *
+ * L<IndexReader|Lucy::Index::IndexReader> objects interpret the data
+ * associated with a single Snapshot.
+ */
+
+class Lucy::Index::Snapshot inherits Lucy::Object::Obj : dumpable {
+
+ Hash *entries;
+ CharBuf *path;
+
+ inert int32_t current_file_format;
+
+ public inert incremented Snapshot*
+ new();
+
+ /**
+ * Constructor. Takes no arguments.
+ */
+ public inert Snapshot*
+ init(Snapshot *self);
+
+ /** Return an array of all entries.
+ */
+ public incremented VArray*
+ List(Snapshot *self);
+
+ /** Return the number of entries (including directories).
+ */
+ public uint32_t
+ Num_Entries(Snapshot *self);
+
+ /** Add a filepath to the snapshot.
+ */
+ public void
+ Add_Entry(Snapshot *self, const CharBuf *entry);
+
+ /** Delete a filepath from the snapshot.
+ *
+ * @return true if the entry existed and was successfully deleted, false
+ * otherwise.
+ */
+ public bool_t
+ Delete_Entry(Snapshot *self, const CharBuf *entry);
+
+ /** Decode a snapshot file and initialize the object to reflect its
+ * contents.
+ *
+ * @param folder A Folder.
+ * @param path The location of the snapshot file. If not supplied, the
+ * most recent snapshot file in the base directory will be chosen.
+ * @return the object, allowing an assignment idiom.
+ */
+ public Snapshot*
+ Read_File(Snapshot *self, Folder *folder, const CharBuf *path = NULL);
+
+ /** Write a snapshot file. The caller must lock the index while this
+ * operation takes place, and the operation will fail if the snapshot file
+ * already exists.
+ *
+ * @param folder A Folder.
+ * @param path The path of the file to write. If NULL, a file name will
+ * be chosen which supersedes the latest snapshot file in the index
+ * folder.
+ */
+ public void
+ Write_File(Snapshot *self, Folder *folder, const CharBuf *path = NULL);
+
+ /** Set the path to the file that the Snapshot object serves as a proxy
+ * for.
+ */
+ public void
+ Set_Path(Snapshot *self, const CharBuf *path);
+
+ /** Get the path to the snapshot file. Initially NULL; updated by
+ * Read_File(), Write_File(), and Set_Path().
+ */
+ public nullable CharBuf*
+ Get_Path(Snapshot *self);
+
+ public void
+ Destroy(Snapshot *self);
+}
+
+
diff --git a/core/Lucy/Index/SortCache.c b/core/Lucy/Index/SortCache.c
new file mode 100644
index 0000000..af112d1
--- /dev/null
+++ b/core/Lucy/Index/SortCache.c
@@ -0,0 +1,172 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTCACHE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Plan/FieldType.h"
+
+SortCache*
+SortCache_init(SortCache *self, const CharBuf *field, FieldType *type,
+ void *ords, int32_t cardinality, int32_t doc_max, int32_t null_ord,
+ int32_t ord_width) {
+ // Init.
+ self->native_ords = false;
+
+ // Assign.
+ if (!FType_Sortable(type)) {
+ THROW(ERR, "Non-sortable FieldType for %o", field);
+ }
+ self->field = CB_Clone(field);
+ self->type = (FieldType*)INCREF(type);
+ self->ords = ords;
+ self->cardinality = cardinality;
+ self->doc_max = doc_max;
+ self->null_ord = null_ord;
+ self->ord_width = ord_width;
+
+ ABSTRACT_CLASS_CHECK(self, SORTCACHE);
+ return self;
+}
+
+void
+SortCache_destroy(SortCache *self) {
+ DECREF(self->field);
+ DECREF(self->type);
+ SUPER_DESTROY(self, SORTCACHE);
+}
+
+bool_t
+SortCache_get_native_ords(SortCache *self) {
+ return self->native_ords;
+}
+
+void
+SortCache_set_native_ords(SortCache *self, bool_t native_ords) {
+ self->native_ords = native_ords;
+}
+
+int32_t
+SortCache_ordinal(SortCache *self, int32_t doc_id) {
+ if ((uint32_t)doc_id > (uint32_t)self->doc_max) {
+ THROW(ERR, "Out of range: %i32 > %i32", doc_id, self->doc_max);
+ }
+ switch (self->ord_width) {
+ case 1: return NumUtil_u1get(self->ords, doc_id);
+ case 2: return NumUtil_u2get(self->ords, doc_id);
+ case 4: return NumUtil_u4get(self->ords, doc_id);
+ case 8: {
+ uint8_t *ints = (uint8_t*)self->ords;
+ return ints[doc_id];
+ }
+ case 16:
+ if (self->native_ords) {
+ uint16_t *ints = (uint16_t*)self->ords;
+ return ints[doc_id];
+ }
+ else {
+ uint8_t *bytes = (uint8_t*)self->ords;
+ bytes += doc_id * sizeof(uint16_t);
+ return NumUtil_decode_bigend_u16(bytes);
+ }
+ case 32:
+ if (self->native_ords) {
+ uint32_t *ints = (uint32_t*)self->ords;
+ return ints[doc_id];
+ }
+ else {
+ uint8_t *bytes = (uint8_t*)self->ords;
+ bytes += doc_id * sizeof(uint32_t);
+ return NumUtil_decode_bigend_u32(bytes);
+ }
+ default: {
+ THROW(ERR, "Invalid ord width: %i32", self->ord_width);
+ UNREACHABLE_RETURN(int32_t);
+ }
+ }
+}
+
+int32_t
+SortCache_find(SortCache *self, Obj *term) {
+ FieldType *const type = self->type;
+ int32_t lo = 0;
+ int32_t hi = self->cardinality - 1;
+ int32_t result = -100;
+ Obj *blank = SortCache_Make_Blank(self);
+
+ if (term != NULL
+ && !Obj_Is_A(term, Obj_Get_VTable(blank))
+ && !Obj_Is_A(blank, Obj_Get_VTable(term))
+ ) {
+ THROW(ERR, "SortCache error for field %o: term is a %o, and not "
+ "comparable to a %o", self->field, Obj_Get_Class_Name(term),
+ Obj_Get_Class_Name(blank));
+ }
+
+ // Binary search.
+ while (hi >= lo) {
+ const int32_t mid = lo + ((hi - lo) / 2);
+ Obj *val = SortCache_Value(self, mid, blank);
+ int32_t comparison = FType_null_back_compare_values(type, term, val);
+ if (comparison < 0) {
+ hi = mid - 1;
+ }
+ else if (comparison > 0) {
+ lo = mid + 1;
+ }
+ else {
+ result = mid;
+ break;
+ }
+ }
+
+ DECREF(blank);
+
+ if (hi < 0) {
+ // Target is "less than" the first cache entry.
+ return -1;
+ }
+ else if (result == -100) {
+ // If result is still -100, it wasn't set.
+ return hi;
+ }
+ else {
+ return result;
+ }
+}
+
+void*
+SortCache_get_ords(SortCache *self) {
+ return self->ords;
+}
+
+int32_t
+SortCache_get_cardinality(SortCache *self) {
+ return self->cardinality;
+}
+
+int32_t
+SortCache_get_null_ord(SortCache *self) {
+ return self->null_ord;
+}
+
+int32_t
+SortCache_get_ord_width(SortCache *self) {
+ return self->ord_width;
+}
+
+
diff --git a/core/Lucy/Index/SortCache.cfh b/core/Lucy/Index/SortCache.cfh
new file mode 100644
index 0000000..e0de55d
--- /dev/null
+++ b/core/Lucy/Index/SortCache.cfh
@@ -0,0 +1,88 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read a segment's sort caches.
+ */
+class Lucy::Index::SortCache inherits Lucy::Object::Obj {
+
+ CharBuf *field;
+ FieldType *type;
+ void *ords;
+ int32_t doc_max;
+ int32_t cardinality;
+ int32_t ord_width;
+ int32_t null_ord;
+ bool_t native_ords;
+
+ public inert SortCache*
+ init(SortCache *self, const CharBuf *field, FieldType *type,
+ void *ords, int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width);
+
+ /** Assign the value for ordinal <code>ord</code> to <code>blank</code>.
+ *
+ * @return either <code>blank</code> (no longer blank), or NULL if the
+ * value for <code>ord</code> is NULL.
+ */
+ public abstract nullable Obj*
+ Value(SortCache *self, int32_t ord, Obj *blank);
+
+ /** Return an object appropriate for use as an argument to Value().
+ */
+ public abstract incremented Obj*
+ Make_Blank(SortCache *self);
+
+ public void*
+ Get_Ords(SortCache *self);
+
+ public int32_t
+ Get_Cardinality(SortCache *self);
+
+ public int32_t
+ Get_Ord_Width(SortCache *self);
+
+ public int32_t
+ Get_Null_Ord(SortCache *self);
+
+ public int32_t
+ Ordinal(SortCache *self, int32_t doc_id);
+
+ /** Attempt to find the ordinal of the supplied <code>term</code>. If the
+ * term cannot be found, return the ordinal of the term that would appear
+ * immediately before it in sort order.
+ *
+ * @return an integer between -1 and the highest ordinal.
+ */
+ public int32_t
+ Find(SortCache *self, Obj *term = NULL);
+
+ /** Early versions of SortCache had a bug where ords were written using
+ * native byte order rather than big-endian byte order. The "native_ords"
+ * setting indicates whether this SortCache has such a bug.
+ */
+ void
+ Set_Native_Ords(SortCache *self, bool_t native_ords);
+
+ bool_t
+ Get_Native_Ords(SortCache *self);
+
+ public void
+ Destroy(SortCache *self);
+}
+
+
diff --git a/core/Lucy/Index/SortCache/NumericSortCache.c b/core/Lucy/Index/SortCache/NumericSortCache.c
new file mode 100644
index 0000000..0898062
--- /dev/null
+++ b/core/Lucy/Index/SortCache/NumericSortCache.c
@@ -0,0 +1,256 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NUMERICSORTCACHE
+#define C_LUCY_INT32SORTCACHE
+#define C_LUCY_INT64SORTCACHE
+#define C_LUCY_FLOAT32SORTCACHE
+#define C_LUCY_FLOAT64SORTCACHE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SortCache/NumericSortCache.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/NumericType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/Folder.h"
+
+NumericSortCache*
+NumSortCache_init(NumericSortCache *self, const CharBuf *field,
+ FieldType *type, int32_t cardinality, int32_t doc_max,
+ int32_t null_ord, int32_t ord_width, InStream *ord_in,
+ InStream *dat_in) {
+ // Validate.
+ if (!type || !FType_Sortable(type) || !FType_Is_A(type, NUMERICTYPE)) {
+ DECREF(self);
+ THROW(ERR, "'%o' isn't a sortable NumericType field", field);
+ }
+
+ // Mmap ords and super-init.
+ int64_t ord_len = InStream_Length(ord_in);
+ void *ords = InStream_Buf(ord_in, (size_t)ord_len);
+ SortCache_init((SortCache*)self, field, type, ords, cardinality, doc_max,
+ null_ord, ord_width);
+
+ // Assign.
+ self->ord_in = (InStream*)INCREF(ord_in);
+ self->dat_in = (InStream*)INCREF(dat_in);
+
+ // Validate ord file length.
+ double BITS_PER_BYTE = 8.0;
+ double docs_per_byte = BITS_PER_BYTE / self->ord_width;
+ double max_ords = ord_len * docs_per_byte;
+ if (max_ords < self->doc_max + 1) {
+ DECREF(self);
+ THROW(ERR, "Conflict between ord count max %f64 and doc_max %i32 for "
+ "field %o", max_ords, self->doc_max, field);
+ }
+
+ ABSTRACT_CLASS_CHECK(self, NUMERICSORTCACHE);
+ return self;
+}
+
+void
+NumSortCache_destroy(NumericSortCache *self) {
+ if (self->ord_in) {
+ InStream_Close(self->ord_in);
+ InStream_Dec_RefCount(self->ord_in);
+ }
+ if (self->dat_in) {
+ InStream_Close(self->dat_in);
+ InStream_Dec_RefCount(self->dat_in);
+ }
+ SUPER_DESTROY(self, NUMERICSORTCACHE);
+}
+
+/***************************************************************************/
+
+Float64SortCache*
+F64SortCache_new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in) {
+ Float64SortCache *self
+ = (Float64SortCache*)VTable_Make_Obj(FLOAT64SORTCACHE);
+ return F64SortCache_init(self, field, type, cardinality, doc_max,
+ null_ord, ord_width, ord_in, dat_in);
+}
+
+Float64SortCache*
+F64SortCache_init(Float64SortCache *self, const CharBuf *field,
+ FieldType *type, int32_t cardinality, int32_t doc_max,
+ int32_t null_ord, int32_t ord_width, InStream *ord_in,
+ InStream *dat_in) {
+ NumSortCache_init((NumericSortCache*)self, field, type, cardinality,
+ doc_max, null_ord, ord_width, ord_in, dat_in);
+ return self;
+}
+
+Obj*
+F64SortCache_value(Float64SortCache *self, int32_t ord, Obj *blank) {
+ if (ord == self->null_ord) {
+ return NULL;
+ }
+ else if (ord < 0) {
+ THROW(ERR, "Ordinal less than 0 for %o: %i32", self->field, ord);
+ }
+ else {
+ Float64 *num_blank = (Float64*)CERTIFY(blank, FLOAT64);
+ InStream_Seek(self->dat_in, ord * sizeof(double));
+ Float64_Set_Value(num_blank, InStream_Read_F64(self->dat_in));
+ }
+ return blank;
+}
+
+Float64*
+F64SortCache_make_blank(Float64SortCache *self) {
+ UNUSED_VAR(self);
+ return Float64_new(0.0);
+}
+
+/***************************************************************************/
+
+Float32SortCache*
+F32SortCache_new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in) {
+ Float32SortCache *self
+ = (Float32SortCache*)VTable_Make_Obj(FLOAT32SORTCACHE);
+ return F32SortCache_init(self, field, type, cardinality, doc_max,
+ null_ord, ord_width, ord_in, dat_in);
+}
+
+Float32SortCache*
+F32SortCache_init(Float32SortCache *self, const CharBuf *field,
+ FieldType *type, int32_t cardinality, int32_t doc_max,
+ int32_t null_ord, int32_t ord_width, InStream *ord_in,
+ InStream *dat_in) {
+ NumSortCache_init((NumericSortCache*)self, field, type, cardinality,
+ doc_max, null_ord, ord_width, ord_in, dat_in);
+ return self;
+}
+
+Obj*
+F32SortCache_value(Float32SortCache *self, int32_t ord, Obj *blank) {
+ if (ord == self->null_ord) {
+ return NULL;
+ }
+ else if (ord < 0) {
+ THROW(ERR, "Ordinal less than 0 for %o: %i32", self->field, ord);
+ }
+ else {
+ Float32 *num_blank = (Float32*)CERTIFY(blank, FLOAT32);
+ InStream_Seek(self->dat_in, ord * sizeof(float));
+ Float32_Set_Value(num_blank, InStream_Read_F32(self->dat_in));
+ }
+ return blank;
+}
+
+Float32*
+F32SortCache_make_blank(Float32SortCache *self) {
+ UNUSED_VAR(self);
+ return Float32_new(0.0f);
+}
+
+/***************************************************************************/
+
+Int32SortCache*
+I32SortCache_new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in) {
+ Int32SortCache *self
+ = (Int32SortCache*)VTable_Make_Obj(INT32SORTCACHE);
+ return I32SortCache_init(self, field, type, cardinality, doc_max,
+ null_ord, ord_width, ord_in, dat_in);
+}
+
+Int32SortCache*
+I32SortCache_init(Int32SortCache *self, const CharBuf *field,
+ FieldType *type, int32_t cardinality, int32_t doc_max,
+ int32_t null_ord, int32_t ord_width, InStream *ord_in,
+ InStream *dat_in) {
+ NumSortCache_init((NumericSortCache*)self, field, type, cardinality,
+ doc_max, null_ord, ord_width, ord_in, dat_in);
+ return self;
+}
+
+Obj*
+I32SortCache_value(Int32SortCache *self, int32_t ord, Obj *blank) {
+ if (ord == self->null_ord) {
+ return NULL;
+ }
+ else if (ord < 0) {
+ THROW(ERR, "Ordinal less than 0 for %o: %i32", self->field, ord);
+ }
+ else {
+ Integer32 *int_blank = (Integer32*)CERTIFY(blank, INTEGER32);
+ InStream_Seek(self->dat_in, ord * sizeof(int32_t));
+ Int32_Set_Value(int_blank, InStream_Read_I32(self->dat_in));
+ }
+ return blank;
+}
+
+Integer32*
+I32SortCache_make_blank(Int32SortCache *self) {
+ UNUSED_VAR(self);
+ return Int32_new(0);
+}
+
+/***************************************************************************/
+
+Int64SortCache*
+I64SortCache_new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in) {
+ Int64SortCache *self
+ = (Int64SortCache*)VTable_Make_Obj(INT64SORTCACHE);
+ return I64SortCache_init(self, field, type, cardinality, doc_max,
+ null_ord, ord_width, ord_in, dat_in);
+}
+
+Int64SortCache*
+I64SortCache_init(Int64SortCache *self, const CharBuf *field,
+ FieldType *type, int32_t cardinality, int32_t doc_max,
+ int32_t null_ord, int32_t ord_width, InStream *ord_in,
+ InStream *dat_in) {
+ NumSortCache_init((NumericSortCache*)self, field, type, cardinality,
+ doc_max, null_ord, ord_width, ord_in, dat_in);
+ return self;
+}
+
+Obj*
+I64SortCache_value(Int64SortCache *self, int32_t ord, Obj *blank) {
+ if (ord == self->null_ord) {
+ return NULL;
+ }
+ else if (ord < 0) {
+ THROW(ERR, "Ordinal less than 0 for %o: %i32", self->field, ord);
+ }
+ else {
+ Integer64 *int_blank = (Integer64*)CERTIFY(blank, INTEGER64);
+ InStream_Seek(self->dat_in, ord * sizeof(int64_t));
+ Int64_Set_Value(int_blank, InStream_Read_I64(self->dat_in));
+ }
+ return blank;
+}
+
+Integer64*
+I64SortCache_make_blank(Int64SortCache *self) {
+ UNUSED_VAR(self);
+ return Int64_new(0);
+}
+
+
diff --git a/core/Lucy/Index/SortCache/NumericSortCache.cfh b/core/Lucy/Index/SortCache/NumericSortCache.cfh
new file mode 100644
index 0000000..375d7af
--- /dev/null
+++ b/core/Lucy/Index/SortCache/NumericSortCache.cfh
@@ -0,0 +1,114 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::SortCache::NumericSortCache cnick NumSortCache
+ inherits Lucy::Index::SortCache {
+
+ InStream *ord_in;
+ InStream *dat_in;
+
+ inert NumericSortCache*
+ init(NumericSortCache *self, const CharBuf *field, FieldType *type,
+ int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width, InStream *ord_in, InStream *dat_in);
+
+ public void
+ Destroy(NumericSortCache *self);
+}
+
+class Lucy::Index::SortCache::Float64SortCache cnick F64SortCache
+ inherits Lucy::Index::SortCache::NumericSortCache {
+
+ public inert incremented Float64SortCache*
+ new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord = -1, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in);
+
+ public inert Float64SortCache*
+ init(Float64SortCache *self, const CharBuf *field, FieldType *type,
+ int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width, InStream *ord_in, InStream *dat_in);
+
+ public nullable Obj*
+ Value(Float64SortCache *self, int32_t ord, Obj *blank);
+
+ public incremented Float64*
+ Make_Blank(Float64SortCache *self);
+}
+
+class Lucy::Index::SortCache::Float32SortCache cnick F32SortCache
+ inherits Lucy::Index::SortCache::NumericSortCache {
+
+ public inert incremented Float32SortCache*
+ new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord = -1, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in);
+
+ public inert Float32SortCache*
+ init(Float32SortCache *self, const CharBuf *field, FieldType *type,
+ int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width, InStream *ord_in, InStream *dat_in);
+
+ public nullable Obj*
+ Value(Float32SortCache *self, int32_t ord, Obj *blank);
+
+ public incremented Float32*
+ Make_Blank(Float32SortCache *self);
+}
+
+class Lucy::Index::SortCache::Int32SortCache cnick I32SortCache
+ inherits Lucy::Index::SortCache::NumericSortCache {
+
+ public inert incremented Int32SortCache*
+ new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord = -1, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in);
+
+ public inert Int32SortCache*
+ init(Int32SortCache *self, const CharBuf *field, FieldType *type,
+ int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width, InStream *ord_in, InStream *dat_in);
+
+ public nullable Obj*
+ Value(Int32SortCache *self, int32_t ord, Obj *blank);
+
+ public incremented Integer32*
+ Make_Blank(Int32SortCache *self);
+}
+
+class Lucy::Index::SortCache::Int64SortCache cnick I64SortCache
+ inherits Lucy::Index::SortCache::NumericSortCache {
+
+ public inert incremented Int64SortCache*
+ new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord = -1, int32_t ord_width,
+ InStream *ord_in, InStream *dat_in);
+
+ public inert Int64SortCache*
+ init(Int64SortCache *self, const CharBuf *field, FieldType *type,
+ int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width, InStream *ord_in, InStream *dat_in);
+
+ public nullable Obj*
+ Value(Int64SortCache *self, int32_t ord, Obj *blank);
+
+ public incremented Integer64*
+ Make_Blank(Int64SortCache *self);
+}
+
+
diff --git a/core/Lucy/Index/SortCache/TextSortCache.c b/core/Lucy/Index/SortCache/TextSortCache.c
new file mode 100644
index 0000000..1905215
--- /dev/null
+++ b/core/Lucy/Index/SortCache/TextSortCache.c
@@ -0,0 +1,132 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TEXTSORTCACHE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SortCache/TextSortCache.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/Folder.h"
+
+TextSortCache*
+TextSortCache_new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord, int32_t ord_width,
+ InStream *ord_in, InStream *ix_in, InStream *dat_in) {
+ TextSortCache *self = (TextSortCache*)VTable_Make_Obj(TEXTSORTCACHE);
+ return TextSortCache_init(self, field, type, cardinality, doc_max,
+ null_ord, ord_width, ord_in, ix_in, dat_in);
+}
+
+TextSortCache*
+TextSortCache_init(TextSortCache *self, const CharBuf *field,
+ FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord, int32_t ord_width,
+ InStream *ord_in, InStream *ix_in, InStream *dat_in) {
+ // Validate.
+ if (!type || !FType_Sortable(type)) {
+ DECREF(self);
+ THROW(ERR, "'%o' isn't a sortable field", field);
+ }
+
+ // Memory map ords and super-init.
+ int64_t ord_len = InStream_Length(ord_in);
+ void *ords = InStream_Buf(ord_in, (size_t)ord_len);
+ SortCache_init((SortCache*)self, field, type, ords, cardinality, doc_max,
+ null_ord, ord_width);
+
+ // Validate ords file length.
+ double bytes_per_doc = self->ord_width / 8.0;
+ double max_ords = ord_len / bytes_per_doc;
+ if (max_ords < self->doc_max + 1) {
+ WARN("ORD WIDTH: %i32 %i32", ord_width, self->ord_width);
+ THROW(ERR, "Conflict between ord count max %f64 and doc_max %i32 for "
+ "field %o", max_ords, doc_max, field);
+ }
+
+ // Assign.
+ self->ord_in = (InStream*)INCREF(ord_in);
+ self->ix_in = (InStream*)INCREF(ix_in);
+ self->dat_in = (InStream*)INCREF(dat_in);
+
+ return self;
+}
+
+void
+TextSortCache_destroy(TextSortCache *self) {
+ if (self->ord_in) {
+ InStream_Close(self->ord_in);
+ InStream_Dec_RefCount(self->ord_in);
+ }
+ if (self->ix_in) {
+ InStream_Close(self->ix_in);
+ InStream_Dec_RefCount(self->ix_in);
+ }
+ if (self->dat_in) {
+ InStream_Close(self->dat_in);
+ InStream_Dec_RefCount(self->dat_in);
+ }
+ SUPER_DESTROY(self, TEXTSORTCACHE);
+}
+
+#define NULL_SENTINEL -1
+
+Obj*
+TextSortCache_value(TextSortCache *self, int32_t ord, Obj *blank) {
+ if (ord == self->null_ord) {
+ return NULL;
+ }
+ InStream_Seek(self->ix_in, ord * sizeof(int64_t));
+ int64_t offset = InStream_Read_I64(self->ix_in);
+ if (offset == NULL_SENTINEL) {
+ return NULL;
+ }
+ else {
+ uint32_t next_ord = ord + 1;
+ int64_t next_offset;
+ while (1) {
+ InStream_Seek(self->ix_in, next_ord * sizeof(int64_t));
+ next_offset = InStream_Read_I64(self->ix_in);
+ if (next_offset != NULL_SENTINEL) { break; }
+ next_ord++;
+ }
+
+ // Read character data into CharBuf.
+ CERTIFY(blank, CHARBUF);
+ int64_t len = next_offset - offset;
+ char *ptr = CB_Grow((CharBuf*)blank, (size_t)len);
+ InStream_Seek(self->dat_in, offset);
+ InStream_Read_Bytes(self->dat_in, ptr, (size_t)len);
+ ptr[len] = '\0';
+ if (!StrHelp_utf8_valid(ptr, (size_t)len)) {
+ CB_Set_Size((CharBuf*)blank, 0);
+ THROW(ERR, "Invalid UTF-8 at %i64 in %o", offset,
+ InStream_Get_Filename(self->dat_in));
+ }
+ CB_Set_Size((CharBuf*)blank, (size_t)len);
+ }
+ return blank;
+}
+
+CharBuf*
+TextSortCache_make_blank(TextSortCache *self) {
+ UNUSED_VAR(self);
+ return CB_new_from_trusted_utf8("", 0);
+}
+
+
diff --git a/core/Lucy/Index/SortCache/TextSortCache.cfh b/core/Lucy/Index/SortCache/TextSortCache.cfh
new file mode 100644
index 0000000..4819303
--- /dev/null
+++ b/core/Lucy/Index/SortCache/TextSortCache.cfh
@@ -0,0 +1,49 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** SortCache for TextType fields.
+ */
+class Lucy::Index::SortCache::TextSortCache
+ inherits Lucy::Index::SortCache {
+
+ InStream *ord_in;
+ InStream *ix_in;
+ InStream *dat_in;
+
+ inert incremented TextSortCache*
+ new(const CharBuf *field, FieldType *type, int32_t cardinality,
+ int32_t doc_max, int32_t null_ord = -1, int32_t ord_width,
+ InStream *ord_in, InStream *ix_in, InStream *dat_in);
+
+ inert TextSortCache*
+ init(TextSortCache *self, const CharBuf *field, FieldType *type,
+ int32_t cardinality, int32_t doc_max, int32_t null_ord = -1,
+ int32_t ord_width, InStream *ord_in, InStream *ix_in,
+ InStream *dat_in);
+
+ public nullable Obj*
+ Value(TextSortCache *self, int32_t ord, Obj *blank);
+
+ public incremented CharBuf*
+ Make_Blank(TextSortCache *self);
+
+ public void
+ Destroy(TextSortCache *self);
+}
+
+
diff --git a/core/Lucy/Index/SortFieldWriter.c b/core/Lucy/Index/SortFieldWriter.c
new file mode 100644
index 0000000..796da8d
--- /dev/null
+++ b/core/Lucy/Index/SortFieldWriter.c
@@ -0,0 +1,711 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTFIELDWRITER
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Index/SortFieldWriter.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Index/SortCache/NumericSortCache.h"
+#include "Lucy/Index/SortCache/TextSortCache.h"
+#include "Lucy/Index/SortReader.h"
+#include "Lucy/Index/ZombieKeyedHash.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/MemoryPool.h"
+#include "Lucy/Util/SortUtils.h"
+
+// Prepare to read back a run.
+static void
+S_flip_run(SortFieldWriter *run, size_t sub_thresh, InStream *ord_in,
+ InStream *ix_in, InStream *dat_in);
+
+// Write out a sort cache. Returns the number of unique values in the sort
+// cache.
+static int32_t
+S_write_files(SortFieldWriter *self, OutStream *ord_out, OutStream *ix_out,
+ OutStream *dat_out);
+
+typedef struct lucy_SFWriterElem {
+ Obj *value;
+ int32_t doc_id;
+} lucy_SFWriterElem;
+#define SFWriterElem lucy_SFWriterElem
+
+SortFieldWriter*
+SortFieldWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, const CharBuf *field,
+ MemoryPool *memory_pool, size_t mem_thresh,
+ OutStream *temp_ord_out, OutStream *temp_ix_out,
+ OutStream *temp_dat_out) {
+ SortFieldWriter *self
+ = (SortFieldWriter*)VTable_Make_Obj(SORTFIELDWRITER);
+ return SortFieldWriter_init(self, schema, snapshot, segment, polyreader,
+ field, memory_pool, mem_thresh, temp_ord_out,
+ temp_ix_out, temp_dat_out);
+}
+
+SortFieldWriter*
+SortFieldWriter_init(SortFieldWriter *self, Schema *schema,
+ Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, const CharBuf *field,
+ MemoryPool *memory_pool, size_t mem_thresh,
+ OutStream *temp_ord_out, OutStream *temp_ix_out,
+ OutStream *temp_dat_out) {
+ // Init.
+ SortEx_init((SortExternal*)self, sizeof(SFWriterElem));
+ self->null_ord = -1;
+ self->count = 0;
+ self->ord_start = 0;
+ self->ord_end = 0;
+ self->ix_start = 0;
+ self->ix_end = 0;
+ self->dat_start = 0;
+ self->dat_end = 0;
+ self->run_cardinality = -1;
+ self->run_max = -1;
+ self->sort_cache = NULL;
+ self->doc_map = NULL;
+ self->sorted_ids = NULL;
+ self->run_ord = 0;
+ self->run_tick = 0;
+ self->ord_width = 0;
+
+ // Assign.
+ self->field = CB_Clone(field);
+ self->schema = (Schema*)INCREF(schema);
+ self->snapshot = (Snapshot*)INCREF(snapshot);
+ self->segment = (Segment*)INCREF(segment);
+ self->polyreader = (PolyReader*)INCREF(polyreader);
+ self->mem_pool = (MemoryPool*)INCREF(memory_pool);
+ self->temp_ord_out = (OutStream*)INCREF(temp_ord_out);
+ self->temp_ix_out = (OutStream*)INCREF(temp_ix_out);
+ self->temp_dat_out = (OutStream*)INCREF(temp_dat_out);
+ self->mem_thresh = mem_thresh;
+
+ // Derive.
+ self->field_num = Seg_Field_Num(segment, field);
+ FieldType *type = (FieldType*)CERTIFY(
+ Schema_Fetch_Type(self->schema, field), FIELDTYPE);
+ self->type = (FieldType*)INCREF(type);
+ self->prim_id = FType_Primitive_ID(type);
+ if (self->prim_id == FType_TEXT || self->prim_id == FType_BLOB) {
+ self->var_width = true;
+ }
+ else {
+ self->var_width = false;
+ }
+ self->uniq_vals = (Hash*)ZKHash_new(memory_pool, self->prim_id);
+
+ return self;
+}
+
+void
+SortFieldWriter_clear_cache(SortFieldWriter *self) {
+ if (self->uniq_vals) {
+ Hash_Clear(self->uniq_vals);
+ }
+ SortFieldWriter_clear_cache_t super_clear_cache
+ = (SortFieldWriter_clear_cache_t)SUPER_METHOD(
+ self->vtable, SortFieldWriter, Clear_Cache);
+ super_clear_cache(self);
+}
+
+void
+SortFieldWriter_destroy(SortFieldWriter *self) {
+ DECREF(self->uniq_vals);
+ self->uniq_vals = NULL;
+ DECREF(self->field);
+ DECREF(self->schema);
+ DECREF(self->snapshot);
+ DECREF(self->segment);
+ DECREF(self->polyreader);
+ DECREF(self->type);
+ DECREF(self->mem_pool);
+ DECREF(self->temp_ord_out);
+ DECREF(self->temp_ix_out);
+ DECREF(self->temp_dat_out);
+ DECREF(self->ord_in);
+ DECREF(self->ix_in);
+ DECREF(self->dat_in);
+ DECREF(self->sort_cache);
+ DECREF(self->doc_map);
+ FREEMEM(self->sorted_ids);
+ SUPER_DESTROY(self, SORTFIELDWRITER);
+}
+
+int32_t
+SortFieldWriter_get_null_ord(SortFieldWriter *self) {
+ return self->null_ord;
+}
+
+int32_t
+SortFieldWriter_get_ord_width(SortFieldWriter *self) {
+ return self->ord_width;
+}
+
+static Obj*
+S_find_unique_value(Hash *uniq_vals, Obj *val) {
+ int32_t hash_sum = Obj_Hash_Sum(val);
+ Obj *uniq_val = Hash_Find_Key(uniq_vals, val, hash_sum);
+ if (!uniq_val) {
+ Hash_Store(uniq_vals, val, INCREF(&EMPTY));
+ uniq_val = Hash_Find_Key(uniq_vals, val, hash_sum);
+ }
+ return uniq_val;
+}
+
+void
+SortFieldWriter_add(SortFieldWriter *self, int32_t doc_id, Obj *value) {
+ // Uniq-ify the value, and record it for this document.
+ SFWriterElem elem;
+ elem.value = S_find_unique_value(self->uniq_vals, value);
+ elem.doc_id = doc_id;
+ SortFieldWriter_Feed(self, &elem);
+ self->count++;
+}
+
+void
+SortFieldWriter_add_segment(SortFieldWriter *self, SegReader *reader,
+ I32Array *doc_map, SortCache *sort_cache) {
+ if (!sort_cache) { return; }
+ SortFieldWriter *run
+ = SortFieldWriter_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, self->field, self->mem_pool,
+ self->mem_thresh, NULL, NULL, NULL);
+ run->sort_cache = (SortCache*)INCREF(sort_cache);
+ run->doc_map = (I32Array*)INCREF(doc_map);
+ run->run_max = SegReader_Doc_Max(reader);
+ run->run_cardinality = SortCache_Get_Cardinality(sort_cache);
+ run->null_ord = SortCache_Get_Null_Ord(sort_cache);
+ run->run_tick = 1;
+ SortFieldWriter_Add_Run(self, (SortExternal*)run);
+}
+
+static int32_t
+S_calc_width(int32_t cardinality) {
+ if (cardinality <= 0x00000002) { return 1; }
+ else if (cardinality <= 0x00000004) { return 2; }
+ else if (cardinality <= 0x0000000F) { return 4; }
+ else if (cardinality <= 0x000000FF) { return 8; }
+ else if (cardinality <= 0x0000FFFF) { return 16; }
+ else { return 32; }
+}
+
+static void
+S_write_ord(void *ords, int32_t width, int32_t doc_id, int32_t ord) {
+ switch (width) {
+ case 1:
+ if (ord) { NumUtil_u1set(ords, doc_id); }
+ else { NumUtil_u1clear(ords, doc_id); }
+ break;
+ case 2:
+ NumUtil_u2set(ords, doc_id, ord);
+ break;
+ case 4:
+ NumUtil_u4set(ords, doc_id, ord);
+ break;
+ case 8: {
+ uint8_t *ints = (uint8_t*)ords;
+ ints[doc_id] = ord;
+ }
+ break;
+ case 16: {
+ uint8_t *bytes = (uint8_t*)ords;
+ bytes += doc_id * sizeof(uint16_t);
+ NumUtil_encode_bigend_u16(ord, &bytes);
+ }
+ break;
+ case 32: {
+ uint8_t *bytes = (uint8_t*)ords;
+ bytes += doc_id * sizeof(uint32_t);
+ NumUtil_encode_bigend_u32(ord, &bytes);
+ }
+ break;
+ default:
+ THROW(ERR, "Invalid width: %i32", width);
+ }
+}
+
+static void
+S_write_val(Obj *val, int8_t prim_id, OutStream *ix_out, OutStream *dat_out,
+ int64_t dat_start) {
+ if (val) {
+ switch (prim_id & FType_PRIMITIVE_ID_MASK) {
+ case FType_TEXT: {
+ CharBuf *string = (CharBuf*)val;
+ int64_t dat_pos = OutStream_Tell(dat_out) - dat_start;
+ OutStream_Write_I64(ix_out, dat_pos);
+ OutStream_Write_Bytes(dat_out, (char*)CB_Get_Ptr8(string),
+ CB_Get_Size(string));
+ break;
+ }
+ case FType_BLOB: {
+ ByteBuf *byte_buf = (ByteBuf*)val;
+ int64_t dat_pos = OutStream_Tell(dat_out) - dat_start;
+ OutStream_Write_I64(ix_out, dat_pos);
+ OutStream_Write_Bytes(dat_out, BB_Get_Buf(byte_buf),
+ BB_Get_Size(byte_buf));
+ break;
+ }
+ case FType_INT32: {
+ Integer32 *i32 = (Integer32*)val;
+ OutStream_Write_I32(dat_out, Int32_Get_Value(i32));
+ break;
+ }
+ case FType_INT64: {
+ Integer64 *i64 = (Integer64*)val;
+ OutStream_Write_I64(dat_out, Int64_Get_Value(i64));
+ break;
+ }
+ case FType_FLOAT64: {
+ Float64 *float64 = (Float64*)val;
+ OutStream_Write_F64(dat_out, Float64_Get_Value(float64));
+ break;
+ }
+ case FType_FLOAT32: {
+ Float32 *float32 = (Float32*)val;
+ OutStream_Write_F32(dat_out, Float32_Get_Value(float32));
+ break;
+ }
+ default:
+ THROW(ERR, "Unrecognized primitive id: %i32", (int32_t)prim_id);
+ }
+ }
+ else {
+ switch (prim_id & FType_PRIMITIVE_ID_MASK) {
+ case FType_TEXT:
+ case FType_BLOB: {
+ int64_t dat_pos = OutStream_Tell(dat_out) - dat_start;
+ OutStream_Write_I64(ix_out, dat_pos);
+ }
+ break;
+ case FType_INT32:
+ OutStream_Write_I32(dat_out, 0);
+ break;
+ case FType_INT64:
+ OutStream_Write_I64(dat_out, 0);
+ break;
+ case FType_FLOAT64:
+ OutStream_Write_F64(dat_out, 0.0);
+ break;
+ case FType_FLOAT32:
+ OutStream_Write_F32(dat_out, 0.0f);
+ break;
+ default:
+ THROW(ERR, "Unrecognized primitive id: %i32", (int32_t)prim_id);
+ }
+ }
+}
+
+int
+SortFieldWriter_compare(SortFieldWriter *self, void *va, void *vb) {
+ SFWriterElem *a = (SFWriterElem*)va;
+ SFWriterElem *b = (SFWriterElem*)vb;
+ int32_t comparison
+ = FType_null_back_compare_values(self->type, a->value, b->value);
+ if (comparison == 0) { comparison = b->doc_id - a->doc_id; }
+ return comparison;
+}
+
+static int
+S_compare_doc_ids_by_ord_rev(void *context, const void *va, const void *vb) {
+ SortCache *sort_cache = (SortCache*)context;
+ int32_t a = *(int32_t*)va;
+ int32_t b = *(int32_t*)vb;
+ int32_t ord_a = SortCache_Ordinal(sort_cache, a);
+ int32_t ord_b = SortCache_Ordinal(sort_cache, b);
+ return ord_a - ord_b;
+}
+
+static void
+S_lazy_init_sorted_ids(SortFieldWriter *self) {
+ if (!self->sorted_ids) {
+ self->sorted_ids
+ = (int32_t*)MALLOCATE((self->run_max + 1) * sizeof(int32_t));
+ for (int32_t i = 0, max = self->run_max; i <= max; i++) {
+ self->sorted_ids[i] = i;
+ }
+ Sort_quicksort(self->sorted_ids + 1, self->run_max, sizeof(int32_t),
+ S_compare_doc_ids_by_ord_rev, self->sort_cache);
+ }
+}
+
+void
+SortFieldWriter_flush(SortFieldWriter *self) {
+ // Don't add a run unless we have data to put in it.
+ if (SortFieldWriter_Cache_Count(self) == 0) { return; }
+
+ OutStream *const temp_ord_out = self->temp_ord_out;
+ OutStream *const temp_ix_out = self->temp_ix_out;
+ OutStream *const temp_dat_out = self->temp_dat_out;
+
+ SortFieldWriter_Sort_Cache(self);
+ SortFieldWriter *run
+ = SortFieldWriter_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, self->field, self->mem_pool,
+ self->mem_thresh, NULL, NULL, NULL);
+
+ // Record stream starts and align.
+ run->ord_start = OutStream_Align(temp_ord_out, sizeof(int64_t));
+ if (self->var_width) {
+ run->ix_start = OutStream_Align(temp_ix_out, sizeof(int64_t));
+ }
+ run->dat_start = OutStream_Align(temp_dat_out, sizeof(int64_t));
+
+ // Have the run borrow the array of elems.
+ run->cache = self->cache;
+ run->cache_max = self->cache_max;
+ run->cache_tick = self->cache_tick;
+ run->cache_cap = self->cache_cap;
+
+ // Write files, record stats.
+ run->run_max = (int32_t)Seg_Get_Count(self->segment);
+ run->run_cardinality = S_write_files(run, temp_ord_out, temp_ix_out,
+ temp_dat_out);
+
+ // Reclaim the buffer from the run and empty it.
+ run->cache = NULL;
+ run->cache_max = 0;
+ run->cache_tick = 0;
+ run->cache_cap = 0;
+ self->cache_tick = self->cache_max;
+ SortFieldWriter_Clear_Cache(self);
+
+ // Record stream ends.
+ run->ord_end = OutStream_Tell(temp_ord_out);
+ if (self->var_width) {
+ run->ix_end = OutStream_Tell(temp_ix_out);
+ }
+ run->dat_end = OutStream_Tell(temp_dat_out);
+
+ // Add the run to the array.
+ SortFieldWriter_Add_Run(self, (SortExternal*)run);
+}
+
+uint32_t
+SortFieldWriter_refill(SortFieldWriter *self) {
+ if (!self->sort_cache) { return 0; }
+
+ // Sanity check, then reset the cache and prepare to start loading items.
+ uint32_t cache_count = SortFieldWriter_Cache_Count(self);
+ if (cache_count) {
+ THROW(ERR, "Refill called but cache contains %u32 items",
+ cache_count);
+ }
+ SortFieldWriter_Clear_Cache(self);
+ MemPool_Release_All(self->mem_pool);
+ S_lazy_init_sorted_ids(self);
+
+ const int32_t null_ord = self->null_ord;
+ Hash *const uniq_vals = self->uniq_vals;
+ I32Array *const doc_map = self->doc_map;
+ SortCache *const sort_cache = self->sort_cache;
+ Obj *const blank = SortCache_Make_Blank(sort_cache);
+
+ while (self->run_ord < self->run_cardinality
+ && MemPool_Get_Consumed(self->mem_pool) < self->mem_thresh
+ ) {
+ Obj *val = SortCache_Value(sort_cache, self->run_ord, blank);
+ if (val) {
+ Hash_Store(uniq_vals, val, INCREF(&EMPTY));
+ break;
+ }
+ self->run_ord++;
+ }
+ uint32_t count = 0;
+ while (self->run_tick <= self->run_max) {
+ int32_t raw_doc_id = self->sorted_ids[self->run_tick];
+ int32_t ord = SortCache_Ordinal(sort_cache, raw_doc_id);
+ if (ord != null_ord) {
+ int32_t remapped = doc_map
+ ? I32Arr_Get(doc_map, raw_doc_id)
+ : raw_doc_id;
+ if (remapped) {
+ Obj *val = SortCache_Value(sort_cache, ord, blank);
+ SortFieldWriter_Add(self, remapped, val);
+ count++;
+ }
+ }
+ else if (ord > self->run_ord) {
+ break;
+ }
+ self->run_tick++;
+ }
+ self->run_ord++;
+ SortFieldWriter_Sort_Cache(self);
+
+ if (self->run_ord >= self->run_cardinality) {
+ DECREF(self->sort_cache);
+ self->sort_cache = NULL;
+ }
+
+ DECREF(blank);
+ return count;
+}
+
+void
+SortFieldWriter_flip(SortFieldWriter *self) {
+ uint32_t num_items = SortFieldWriter_Cache_Count(self);
+ uint32_t num_runs = VA_Get_Size(self->runs);
+
+ if (self->flipped) { THROW(ERR, "Can't call Flip() twice"); }
+ self->flipped = true;
+
+ // Sanity check.
+ if (num_runs && num_items) {
+ THROW(ERR, "Sanity check failed: num_runs: %u32 num_items: %u32",
+ num_runs, num_items);
+ }
+
+ if (num_items) {
+ SortFieldWriter_Sort_Cache(self);
+ }
+ else if (num_runs) {
+ Folder *folder = PolyReader_Get_Folder(self->polyreader);
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *filepath = CB_newf("%o/sort_ord_temp", seg_name);
+ self->ord_in = Folder_Open_In(folder, filepath);
+ if (!self->ord_in) { RETHROW(INCREF(Err_get_error())); }
+ if (self->var_width) {
+ CB_setf(filepath, "%o/sort_ix_temp", seg_name);
+ self->ix_in = Folder_Open_In(folder, filepath);
+ if (!self->ix_in) { RETHROW(INCREF(Err_get_error())); }
+ }
+ CB_setf(filepath, "%o/sort_dat_temp", seg_name);
+ self->dat_in = Folder_Open_In(folder, filepath);
+ if (!self->dat_in) { RETHROW(INCREF(Err_get_error())); }
+ DECREF(filepath);
+
+ // Assign streams and a slice of mem_thresh.
+ size_t sub_thresh = self->mem_thresh / num_runs;
+ if (sub_thresh < 65536) { sub_thresh = 65536; }
+ for (uint32_t i = 0; i < num_runs; i++) {
+ SortFieldWriter *run = (SortFieldWriter*)VA_Fetch(self->runs, i);
+ S_flip_run(run, sub_thresh, self->ord_in, self->ix_in,
+ self->dat_in);
+ }
+ }
+
+ self->flipped = true;
+}
+
+static int32_t
+S_write_files(SortFieldWriter *self, OutStream *ord_out, OutStream *ix_out,
+ OutStream *dat_out) {
+ int8_t prim_id = self->prim_id;
+ int32_t doc_max = (int32_t)Seg_Get_Count(self->segment);
+ bool_t has_nulls = self->count == doc_max ? false : true;
+ size_t size = (doc_max + 1) * sizeof(int32_t);
+ int32_t *ords = (int32_t*)MALLOCATE(size);
+ int32_t ord = 0;
+ int64_t dat_start = OutStream_Tell(dat_out);
+
+ // Assign -1 as a stand-in for the NULL ord.
+ for (int32_t i = 0; i <= doc_max; i++) {
+ ords[i] = -1;
+ }
+
+ // Grab the first item and record its ord. Add a dummy ord for invalid
+ // doc id 0.
+ SFWriterElem *elem = (SFWriterElem*)SortFieldWriter_Fetch(self);
+ ords[elem->doc_id] = ord;
+ ords[0] = 0;
+
+ // Build array of ords, write non-NULL sorted values.
+ Obj *val = Obj_Clone(elem->value);
+ Obj *last_val_address = elem->value;
+ S_write_val(elem->value, prim_id, ix_out, dat_out, dat_start);
+ while (NULL != (elem = (SFWriterElem*)SortFieldWriter_Fetch(self))) {
+ if (elem->value != last_val_address) {
+ int32_t comparison
+ = FType_Compare_Values(self->type, elem->value, val);
+ if (comparison != 0) {
+ ord++;
+ S_write_val(elem->value, prim_id, ix_out, dat_out, dat_start);
+ Obj_Mimic(val, elem->value);
+ }
+ last_val_address = elem->value;
+ }
+ ords[elem->doc_id] = ord;
+ }
+ DECREF(val);
+
+ // If there are NULL values, write one now and record the NULL ord.
+ if (has_nulls) {
+ S_write_val(NULL, prim_id, ix_out, dat_out, dat_start);
+ ord++;
+ self->null_ord = ord;
+ }
+ int32_t null_ord = self->null_ord;
+
+ // Write one extra file pointer so that we can always derive length.
+ if (self->var_width) {
+ OutStream_Write_I64(ix_out, OutStream_Tell(dat_out) - dat_start);
+ }
+
+ // Calculate cardinality and ord width.
+ int32_t cardinality = ord + 1;
+ self->ord_width = S_calc_width(cardinality);
+ int32_t ord_width = self->ord_width;
+
+ // Write ords.
+ const double BITS_PER_BYTE = 8.0;
+ double bytes_per_doc = ord_width / BITS_PER_BYTE;
+ double byte_count = ceil((doc_max + 1) * bytes_per_doc);
+ char *compressed_ords
+ = (char*)CALLOCATE((size_t)byte_count, sizeof(char));
+ for (int32_t i = 0; i <= doc_max; i++) {
+ int32_t real_ord = ords[i] == -1 ? null_ord : ords[i];
+ S_write_ord(compressed_ords, ord_width, i, real_ord);
+ }
+ OutStream_Write_Bytes(ord_out, compressed_ords, (size_t)byte_count);
+ FREEMEM(compressed_ords);
+
+ FREEMEM(ords);
+ return cardinality;
+}
+
+int32_t
+SortFieldWriter_finish(SortFieldWriter *self) {
+ // Bail if there's no data.
+ if (!SortFieldWriter_Peek(self)) { return 0; }
+
+ int32_t field_num = self->field_num;
+ Folder *folder = PolyReader_Get_Folder(self->polyreader);
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *path = CB_newf("%o/sort-%i32.ord", seg_name, field_num);
+
+ // Open streams.
+ OutStream *ord_out = Folder_Open_Out(folder, path);
+ if (!ord_out) { RETHROW(INCREF(Err_get_error())); }
+ OutStream *ix_out = NULL;
+ if (self->var_width) {
+ CB_setf(path, "%o/sort-%i32.ix", seg_name, field_num);
+ ix_out = Folder_Open_Out(folder, path);
+ if (!ix_out) { RETHROW(INCREF(Err_get_error())); }
+ }
+ CB_setf(path, "%o/sort-%i32.dat", seg_name, field_num);
+ OutStream *dat_out = Folder_Open_Out(folder, path);
+ if (!dat_out) { RETHROW(INCREF(Err_get_error())); }
+ DECREF(path);
+
+ int32_t cardinality = S_write_files(self, ord_out, ix_out, dat_out);
+
+ // Close streams.
+ OutStream_Close(ord_out);
+ if (ix_out) { OutStream_Close(ix_out); }
+ OutStream_Close(dat_out);
+ DECREF(dat_out);
+ DECREF(ix_out);
+ DECREF(ord_out);
+
+ return cardinality;
+}
+
+static void
+S_flip_run(SortFieldWriter *run, size_t sub_thresh, InStream *ord_in,
+ InStream *ix_in, InStream *dat_in) {
+ if (run->flipped) { THROW(ERR, "Can't Flip twice"); }
+ run->flipped = true;
+
+ // Get our own MemoryPool, ZombieKeyedHash, and slice of mem_thresh.
+ DECREF(run->uniq_vals);
+ DECREF(run->mem_pool);
+ run->mem_pool = MemPool_new(0);
+ run->uniq_vals = (Hash*)ZKHash_new(run->mem_pool, run->prim_id);
+ run->mem_thresh = sub_thresh;
+
+ // Done if we already have a SortCache to read from.
+ if (run->sort_cache) { return; }
+
+ // Open the temp files for reading.
+ CharBuf *seg_name = Seg_Get_Name(run->segment);
+ CharBuf *alias = CB_newf("%o/sort_ord_temp-%i64-to-%i64", seg_name,
+ run->ord_start, run->ord_end);
+ InStream *ord_in_dupe = InStream_Reopen(ord_in, alias, run->ord_start,
+ run->ord_end - run->ord_start);
+ InStream *ix_in_dupe = NULL;
+ if (run->var_width) {
+ CB_setf(alias, "%o/sort_ix_temp-%i64-to-%i64", seg_name,
+ run->ix_start, run->ix_end);
+ ix_in_dupe = InStream_Reopen(ix_in, alias, run->ix_start,
+ run->ix_end - run->ix_start);
+ }
+ CB_setf(alias, "%o/sort_dat_temp-%i64-to-%i64", seg_name,
+ run->dat_start, run->dat_end);
+ InStream *dat_in_dupe = InStream_Reopen(dat_in, alias, run->dat_start,
+ run->dat_end - run->dat_start);
+ DECREF(alias);
+
+ // Get a SortCache.
+ CharBuf *field = Seg_Field_Name(run->segment, run->field_num);
+ switch (run->prim_id & FType_PRIMITIVE_ID_MASK) {
+ case FType_TEXT:
+ run->sort_cache = (SortCache*)TextSortCache_new(
+ field, run->type, run->run_cardinality,
+ run->run_max, run->null_ord,
+ run->ord_width, ord_in_dupe,
+ ix_in_dupe, dat_in_dupe);
+ break;
+ case FType_INT32:
+ run->sort_cache = (SortCache*)I32SortCache_new(
+ field, run->type, run->run_cardinality,
+ run->run_max, run->null_ord,
+ run->ord_width, ord_in_dupe,
+ dat_in_dupe);
+ break;
+ case FType_INT64:
+ run->sort_cache = (SortCache*)I64SortCache_new(
+ field, run->type, run->run_cardinality,
+ run->run_max, run->null_ord,
+ run->ord_width, ord_in_dupe,
+ dat_in_dupe);
+ break;
+ case FType_FLOAT32:
+ run->sort_cache = (SortCache*)F32SortCache_new(
+ field, run->type, run->run_cardinality,
+ run->run_max, run->null_ord,
+ run->ord_width, ord_in_dupe,
+ dat_in_dupe);
+ break;
+ case FType_FLOAT64:
+ run->sort_cache = (SortCache*)F64SortCache_new(
+ field, run->type, run->run_cardinality,
+ run->run_max, run->null_ord,
+ run->ord_width, ord_in_dupe,
+ dat_in_dupe);
+ break;
+ default:
+ THROW(ERR, "No SortCache class for %o", run->type);
+ }
+
+ DECREF(ord_in_dupe);
+ DECREF(ix_in_dupe);
+ DECREF(dat_in_dupe);
+}
+
+
diff --git a/core/Lucy/Index/SortFieldWriter.cfh b/core/Lucy/Index/SortFieldWriter.cfh
new file mode 100644
index 0000000..843c7af
--- /dev/null
+++ b/core/Lucy/Index/SortFieldWriter.cfh
@@ -0,0 +1,102 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Index::SortFieldWriter
+ inherits Lucy::Util::SortExternal {
+ CharBuf *field;
+ Hash *uniq_vals;
+ Schema *schema;
+ Snapshot *snapshot;
+ Segment *segment;
+ PolyReader *polyreader;
+ FieldType *type;
+ I32Array *doc_map;
+ MemoryPool *mem_pool;
+ int32_t field_num;
+ int32_t null_ord;
+ int8_t prim_id;
+ int32_t count;
+ OutStream *temp_ord_out;
+ OutStream *temp_ix_out;
+ OutStream *temp_dat_out;
+ InStream *ord_in;
+ InStream *ix_in;
+ InStream *dat_in;
+ SortCache *sort_cache;
+ int64_t ord_start;
+ int64_t ord_end;
+ int64_t ix_start;
+ int64_t ix_end;
+ int64_t dat_start;
+ int64_t dat_end;
+ int32_t run_cardinality;
+ int32_t run_max;
+ bool_t var_width;
+ int32_t *sorted_ids;
+ int32_t run_ord;
+ int32_t run_tick;
+ int32_t ord_width;
+
+ inert incremented SortFieldWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader, const CharBuf *field, MemoryPool *memory_pool,
+ size_t mem_thresh, OutStream *temp_ord_out, OutStream *temp_ix_out,
+ OutStream *temp_dat_out);
+
+ inert SortFieldWriter*
+ init(SortFieldWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader, const CharBuf *field,
+ MemoryPool *memory_pool, size_t mem_thresh, OutStream *temp_ord_out,
+ OutStream *temp_ix_out, OutStream *temp_dat_out);
+
+ void
+ Add(SortFieldWriter *self, int32_t doc_id, Obj *value);
+
+ void
+ Add_Segment(SortFieldWriter *self, SegReader *reader, I32Array *doc_map,
+ SortCache *sort_cache);
+
+ void
+ Flush(SortFieldWriter *self);
+
+ void
+ Flip(SortFieldWriter *self);
+
+ uint32_t
+ Refill(SortFieldWriter *self);
+
+ int32_t
+ Finish(SortFieldWriter *self);
+
+ int
+ Compare(SortFieldWriter *self, void *va, void *vb);
+
+ void
+ Clear_Cache(SortFieldWriter *self);
+
+ int32_t
+ Get_Null_Ord(SortFieldWriter *self);
+
+ int32_t
+ Get_Ord_Width(SortFieldWriter *self);
+
+ public void
+ Destroy(SortFieldWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/SortReader.c b/core/Lucy/Index/SortReader.c
new file mode 100644
index 0000000..5d3a15e
--- /dev/null
+++ b/core/Lucy/Index/SortReader.c
@@ -0,0 +1,271 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTREADER
+#define C_LUCY_DEFAULTSORTREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SortReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SortCache/NumericSortCache.h"
+#include "Lucy/Index/SortCache/TextSortCache.h"
+#include "Lucy/Index/SortWriter.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+
+SortReader*
+SortReader_init(SortReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ ABSTRACT_CLASS_CHECK(self, SORTREADER);
+ return self;
+}
+
+DataReader*
+SortReader_aggregator(SortReader *self, VArray *readers, I32Array *offsets) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(readers);
+ UNUSED_VAR(offsets);
+ return NULL;
+}
+
+DefaultSortReader*
+DefSortReader_new(Schema *schema, Folder *folder, Snapshot *snapshot,
+ VArray *segments, int32_t seg_tick) {
+ DefaultSortReader *self
+ = (DefaultSortReader*)VTable_Make_Obj(DEFAULTSORTREADER);
+ return DefSortReader_init(self, schema, folder, snapshot, segments,
+ seg_tick);
+}
+
+DefaultSortReader*
+DefSortReader_init(DefaultSortReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick) {
+ Segment *segment;
+ Hash *metadata;
+ DataReader_init((DataReader*)self, schema, folder, snapshot, segments,
+ seg_tick);
+ segment = DefSortReader_Get_Segment(self);
+ metadata = (Hash*)Seg_Fetch_Metadata_Str(segment, "sort", 4);
+
+ // Check format.
+ self->format = 0;
+ if (metadata) {
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+ if (!format) { THROW(ERR, "Missing 'format' var"); }
+ else {
+ self->format = (int32_t)Obj_To_I64(format);
+ if (self->format < 2 || self->format > 3) {
+ THROW(ERR, "Unsupported sort cache format: %i32",
+ self->format);
+ }
+ }
+ }
+
+ // Init.
+ self->caches = Hash_new(0);
+
+ // Either extract or fake up the "counts", "null_ords", and "ord_widths"
+ // hashes.
+ if (metadata) {
+ self->counts
+ = (Hash*)INCREF(CERTIFY(Hash_Fetch_Str(metadata, "counts", 6),
+ HASH));
+ self->null_ords = (Hash*)Hash_Fetch_Str(metadata, "null_ords", 9);
+ if (self->null_ords) {
+ CERTIFY(self->null_ords, HASH);
+ INCREF(self->null_ords);
+ }
+ else {
+ self->null_ords = Hash_new(0);
+ }
+ self->ord_widths = (Hash*)Hash_Fetch_Str(metadata, "ord_widths", 10);
+ if (self->ord_widths) {
+ CERTIFY(self->ord_widths, HASH);
+ INCREF(self->ord_widths);
+ }
+ else {
+ self->ord_widths = Hash_new(0);
+ }
+ }
+ else {
+ self->counts = Hash_new(0);
+ self->null_ords = Hash_new(0);
+ self->ord_widths = Hash_new(0);
+ }
+
+ return self;
+}
+
+void
+DefSortReader_close(DefaultSortReader *self) {
+ if (self->caches) {
+ Hash_Dec_RefCount(self->caches);
+ self->caches = NULL;
+ }
+ if (self->counts) {
+ Hash_Dec_RefCount(self->counts);
+ self->counts = NULL;
+ }
+ if (self->null_ords) {
+ Hash_Dec_RefCount(self->null_ords);
+ self->null_ords = NULL;
+ }
+ if (self->ord_widths) {
+ Hash_Dec_RefCount(self->ord_widths);
+ self->ord_widths = NULL;
+ }
+}
+
+void
+DefSortReader_destroy(DefaultSortReader *self) {
+ DECREF(self->caches);
+ DECREF(self->counts);
+ DECREF(self->null_ords);
+ DECREF(self->ord_widths);
+ SUPER_DESTROY(self, DEFAULTSORTREADER);
+}
+
+static int32_t
+S_calc_ord_width(int32_t cardinality) {
+ if (cardinality <= 0x00000002) { return 1; }
+ else if (cardinality <= 0x00000004) { return 2; }
+ else if (cardinality <= 0x0000000F) { return 4; }
+ else if (cardinality <= 0x000000FF) { return 8; }
+ else if (cardinality <= 0x0000FFFF) { return 16; }
+ else { return 32; }
+}
+
+static SortCache*
+S_lazy_init_sort_cache(DefaultSortReader *self, const CharBuf *field) {
+ // See if we have any values.
+ Obj *count_obj = Hash_Fetch(self->counts, (Obj*)field);
+ int32_t count = count_obj ? (int32_t)Obj_To_I64(count_obj) : 0;
+ if (!count) { return NULL; }
+
+ // Get a FieldType and sanity check that the field is sortable.
+ Schema *schema = DefSortReader_Get_Schema(self);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ if (!type || !FType_Sortable(type)) {
+ THROW(ERR, "'%o' isn't a sortable field", field);
+ }
+
+ // Open streams.
+ Folder *folder = DefSortReader_Get_Folder(self);
+ Segment *segment = DefSortReader_Get_Segment(self);
+ CharBuf *seg_name = Seg_Get_Name(segment);
+ CharBuf *path = CB_new(40);
+ int32_t field_num = Seg_Field_Num(segment, field);
+ int8_t prim_id = FType_Primitive_ID(type);
+ bool_t var_width = (prim_id == FType_TEXT || prim_id == FType_BLOB)
+ ? true
+ : false;
+ CB_setf(path, "%o/sort-%i32.ord", seg_name, field_num);
+ InStream *ord_in = Folder_Open_In(folder, path);
+ if (!ord_in) {
+ DECREF(path);
+ THROW(ERR, "Error building sort cache for '%o': %o",
+ field, Err_get_error());
+ }
+ InStream *ix_in = NULL;
+ if (var_width) {
+ CB_setf(path, "%o/sort-%i32.ix", seg_name, field_num);
+ ix_in = Folder_Open_In(folder, path);
+ if (!ix_in) {
+ DECREF(path);
+ THROW(ERR, "Error building sort cache for '%o': %o",
+ field, Err_get_error());
+ }
+ }
+ CB_setf(path, "%o/sort-%i32.dat", seg_name, field_num);
+ InStream *dat_in = Folder_Open_In(folder, path);
+ if (!dat_in) {
+ DECREF(path);
+ THROW(ERR, "Error building sort cache for '%o': %o",
+ field, Err_get_error());
+ }
+ DECREF(path);
+
+ Obj *null_ord_obj = Hash_Fetch(self->null_ords, (Obj*)field);
+ int32_t null_ord = null_ord_obj ? (int32_t)Obj_To_I64(null_ord_obj) : -1;
+ Obj *ord_width_obj = Hash_Fetch(self->ord_widths, (Obj*)field);
+ int32_t ord_width = ord_width_obj
+ ? (int32_t)Obj_To_I64(ord_width_obj)
+ : S_calc_ord_width(count);
+ int32_t doc_max = (int32_t)Seg_Get_Count(segment);
+
+ SortCache *cache = NULL;
+ switch (prim_id & FType_PRIMITIVE_ID_MASK) {
+ case FType_TEXT:
+ cache = (SortCache*)TextSortCache_new(field, type, count, doc_max,
+ null_ord, ord_width, ord_in,
+ ix_in, dat_in);
+ break;
+ case FType_INT32:
+ cache = (SortCache*)I32SortCache_new(field, type, count, doc_max,
+ null_ord, ord_width, ord_in,
+ dat_in);
+ break;
+ case FType_INT64:
+ cache = (SortCache*)I64SortCache_new(field, type, count, doc_max,
+ null_ord, ord_width, ord_in,
+ dat_in);
+ break;
+ case FType_FLOAT32:
+ cache = (SortCache*)F32SortCache_new(field, type, count, doc_max,
+ null_ord, ord_width, ord_in,
+ dat_in);
+ break;
+ case FType_FLOAT64:
+ cache = (SortCache*)F64SortCache_new(field, type, count, doc_max,
+ null_ord, ord_width, ord_in,
+ dat_in);
+ break;
+ default:
+ THROW(ERR, "No SortCache class for %o", type);
+ }
+ Hash_Store(self->caches, (Obj*)field, (Obj*)cache);
+
+ if (self->format == 2) { // bug compatibility
+ SortCache_Set_Native_Ords(cache, true);
+ }
+
+ DECREF(ord_in);
+ DECREF(ix_in);
+ DECREF(dat_in);
+
+ return cache;
+}
+
+SortCache*
+DefSortReader_fetch_sort_cache(DefaultSortReader *self, const CharBuf *field) {
+ SortCache *cache = NULL;
+
+ if (field) {
+ cache = (SortCache*)Hash_Fetch(self->caches, (Obj*)field);
+ if (!cache) {
+ cache = S_lazy_init_sort_cache(self, field);
+ }
+ }
+
+ return cache;
+}
+
+
diff --git a/core/Lucy/Index/SortReader.cfh b/core/Lucy/Index/SortReader.cfh
new file mode 100644
index 0000000..f8f9b38
--- /dev/null
+++ b/core/Lucy/Index/SortReader.cfh
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read a segment's sort caches.
+ */
+abstract class Lucy::Index::SortReader
+ inherits Lucy::Index::DataReader {
+
+ inert SortReader*
+ init(SortReader *self, Schema *schema = NULL, Folder *folder = NULL,
+ Snapshot *snapshot = NULL, VArray *segments = NULL,
+ int32_t seg_tick = -1);
+
+ abstract nullable SortCache*
+ Fetch_Sort_Cache(SortReader *self, const CharBuf *field);
+
+ /** Returns NULL, since multi-segment sort caches cannot be produced by
+ * the default implementation.
+ */
+ public incremented nullable DataReader*
+ Aggregator(SortReader *self, VArray *readers, I32Array *offsets);
+
+}
+
+class Lucy::Index::DefaultSortReader cnick DefSortReader
+ inherits Lucy::Index::SortReader {
+
+ Hash *caches;
+ Hash *counts;
+ Hash *null_ords;
+ Hash *ord_widths;
+ int32_t format;
+
+ inert incremented DefaultSortReader*
+ new(Schema *schema, Folder *folder, Snapshot *snapshot, VArray *segments,
+ int32_t seg_tick);
+
+ inert DefaultSortReader*
+ init(DefaultSortReader *self, Schema *schema, Folder *folder,
+ Snapshot *snapshot, VArray *segments, int32_t seg_tick);
+
+ nullable SortCache*
+ Fetch_Sort_Cache(DefaultSortReader *self, const CharBuf *field);
+
+ public void
+ Close(DefaultSortReader *self);
+
+ public void
+ Destroy(DefaultSortReader *self);
+}
+
+
diff --git a/core/Lucy/Index/SortWriter.c b/core/Lucy/Index/SortWriter.c
new file mode 100644
index 0000000..ff37027
--- /dev/null
+++ b/core/Lucy/Index/SortWriter.c
@@ -0,0 +1,263 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTWRITER
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Index/SortWriter.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Index/SortFieldWriter.h"
+#include "Lucy/Index/SortReader.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/MemoryPool.h"
+#include "Lucy/Util/SortUtils.h"
+
+int32_t SortWriter_current_file_format = 3;
+
+static size_t default_mem_thresh = 0x400000; // 4 MB
+
+SortWriter*
+SortWriter_new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader) {
+ SortWriter *self = (SortWriter*)VTable_Make_Obj(SORTWRITER);
+ return SortWriter_init(self, schema, snapshot, segment, polyreader);
+}
+
+SortWriter*
+SortWriter_init(SortWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader) {
+ uint32_t field_max = Schema_Num_Fields(schema) + 1;
+ DataWriter_init((DataWriter*)self, schema, snapshot, segment, polyreader);
+
+ // Init.
+ self->field_writers = VA_new(field_max);
+ self->counts = Hash_new(0);
+ self->null_ords = Hash_new(0);
+ self->ord_widths = Hash_new(0);
+ self->temp_ord_out = NULL;
+ self->temp_ix_out = NULL;
+ self->temp_dat_out = NULL;
+ self->mem_pool = MemPool_new(0);
+ self->mem_thresh = default_mem_thresh;
+ self->flush_at_finish = false;
+
+ return self;
+}
+
+void
+SortWriter_destroy(SortWriter *self) {
+ DECREF(self->field_writers);
+ DECREF(self->counts);
+ DECREF(self->null_ords);
+ DECREF(self->ord_widths);
+ DECREF(self->temp_ord_out);
+ DECREF(self->temp_ix_out);
+ DECREF(self->temp_dat_out);
+ DECREF(self->mem_pool);
+ SUPER_DESTROY(self, SORTWRITER);
+}
+
+void
+SortWriter_set_default_mem_thresh(size_t mem_thresh) {
+ default_mem_thresh = mem_thresh;
+}
+
+static SortFieldWriter*
+S_lazy_init_field_writer(SortWriter *self, int32_t field_num) {
+ SortFieldWriter *field_writer
+ = (SortFieldWriter*)VA_Fetch(self->field_writers, field_num);
+ if (!field_writer) {
+
+ // Open temp files.
+ if (!self->temp_ord_out) {
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *path = CB_newf("%o/sort_ord_temp", seg_name);
+ self->temp_ord_out = Folder_Open_Out(folder, path);
+ if (!self->temp_ord_out) {
+ DECREF(path);
+ RETHROW(INCREF(Err_get_error()));
+ }
+ CB_setf(path, "%o/sort_ix_temp", seg_name);
+ self->temp_ix_out = Folder_Open_Out(folder, path);
+ if (!self->temp_ix_out) {
+ DECREF(path);
+ RETHROW(INCREF(Err_get_error()));
+ }
+ CB_setf(path, "%o/sort_dat_temp", seg_name);
+ self->temp_dat_out = Folder_Open_Out(folder, path);
+ if (!self->temp_dat_out) {
+ DECREF(path);
+ RETHROW(INCREF(Err_get_error()));
+ }
+ DECREF(path);
+ }
+
+ CharBuf *field = Seg_Field_Name(self->segment, field_num);
+ field_writer
+ = SortFieldWriter_new(self->schema, self->snapshot, self->segment,
+ self->polyreader, field, self->mem_pool,
+ self->mem_thresh, self->temp_ord_out,
+ self->temp_ix_out, self->temp_dat_out);
+ VA_Store(self->field_writers, field_num, (Obj*)field_writer);
+ }
+ return field_writer;
+}
+
+void
+SortWriter_add_inverted_doc(SortWriter *self, Inverter *inverter,
+ int32_t doc_id) {
+ int32_t field_num;
+
+ Inverter_Iterate(inverter);
+ while (0 != (field_num = Inverter_Next(inverter))) {
+ FieldType *type = Inverter_Get_Type(inverter);
+ if (FType_Sortable(type)) {
+ SortFieldWriter *field_writer
+ = S_lazy_init_field_writer(self, field_num);
+ SortFieldWriter_Add(field_writer, doc_id,
+ Inverter_Get_Value(inverter));
+ }
+ }
+
+ // If our SortFieldWriters have collectively passed the memory threshold,
+ // flush all of them, then release all unique values with a single action.
+ if (MemPool_Get_Consumed(self->mem_pool) > self->mem_thresh) {
+ for (uint32_t i = 0; i < VA_Get_Size(self->field_writers); i++) {
+ SortFieldWriter *const field_writer
+ = (SortFieldWriter*)VA_Fetch(self->field_writers, i);
+ if (field_writer) { SortFieldWriter_Flush(field_writer); }
+ }
+ MemPool_Release_All(self->mem_pool);
+ self->flush_at_finish = true;
+ }
+}
+
+void
+SortWriter_add_segment(SortWriter *self, SegReader *reader,
+ I32Array *doc_map) {
+ VArray *fields = Schema_All_Fields(self->schema);
+
+ // Proceed field-at-a-time, rather than doc-at-a-time.
+ for (uint32_t i = 0, max = VA_Get_Size(fields); i < max; i++) {
+ CharBuf *field = (CharBuf*)VA_Fetch(fields, i);
+ SortReader *sort_reader = (SortReader*)SegReader_Fetch(
+ reader, VTable_Get_Name(SORTREADER));
+ SortCache *cache = sort_reader
+ ? SortReader_Fetch_Sort_Cache(sort_reader, field)
+ : NULL;
+ if (cache) {
+ int32_t field_num = Seg_Field_Num(self->segment, field);
+ SortFieldWriter *field_writer
+ = S_lazy_init_field_writer(self, field_num);
+ SortFieldWriter_Add_Segment(field_writer, reader, doc_map, cache);
+ self->flush_at_finish = true;
+ }
+ }
+
+ DECREF(fields);
+}
+
+void
+SortWriter_finish(SortWriter *self) {
+ VArray *const field_writers = self->field_writers;
+
+ // If we have no data, bail out.
+ if (!self->temp_ord_out) { return; }
+
+ // If we've either flushed or added segments, flush everything so that any
+ // one field can use the entire margin up to mem_thresh.
+ if (self->flush_at_finish) {
+ for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) {
+ SortFieldWriter *field_writer
+ = (SortFieldWriter*)VA_Fetch(field_writers, i);
+ if (field_writer) {
+ SortFieldWriter_Flush(field_writer);
+ }
+ }
+ }
+
+ // Close down temp streams.
+ OutStream_Close(self->temp_ord_out);
+ OutStream_Close(self->temp_ix_out);
+ OutStream_Close(self->temp_dat_out);
+
+ for (uint32_t i = 1, max = VA_Get_Size(field_writers); i < max; i++) {
+ SortFieldWriter *field_writer
+ = (SortFieldWriter*)VA_Delete(field_writers, i);
+ if (field_writer) {
+ CharBuf *field = Seg_Field_Name(self->segment, i);
+ SortFieldWriter_Flip(field_writer);
+ int32_t count = SortFieldWriter_Finish(field_writer);
+ Hash_Store(self->counts, (Obj*)field,
+ (Obj*)CB_newf("%i32", count));
+ int32_t null_ord = SortFieldWriter_Get_Null_Ord(field_writer);
+ if (null_ord != -1) {
+ Hash_Store(self->null_ords, (Obj*)field,
+ (Obj*)CB_newf("%i32", null_ord));
+ }
+ int32_t ord_width = SortFieldWriter_Get_Ord_Width(field_writer);
+ Hash_Store(self->ord_widths, (Obj*)field,
+ (Obj*)CB_newf("%i32", ord_width));
+ }
+
+ DECREF(field_writer);
+ }
+ VA_Clear(field_writers);
+
+ // Store metadata.
+ Seg_Store_Metadata_Str(self->segment, "sort", 4,
+ (Obj*)SortWriter_Metadata(self));
+
+ // Clean up.
+ Folder *folder = self->folder;
+ CharBuf *seg_name = Seg_Get_Name(self->segment);
+ CharBuf *path = CB_newf("%o/sort_ord_temp", seg_name);
+ Folder_Delete(folder, path);
+ CB_setf(path, "%o/sort_ix_temp", seg_name);
+ Folder_Delete(folder, path);
+ CB_setf(path, "%o/sort_dat_temp", seg_name);
+ Folder_Delete(folder, path);
+ DECREF(path);
+}
+
+Hash*
+SortWriter_metadata(SortWriter *self) {
+ Hash *const metadata = DataWriter_metadata((DataWriter*)self);
+ Hash_Store_Str(metadata, "counts", 6, INCREF(self->counts));
+ Hash_Store_Str(metadata, "null_ords", 9, INCREF(self->null_ords));
+ Hash_Store_Str(metadata, "ord_widths", 10, INCREF(self->ord_widths));
+ return metadata;
+}
+
+int32_t
+SortWriter_format(SortWriter *self) {
+ UNUSED_VAR(self);
+ return SortWriter_current_file_format;
+}
+
+
diff --git a/core/Lucy/Index/SortWriter.cfh b/core/Lucy/Index/SortWriter.cfh
new file mode 100644
index 0000000..1061642
--- /dev/null
+++ b/core/Lucy/Index/SortWriter.cfh
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Writer for sortable fields.
+ *
+ * Changes for format version 3:
+ *
+ * * Big-endian byte order instead of native byte order for ".ord" files.
+ * * "ord_widths" key added to metadata.
+ * * In variable-width cache formats, NULL entries get a file pointer in the
+ * ".ix" file instead of -1.
+ */
+
+class Lucy::Index::SortWriter inherits Lucy::Index::DataWriter {
+
+ VArray *field_writers;
+ Hash *counts;
+ Hash *null_ords;
+ Hash *ord_widths;
+ OutStream *temp_ord_out;
+ OutStream *temp_ix_out;
+ OutStream *temp_dat_out;
+ MemoryPool *mem_pool;
+ size_t mem_thresh;
+ bool_t flush_at_finish;
+
+ inert int32_t current_file_format;
+
+ inert incremented SortWriter*
+ new(Schema *schema, Snapshot *snapshot, Segment *segment,
+ PolyReader *polyreader);
+
+ inert SortWriter*
+ init(SortWriter *self, Schema *schema, Snapshot *snapshot,
+ Segment *segment, PolyReader *polyreader);
+
+ /* Test only. */
+ inert void
+ set_default_mem_thresh(size_t mem_thresh);
+
+ public void
+ Add_Inverted_Doc(SortWriter *self, Inverter *inverter, int32_t doc_id);
+
+ public void
+ Add_Segment(SortWriter *self, SegReader *reader,
+ I32Array *doc_map = NULL);
+
+ public incremented Hash*
+ Metadata(SortWriter *self);
+
+ public int32_t
+ Format(SortWriter *self);
+
+ public void
+ Finish(SortWriter *self);
+
+ public void
+ Destroy(SortWriter *self);
+}
+
+
diff --git a/core/Lucy/Index/TermInfo.c b/core/Lucy/Index/TermInfo.c
new file mode 100644
index 0000000..aa1f7cb
--- /dev/null
+++ b/core/Lucy/Index/TermInfo.c
@@ -0,0 +1,118 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TERMINFO
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/TermInfo.h"
+#include "Lucy/Util/StringHelper.h"
+
+TermInfo*
+TInfo_new(int32_t doc_freq) {
+ TermInfo *self = (TermInfo*)VTable_Make_Obj(TERMINFO);
+ return TInfo_init(self, doc_freq);
+}
+
+TermInfo*
+TInfo_init(TermInfo *self, int32_t doc_freq) {
+ self->doc_freq = doc_freq;
+ self->post_filepos = 0;
+ self->skip_filepos = 0;
+ self->lex_filepos = 0;
+ return self;
+}
+
+TermInfo*
+TInfo_clone(TermInfo *self) {
+ TermInfo *twin = TInfo_new(self->doc_freq);
+ twin->post_filepos = self->post_filepos;
+ twin->skip_filepos = self->skip_filepos;
+ twin->lex_filepos = self->lex_filepos;
+ return twin;
+}
+
+int32_t
+TInfo_get_doc_freq(TermInfo *self) {
+ return self->doc_freq;
+}
+
+int64_t
+TInfo_get_lex_filepos(TermInfo *self) {
+ return self->lex_filepos;
+}
+
+int64_t
+TInfo_get_post_filepos(TermInfo *self) {
+ return self->post_filepos;
+}
+
+int64_t
+TInfo_get_skip_filepos(TermInfo *self) {
+ return self->skip_filepos;
+}
+
+void
+TInfo_set_doc_freq(TermInfo *self, int32_t doc_freq) {
+ self->doc_freq = doc_freq;
+}
+
+void
+TInfo_set_lex_filepos(TermInfo *self, int64_t filepos) {
+ self->lex_filepos = filepos;
+}
+
+void
+TInfo_set_post_filepos(TermInfo *self, int64_t filepos) {
+ self->post_filepos = filepos;
+}
+
+void
+TInfo_set_skip_filepos(TermInfo *self, int64_t filepos) {
+ self->skip_filepos = filepos;
+}
+
+// TODO: this should probably be some sort of Dump variant rather than
+// To_String.
+CharBuf*
+TInfo_to_string(TermInfo *self) {
+ return CB_newf(
+ "doc freq: %i32\n"
+ "post filepos: %i64\n"
+ "skip filepos: %i64\n"
+ "index filepos: %i64",
+ self->doc_freq, self->post_filepos,
+ self->skip_filepos, self->lex_filepos
+ );
+}
+
+void
+TInfo_mimic(TermInfo *self, Obj *other) {
+ TermInfo *twin = (TermInfo*)CERTIFY(other, TERMINFO);
+ self->doc_freq = twin->doc_freq;
+ self->post_filepos = twin->post_filepos;
+ self->skip_filepos = twin->skip_filepos;
+ self->lex_filepos = twin->lex_filepos;
+}
+
+void
+TInfo_reset(TermInfo *self) {
+ self->doc_freq = 0;
+ self->post_filepos = 0;
+ self->skip_filepos = 0;
+ self->lex_filepos = 0;
+}
+
+
diff --git a/core/Lucy/Index/TermInfo.cfh b/core/Lucy/Index/TermInfo.cfh
new file mode 100644
index 0000000..eb5ff78
--- /dev/null
+++ b/core/Lucy/Index/TermInfo.cfh
@@ -0,0 +1,82 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Filepointer/statistical data for a Term.
+ *
+ * The TermInfo contains pointer data indicating where information about a
+ * term can be found in various files, plus the document frequency of the
+ * term.
+ *
+ * The lex_filepos member variable is only used if the TermInfo is part of the
+ * .lexx stream; it is a filepointer to a locations in the main .lex file.
+ */
+
+class Lucy::Index::TermInfo cnick TInfo inherits Lucy::Object::Obj {
+
+ int32_t doc_freq;
+ int64_t post_filepos;
+ int64_t skip_filepos;
+ int64_t lex_filepos;
+
+
+ inert incremented TermInfo*
+ new(int32_t doc_freq = 0);
+
+ inert TermInfo*
+ init(TermInfo *self, int32_t doc_freq = 0);
+
+ public int32_t
+ Get_Doc_Freq(TermInfo *self);
+
+ public int64_t
+ Get_Lex_FilePos(TermInfo *self);
+
+ public int64_t
+ Get_Post_FilePos(TermInfo *self);
+
+ public int64_t
+ Get_Skip_FilePos(TermInfo *self);
+
+ public void
+ Set_Doc_Freq(TermInfo *self, int32_t doc_freq);
+
+ public void
+ Set_Lex_FilePos(TermInfo *self, int64_t filepos);
+
+ public void
+ Set_Post_FilePos(TermInfo *self, int64_t filepos);
+
+ public void
+ Set_Skip_FilePos(TermInfo *self, int64_t filepos);
+
+ /** "Zero out" the TermInfo.
+ */
+ void
+ Reset(TermInfo *self);
+
+ public void
+ Mimic(TermInfo *self, Obj *other);
+
+ public incremented TermInfo*
+ Clone(TermInfo *self);
+
+ public incremented CharBuf*
+ To_String(TermInfo *self);
+}
+
+
diff --git a/core/Lucy/Index/TermStepper.c b/core/Lucy/Index/TermStepper.c
new file mode 100644
index 0000000..8b302ea
--- /dev/null
+++ b/core/Lucy/Index/TermStepper.c
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TERMSTEPPER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/TermStepper.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/StringHelper.h"
+
+TermStepper*
+TermStepper_init(TermStepper *self) {
+ Stepper_init((Stepper*)self);
+ self->value = NULL;
+ return self;
+}
+
+void
+TermStepper_destroy(TermStepper *self) {
+ DECREF(self->value);
+ SUPER_DESTROY(self, TERMSTEPPER);
+}
+
+void
+TermStepper_reset(TermStepper *self) {
+ DECREF(self->value);
+ self->value = NULL;
+}
+
+Obj*
+TermStepper_get_value(TermStepper *self) {
+ return self->value;
+}
+
+void
+TermStepper_set_value(TermStepper *self, Obj *value) {
+ DECREF(self->value);
+ self->value = value ? INCREF(value) : NULL;
+}
+
+
diff --git a/core/Lucy/Index/TermStepper.cfh b/core/Lucy/Index/TermStepper.cfh
new file mode 100644
index 0000000..77c7533
--- /dev/null
+++ b/core/Lucy/Index/TermStepper.cfh
@@ -0,0 +1,37 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+abstract class Lucy::Index::TermStepper
+ inherits Lucy::Util::Stepper {
+
+ Obj *value;
+
+ public inert TermStepper*
+ init(TermStepper *self);
+
+ public void
+ Set_Value(TermStepper *self, Obj *value = NULL);
+
+ public nullable Obj*
+ Get_Value(TermStepper *self);
+
+ public void
+ Destroy(TermStepper *self);
+}
+
+
diff --git a/core/Lucy/Index/TermVector.c b/core/Lucy/Index/TermVector.c
new file mode 100644
index 0000000..7b3e5ad
--- /dev/null
+++ b/core/Lucy/Index/TermVector.c
@@ -0,0 +1,156 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TERMVECTOR
+#define C_LUCY_I32ARRAY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/TermVector.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+TermVector*
+TV_new(const CharBuf *field, const CharBuf *text, I32Array *positions,
+ I32Array *start_offsets, I32Array *end_offsets) {
+ TermVector *self = (TermVector*)VTable_Make_Obj(TERMVECTOR);
+ return TV_init(self, field, text, positions, start_offsets, end_offsets);
+}
+
+TermVector*
+TV_init(TermVector *self, const CharBuf *field, const CharBuf *text,
+ I32Array *positions, I32Array *start_offsets, I32Array *end_offsets) {
+ // Assign.
+ self->field = CB_Clone(field);
+ self->text = CB_Clone(text);
+ self->num_pos = I32Arr_Get_Size(positions);
+ self->positions = (I32Array*)INCREF(positions);
+ self->start_offsets = (I32Array*)INCREF(start_offsets);
+ self->end_offsets = (I32Array*)INCREF(end_offsets);
+
+ if (I32Arr_Get_Size(start_offsets) != self->num_pos
+ || I32Arr_Get_Size(end_offsets) != self->num_pos
+ ) {
+ THROW(ERR, "Unbalanced arrays: %u32 %u32 %u32", self->num_pos,
+ I32Arr_Get_Size(start_offsets), I32Arr_Get_Size(end_offsets));
+ }
+
+ return self;
+}
+
+void
+TV_destroy(TermVector *self) {
+ DECREF(self->field);
+ DECREF(self->text);
+ DECREF(self->positions);
+ DECREF(self->start_offsets);
+ DECREF(self->end_offsets);
+ SUPER_DESTROY(self, TERMVECTOR);
+}
+
+I32Array*
+TV_get_positions(TermVector *self) {
+ return self->positions;
+}
+
+I32Array*
+TV_get_start_offsets(TermVector *self) {
+ return self->start_offsets;
+}
+
+I32Array*
+TV_get_end_offsets(TermVector *self) {
+ return self->end_offsets;
+}
+
+void
+TV_serialize(TermVector *self, OutStream *target) {
+ uint32_t i;
+ int32_t *posits = self->positions->ints;
+ int32_t *starts = self->start_offsets->ints;
+ int32_t *ends = self->start_offsets->ints;
+
+ CB_Serialize(self->field, target);
+ CB_Serialize(self->text, target);
+ OutStream_Write_C32(target, self->num_pos);
+
+ for (i = 0; i < self->num_pos; i++) {
+ OutStream_Write_C32(target, posits[i]);
+ OutStream_Write_C32(target, starts[i]);
+ OutStream_Write_C32(target, ends[i]);
+ }
+}
+
+TermVector*
+TV_deserialize(TermVector *self, InStream *instream) {
+ uint32_t i;
+ CharBuf *field = (CharBuf*)CB_deserialize(NULL, instream);
+ CharBuf *text = (CharBuf*)CB_deserialize(NULL, instream);
+ uint32_t num_pos = InStream_Read_C32(instream);
+ int32_t *posits, *starts, *ends;
+ I32Array *positions, *start_offsets, *end_offsets;
+
+ // Read positional data.
+ posits = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ starts = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ ends = (int32_t*)MALLOCATE(num_pos * sizeof(int32_t));
+ for (i = 0; i < num_pos; i++) {
+ posits[i] = InStream_Read_C32(instream);
+ starts[i] = InStream_Read_C32(instream);
+ ends[i] = InStream_Read_C32(instream);
+ }
+ positions = I32Arr_new_steal(posits, num_pos);
+ start_offsets = I32Arr_new_steal(starts, num_pos);
+ end_offsets = I32Arr_new_steal(ends, num_pos);
+
+ self = self ? self : (TermVector*)VTable_Make_Obj(TERMVECTOR);
+ self = TV_init(self, field, text, positions, start_offsets, end_offsets);
+
+ DECREF(positions);
+ DECREF(start_offsets);
+ DECREF(end_offsets);
+ DECREF(text);
+ DECREF(field);
+
+ return self;
+}
+
+bool_t
+TV_equals(TermVector *self, Obj *other) {
+ TermVector *const twin = (TermVector*)other;
+ uint32_t i;
+ int32_t *const posits = self->positions->ints;
+ int32_t *const starts = self->start_offsets->ints;
+ int32_t *const ends = self->start_offsets->ints;
+ int32_t *const other_posits = twin->positions->ints;
+ int32_t *const other_starts = twin->start_offsets->ints;
+ int32_t *const other_ends = twin->start_offsets->ints;
+
+ if (twin == self) { return true; }
+
+ if (!CB_Equals(self->field, (Obj*)twin->field)) { return false; }
+ if (!CB_Equals(self->text, (Obj*)twin->text)) { return false; }
+ if (self->num_pos != twin->num_pos) { return false; }
+
+ for (i = 0; i < self->num_pos; i++) {
+ if (posits[i] != other_posits[i]) { return false; }
+ if (starts[i] != other_starts[i]) { return false; }
+ if (ends[i] != other_ends[i]) { return false; }
+ }
+
+ return true;
+}
+
+
diff --git a/core/Lucy/Index/TermVector.cfh b/core/Lucy/Index/TermVector.cfh
new file mode 100644
index 0000000..0746b85
--- /dev/null
+++ b/core/Lucy/Index/TermVector.cfh
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Term freq and positional data.
+ */
+
+class Lucy::Index::TermVector cnick TV
+ inherits Lucy::Object::Obj {
+
+ CharBuf *field;
+ CharBuf *text;
+ uint32_t num_pos;
+ I32Array *positions;
+ I32Array *start_offsets;
+ I32Array *end_offsets;
+
+ /** Constructor. The object will assume ownership of the positions,
+ * start_offsets, and end_offsets arrays.
+ */
+ inert incremented TermVector*
+ new(const CharBuf *field, const CharBuf *text, I32Array *positions,
+ I32Array *start_offsets, I32Array *end_offsets);
+
+ inert TermVector*
+ init(TermVector *self, const CharBuf *field, const CharBuf *text,
+ I32Array *positions, I32Array *start_offsets, I32Array *end_offsets);
+
+ I32Array*
+ Get_Positions(TermVector *self);
+
+ I32Array*
+ Get_Start_Offsets(TermVector *self);
+
+ I32Array*
+ Get_End_Offsets(TermVector *self);
+
+ public incremented TermVector*
+ Deserialize(TermVector *self, InStream *instream);
+
+ public bool_t
+ Equals(TermVector *self, Obj *other);
+
+ public void
+ Destroy(TermVector *self);
+
+ public void
+ Serialize(TermVector *self, OutStream *outstream);
+}
+
+
diff --git a/core/Lucy/Index/ZombieKeyedHash.c b/core/Lucy/Index/ZombieKeyedHash.c
new file mode 100644
index 0000000..a69a5a7
--- /dev/null
+++ b/core/Lucy/Index/ZombieKeyedHash.c
@@ -0,0 +1,104 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ZOMBIEKEYEDHASH
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/ZombieKeyedHash.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Util/MemoryPool.h"
+
+ZombieKeyedHash*
+ZKHash_new(MemoryPool *memory_pool, uint8_t primitive_id) {
+ ZombieKeyedHash *self
+ = (ZombieKeyedHash*)VTable_Make_Obj(ZOMBIEKEYEDHASH);
+ Hash_init((Hash*)self, 0);
+ self->mem_pool = (MemoryPool*)INCREF(memory_pool);
+ self->prim_id = primitive_id;
+ return self;
+}
+
+void
+ZKHash_destroy(ZombieKeyedHash *self) {
+ DECREF(self->mem_pool);
+ SUPER_DESTROY(self, ZOMBIEKEYEDHASH);
+}
+
+Obj*
+ZKHash_make_key(ZombieKeyedHash *self, Obj *key, int32_t hash_sum) {
+ UNUSED_VAR(hash_sum);
+ Obj *retval = NULL;
+ switch (self->prim_id & FType_PRIMITIVE_ID_MASK) {
+ case FType_TEXT: {
+ CharBuf *source = (CharBuf*)key;
+ size_t size = ZCB_size() + CB_Get_Size(source) + 1;
+ void *allocation = MemPool_grab(self->mem_pool, size);
+ retval = (Obj*)ZCB_newf(allocation, size, "%o", source);
+ }
+ break;
+ case FType_INT32: {
+ size_t size = VTable_Get_Obj_Alloc_Size(INTEGER32);
+ Integer32 *copy
+ = (Integer32*)MemPool_grab(self->mem_pool, size);
+ VTable_Init_Obj(INTEGER32, copy);
+ Int32_init(copy, 0);
+ Int32_Mimic(copy, key);
+ retval = (Obj*)copy;
+ }
+ break;
+ case FType_INT64: {
+ size_t size = VTable_Get_Obj_Alloc_Size(INTEGER64);
+ Integer64 *copy
+ = (Integer64*)MemPool_Grab(self->mem_pool, size);
+ VTable_Init_Obj(INTEGER64, copy);
+ Int64_init(copy, 0);
+ Int64_Mimic(copy, key);
+ retval = (Obj*)copy;
+ }
+ break;
+ case FType_FLOAT32: {
+ size_t size = VTable_Get_Obj_Alloc_Size(FLOAT32);
+ Float32 *copy = (Float32*)MemPool_Grab(self->mem_pool, size);
+ VTable_Init_Obj(FLOAT32, copy);
+ Float32_init(copy, 0);
+ Float32_Mimic(copy, key);
+ retval = (Obj*)copy;
+ }
+ break;
+ case FType_FLOAT64: {
+ size_t size = VTable_Get_Obj_Alloc_Size(FLOAT64);
+ Float64 *copy = (Float64*)MemPool_Grab(self->mem_pool, size);
+ VTable_Init_Obj(FLOAT64, copy);
+ Float64_init(copy, 0);
+ Float64_Mimic(copy, key);
+ retval = (Obj*)copy;
+ }
+ break;
+ default:
+ THROW(ERR, "Unrecognized primitive id: %i8", self->prim_id);
+ }
+
+ /* FIXME This is a hack. It will leak memory if host objects get cached,
+ * which in the present implementation will happen as soon as the refcount
+ * reaches 4. However, we must never call Destroy() for these objects,
+ * because they will try to free() their initial allocation, which is
+ * invalid because it's part of a MemoryPool arena. */
+ INCREF(retval);
+
+ return retval;
+}
+
+
diff --git a/core/Lucy/Index/ZombieKeyedHash.cfh b/core/Lucy/Index/ZombieKeyedHash.cfh
new file mode 100644
index 0000000..6ffebdf
--- /dev/null
+++ b/core/Lucy/Index/ZombieKeyedHash.cfh
@@ -0,0 +1,37 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Hash which creates keys allocated from a MemoryPool rather than malloc.
+ */
+class Lucy::Index::SortFieldWriter::ZombieKeyedHash cnick ZKHash
+ inherits Lucy::Object::Hash {
+
+ MemoryPool *mem_pool;
+ uint8_t prim_id;
+
+ inert incremented ZombieKeyedHash*
+ new(MemoryPool *memory_pool, uint8_t primitive_id);
+
+ public void
+ Destroy(ZombieKeyedHash *self);
+
+ public incremented Obj*
+ Make_Key(ZombieKeyedHash *self, Obj *key, int32_t hash_sum);
+}
+
+
diff --git a/core/Lucy/Object/BitVector.c b/core/Lucy/Object/BitVector.c
new file mode 100644
index 0000000..95ec274
--- /dev/null
+++ b/core/Lucy/Object/BitVector.c
@@ -0,0 +1,403 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BITVECTOR
+#include "Lucy/Util/ToolSet.h"
+
+#include <math.h>
+
+#include "Lucy/Object/BitVector.h"
+
+// Shared subroutine for performing both OR and XOR ops.
+#define DO_OR 1
+#define DO_XOR 2
+static void
+S_do_or_or_xor(BitVector *self, const BitVector *other, int operation);
+
+// Number of 1 bits given a u8 value.
+static const uint32_t BYTE_COUNTS[256] = {
+ 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+ 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+};
+
+
+BitVector*
+BitVec_new(uint32_t capacity) {
+ BitVector *self = (BitVector*)VTable_Make_Obj(BITVECTOR);
+ return BitVec_init(self, capacity);
+}
+
+BitVector*
+BitVec_init(BitVector *self, uint32_t capacity) {
+ const uint32_t byte_size = (uint32_t)ceil(capacity / 8.0);
+
+ // Derive.
+ self->bits = capacity
+ ? (uint8_t*)CALLOCATE(byte_size, sizeof(uint8_t))
+ : NULL;
+
+ // Assign.
+ self->cap = byte_size * 8;
+
+ return self;
+}
+
+void
+BitVec_destroy(BitVector* self) {
+ FREEMEM(self->bits);
+ SUPER_DESTROY(self, BITVECTOR);
+}
+
+BitVector*
+BitVec_clone(BitVector *self) {
+ BitVector *twin = BitVec_new(self->cap);
+ uint32_t byte_size = (uint32_t)ceil(self->cap / 8.0);
+
+ // Forbid inheritance.
+ if (BitVec_Get_VTable(self) != BITVECTOR) {
+ THROW(ERR, "Attempt by %o to inherit BitVec_Clone",
+ BitVec_Get_Class_Name(self));
+ }
+
+ memcpy(twin->bits, self->bits, byte_size * sizeof(uint8_t));
+
+ return twin;
+}
+
+uint8_t*
+BitVec_get_raw_bits(BitVector *self) {
+ return self->bits;
+}
+
+uint32_t
+BitVec_get_capacity(BitVector *self) {
+ return self->cap;
+}
+
+void
+BitVec_mimic(BitVector *self, Obj *other) {
+ BitVector *twin = (BitVector*)CERTIFY(other, BITVECTOR);
+ const uint32_t my_byte_size = (uint32_t)ceil(self->cap / 8.0);
+ const uint32_t twin_byte_size = (uint32_t)ceil(twin->cap / 8.0);
+ if (my_byte_size > twin_byte_size) {
+ uint32_t space = my_byte_size - twin_byte_size;
+ memset(self->bits + twin_byte_size, 0, space);
+ }
+ else if (my_byte_size < twin_byte_size) {
+ BitVec_Grow(self, twin->cap - 1);
+ }
+ memcpy(self->bits, twin->bits, twin_byte_size);
+}
+
+void
+BitVec_grow(BitVector *self, uint32_t capacity) {
+ if (capacity > self->cap) {
+ const size_t old_byte_cap = (size_t)ceil(self->cap / 8.0);
+ const size_t new_byte_cap = (size_t)ceil((capacity + 1) / 8.0);
+ const size_t num_new_bytes = new_byte_cap - old_byte_cap;
+
+ self->bits = (uint8_t*)REALLOCATE(self->bits, new_byte_cap);
+ memset(self->bits + old_byte_cap, 0, num_new_bytes);
+ self->cap = new_byte_cap * 8;
+ }
+}
+
+void
+BitVec_set(BitVector *self, uint32_t tick) {
+ if (tick >= self->cap) {
+ uint32_t new_cap = (uint32_t)Memory_oversize(tick + 1, 0);
+ BitVec_Grow(self, new_cap);
+ }
+ NumUtil_u1set(self->bits, tick);
+}
+
+void
+BitVec_clear(BitVector *self, uint32_t tick) {
+ if (tick >= self->cap) {
+ return;
+ }
+ NumUtil_u1clear(self->bits, tick);
+}
+
+void
+BitVec_clear_all(BitVector *self) {
+ const size_t byte_size = (size_t)ceil(self->cap / 8.0);
+ memset(self->bits, 0, byte_size);
+}
+
+bool_t
+BitVec_get(BitVector *self, uint32_t tick) {
+ if (tick >= self->cap) {
+ return false;
+ }
+ return NumUtil_u1get(self->bits, tick);
+}
+
+static int32_t
+S_first_bit_in_nonzero_byte(uint8_t num) {
+ int32_t first_bit = 0;
+ if ((num & 0xF) == 0) { first_bit += 4; num >>= 4; }
+ if ((num & 0x3) == 0) { first_bit += 2; num >>= 2; }
+ if ((num & 0x1) == 0) { first_bit += 1; }
+ return first_bit;
+}
+
+int32_t
+BitVec_next_hit(BitVector *self, uint32_t tick) {
+ size_t byte_size = (size_t)ceil(self->cap / 8.0);
+ uint8_t *const limit = self->bits + byte_size;
+ uint8_t *ptr = self->bits + (tick >> 3);
+
+ if (ptr >= limit) {
+ return -1;
+ }
+ else if (*ptr != 0) {
+ // Special case the first byte.
+ const int32_t base = (ptr - self->bits) * 8;
+ const int32_t min_sub_tick = tick & 0x7;
+ unsigned int byte = *ptr >> min_sub_tick;
+ if (byte) {
+ const int32_t candidate
+ = base + min_sub_tick + S_first_bit_in_nonzero_byte(byte);
+ return candidate < (int32_t)self->cap ? candidate : -1;
+ }
+ }
+
+ for (ptr++ ; ptr < limit; ptr++) {
+ if (*ptr != 0) {
+ // There's a non-zero bit in this byte.
+ const int32_t base = (ptr - self->bits) * 8;
+ const int32_t candidate = base + S_first_bit_in_nonzero_byte(*ptr);
+ return candidate < (int32_t)self->cap ? candidate : -1;
+ }
+ }
+
+ return -1;
+}
+
+void
+BitVec_and(BitVector *self, const BitVector *other) {
+ uint8_t *bits_a = self->bits;
+ uint8_t *bits_b = other->bits;
+ const uint32_t min_cap = self->cap < other->cap
+ ? self->cap
+ : other->cap;
+ const size_t byte_size = (size_t)ceil(min_cap / 8.0);
+ uint8_t *const limit = bits_a + byte_size;
+
+ // Intersection.
+ while (bits_a < limit) {
+ *bits_a &= *bits_b;
+ bits_a++, bits_b++;
+ }
+
+ // Set all remaining to zero.
+ if (self->cap > min_cap) {
+ const size_t self_byte_size = (size_t)ceil(self->cap / 8.0);
+ memset(bits_a, 0, self_byte_size - byte_size);
+ }
+}
+
+void
+BitVec_or(BitVector *self, const BitVector *other) {
+ S_do_or_or_xor(self, other, DO_OR);
+}
+
+void
+BitVec_xor(BitVector *self, const BitVector *other) {
+ S_do_or_or_xor(self, other, DO_XOR);
+}
+
+static void
+S_do_or_or_xor(BitVector *self, const BitVector *other, int operation) {
+ uint8_t *bits_a, *bits_b;
+ uint32_t max_cap, min_cap;
+ uint8_t *limit;
+ double byte_size;
+
+ // Sort out what the minimum and maximum caps are.
+ if (self->cap < other->cap) {
+ max_cap = other->cap;
+ min_cap = self->cap;
+ }
+ else {
+ max_cap = self->cap;
+ min_cap = other->cap;
+ }
+
+ // Grow self if smaller than other, then calc pointers.
+ if (max_cap > self->cap) { BitVec_Grow(self, max_cap); }
+ bits_a = self->bits;
+ bits_b = other->bits;
+ byte_size = ceil(min_cap / 8.0);
+ limit = self->bits + (size_t)byte_size;
+
+ // Perform union of common bits.
+ if (operation == DO_OR) {
+ while (bits_a < limit) {
+ *bits_a |= *bits_b;
+ bits_a++, bits_b++;
+ }
+ }
+ else if (operation == DO_XOR) {
+ while (bits_a < limit) {
+ *bits_a ^= *bits_b;
+ bits_a++, bits_b++;
+ }
+ }
+ else {
+ THROW(ERR, "Unrecognized operation: %i32", (int32_t)operation);
+ }
+
+ // Copy remaining bits if other is bigger than self.
+ if (other->cap > min_cap) {
+ const double other_byte_size = ceil(other->cap / 8.0);
+ const size_t bytes_to_copy = (size_t)(other_byte_size - byte_size);
+ memcpy(bits_a, bits_b, bytes_to_copy);
+ }
+}
+
+void
+BitVec_and_not(BitVector *self, const BitVector *other) {
+ uint8_t *bits_a = self->bits;
+ uint8_t *bits_b = other->bits;
+ const uint32_t min_cap = self->cap < other->cap
+ ? self->cap
+ : other->cap;
+ const size_t byte_size = (size_t)ceil(min_cap / 8.0);
+ uint8_t *const limit = bits_a + byte_size;
+
+ // Clear bits set in other.
+ while (bits_a < limit) {
+ *bits_a &= ~(*bits_b);
+ bits_a++, bits_b++;
+ }
+}
+
+void
+BitVec_flip(BitVector *self, uint32_t tick) {
+ if (tick >= self->cap) {
+ uint32_t new_cap = (uint32_t)Memory_oversize(tick + 1, 0);
+ BitVec_Grow(self, new_cap);
+ }
+ NumUtil_u1flip(self->bits, tick);
+}
+
+void
+BitVec_flip_block(BitVector *self, uint32_t offset, uint32_t length) {
+ uint32_t first = offset;
+ uint32_t last = offset + length - 1;
+
+ // Bail if there's nothing to flip.
+ if (!length) { return; }
+
+ if (last >= self->cap) { BitVec_Grow(self, last + 1); }
+
+ // Flip partial bytes.
+ while (last % 8 != 0 && last > first) {
+ NumUtil_u1flip(self->bits, last);
+ last--;
+ }
+ while (first % 8 != 0 && first < last) {
+ NumUtil_u1flip(self->bits, first);
+ first++;
+ }
+
+ // Are first and last equal?
+ if (first == last) {
+ // There's only one bit left to flip.
+ NumUtil_u1flip(self->bits, last);
+ }
+ // They must be multiples of 8, then.
+ else {
+ const uint32_t start_tick = first >> 3;
+ const uint32_t limit_tick = last >> 3;
+ uint8_t *bits = self->bits + start_tick;
+ uint8_t *limit = self->bits + limit_tick;
+
+ // Last actually belongs to the following byte (e.g. 8, in byte 2).
+ NumUtil_u1flip(self->bits, last);
+
+ // Flip whole bytes.
+ for (; bits < limit; bits++) {
+ *bits = ~(*bits);
+ }
+ }
+}
+
+uint32_t
+BitVec_count(BitVector *self) {
+ uint32_t count = 0;
+ const size_t byte_size = (size_t)ceil(self->cap / 8.0);
+ uint8_t *ptr = self->bits;
+ uint8_t *const limit = ptr + byte_size;
+
+ for (; ptr < limit; ptr++) {
+ count += BYTE_COUNTS[*ptr];
+ }
+
+ return count;
+}
+
+I32Array*
+BitVec_to_array(BitVector *self) {
+ uint32_t count = BitVec_Count(self);
+ uint32_t num_left = count;
+ const uint32_t capacity = self->cap;
+ uint32_t *const array = (uint32_t*)CALLOCATE(count, sizeof(uint32_t));
+ const size_t byte_size = (size_t)ceil(self->cap / 8.0);
+ uint8_t *const bits = self->bits;
+ uint8_t *const limit = bits + byte_size;
+ uint32_t num = 0;
+ uint32_t i = 0;
+
+ while (num_left) {
+ uint8_t *ptr = bits + (num >> 3);
+ while (ptr < limit && *ptr == 0) {
+ num += 8;
+ ptr++;
+ }
+ do {
+ if (BitVec_Get(self, num)) {
+ array[i++] = num;
+ if (--num_left == 0) {
+ break;
+ }
+ }
+ if (num >= capacity) {
+ THROW(ERR, "Exceeded capacity: %u32 %u32", num, capacity);
+ }
+ } while (++num % 8);
+ }
+
+ return I32Arr_new_steal((int32_t*)array, count);
+}
+
+
diff --git a/core/Lucy/Object/BitVector.cfh b/core/Lucy/Object/BitVector.cfh
new file mode 100644
index 0000000..34b349e
--- /dev/null
+++ b/core/Lucy/Object/BitVector.cfh
@@ -0,0 +1,163 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** An array of bits.
+ *
+ * BitVector is a growable array of bits. All bits are initially zero.
+ */
+
+class Lucy::Object::BitVector cnick BitVec
+ inherits Lucy::Object::Obj {
+
+ uint32_t cap;
+ uint8_t *bits;
+
+ inert incremented BitVector*
+ new(uint32_t capacity = 0);
+
+ /**
+ * @param capacity The number of bits that the initial array should be
+ * able to hold.
+ */
+ public inert BitVector*
+ init(BitVector *self, uint32_t capacity = 0);
+
+ /** Return true if the bit at <code>tick</code> has been set, false if it
+ * hasn't (regardless of whether it lies within the bounds of the
+ * object's capacity).
+ *
+ * @param tick The requested bit.
+ */
+ public bool_t
+ Get(BitVector *self, uint32_t tick);
+
+ /** Set the bit at <code>tick</code> to 1.
+ *
+ * @param tick The bit to be set.
+ */
+ public void
+ Set(BitVector *self, uint32_t tick);
+
+ /** Accessor for the BitVector's underlying bit array.
+ */
+ nullable uint8_t*
+ Get_Raw_Bits(BitVector *self);
+
+ /** Accessor for capacity.
+ */
+ uint32_t
+ Get_Capacity(BitVector *self);
+
+ /** Returns the next set bit equal to or greater than <code>tick</code>,
+ * or -1 if no such bit exists.
+ */
+ public int32_t
+ Next_Hit(BitVector *self, uint32_t tick);
+
+ /** Clear the indicated bit. (i.e. set it to 0).
+ *
+ * @param tick The bit to be cleared.
+ */
+ public void
+ Clear(BitVector *self, uint32_t tick);
+
+ /** Clear all bits.
+ */
+ public void
+ Clear_All(BitVector *self);
+
+ /** If the BitVector does not already have enough room to hold the
+ * indicated number of bits, allocate more memory so that it can.
+ *
+ * @param capacity Least number of bits the BitVector should accomodate.
+ */
+ public void
+ Grow(BitVector *self, uint32_t capacity);
+
+ /** Modify the contents of this BitVector so that it has the same bits set
+ * as <code>other</code>.
+ *
+ * @param other Another BitVector.
+ */
+ public void
+ Mimic(BitVector *self, Obj *other);
+
+ /** Modify the BitVector so that only bits which remain set are those
+ * which 1) were already set in this BitVector, and 2) were also set in
+ * the other BitVector.
+ *
+ * @param other Another BitVector.
+ */
+ public void
+ And(BitVector *self, const BitVector *other);
+
+ /** Modify the BitVector, setting all bits which are set in the other
+ * BitVector if they were not already set.
+ *
+ * @param other Another BitVector.
+ */
+ public void
+ Or(BitVector *self, const BitVector *other);
+
+ /** Modify the BitVector, performing an XOR operation against the other.
+ *
+ * @param other Another BitVector.
+ */
+ public void
+ Xor(BitVector *self, const BitVector *other);
+
+ /** Modify the BitVector, clearing all bits which are set in the other.
+ *
+ * @param other Another BitVector.
+ */
+ public void
+ And_Not(BitVector *self, const BitVector *other);
+
+ /** Invert the value of a bit.
+ *
+ * @param tick The bit to invert.
+ */
+ public void
+ Flip(BitVector *self, uint32_t tick);
+
+ /** Invert each bit within a contiguous block.
+ *
+ * @param offset Lower bound.
+ * @param length The number of bits to flip.
+ */
+ public void
+ Flip_Block(BitVector *self, uint32_t offset, uint32_t length);
+
+ /** Return a count of the number of set bits.
+ */
+ public uint32_t
+ Count(BitVector *self);
+
+ /** Return an array where each element represents a set bit.
+ */
+ public incremented I32Array*
+ To_Array(BitVector *self);
+
+ public void
+ Destroy(BitVector *self);
+
+ public incremented BitVector*
+ Clone(BitVector *self);
+}
+
+
diff --git a/core/Lucy/Object/ByteBuf.c b/core/Lucy/Object/ByteBuf.c
new file mode 100644
index 0000000..0fcad0e
--- /dev/null
+++ b/core/Lucy/Object/ByteBuf.c
@@ -0,0 +1,266 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BYTEBUF
+#define C_LUCY_VIEWBYTEBUF
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Object/ByteBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Memory.h"
+
+static void
+S_grow(ByteBuf *self, size_t size);
+
+ByteBuf*
+BB_new(size_t capacity) {
+ ByteBuf *self = (ByteBuf*)VTable_Make_Obj(BYTEBUF);
+ return BB_init(self, capacity);
+}
+
+ByteBuf*
+BB_init(ByteBuf *self, size_t capacity) {
+ size_t amount = capacity ? capacity : sizeof(int64_t);
+ self->buf = NULL;
+ self->size = 0;
+ self->cap = 0;
+ S_grow(self, amount);
+ return self;
+}
+
+ByteBuf*
+BB_new_bytes(const void *bytes, size_t size) {
+ ByteBuf *self = (ByteBuf*)VTable_Make_Obj(BYTEBUF);
+ BB_init(self, size);
+ memcpy(self->buf, bytes, size);
+ self->size = size;
+ return self;
+}
+
+ByteBuf*
+BB_new_steal_bytes(void *bytes, size_t size, size_t capacity) {
+ ByteBuf *self = (ByteBuf*)VTable_Make_Obj(BYTEBUF);
+ self->buf = (char*)bytes;
+ self->size = size;
+ self->cap = capacity;
+ return self;
+}
+
+void
+BB_destroy(ByteBuf *self) {
+ FREEMEM(self->buf);
+ SUPER_DESTROY(self, BYTEBUF);
+}
+
+ByteBuf*
+BB_clone(ByteBuf *self) {
+ return BB_new_bytes(self->buf, self->size);
+}
+
+void
+BB_set_size(ByteBuf *self, size_t size) {
+ if (size > self->cap) {
+ THROW(ERR, "Can't set size to %u64 (greater than capacity of %u64)",
+ (uint64_t)size, (uint64_t)self->cap);
+ }
+ self->size = size;
+}
+
+char*
+BB_get_buf(ByteBuf *self) {
+ return self->buf;
+}
+
+size_t
+BB_get_size(ByteBuf *self) {
+ return self->size;
+}
+
+size_t
+BB_get_capacity(ByteBuf *self) {
+ return self->cap;
+}
+
+static INLINE bool_t
+SI_equals_bytes(ByteBuf *self, const void *bytes, size_t size) {
+ if (self->size != size) { return false; }
+ return (memcmp(self->buf, bytes, self->size) == 0);
+}
+
+bool_t
+BB_equals(ByteBuf *self, Obj *other) {
+ ByteBuf *const twin = (ByteBuf*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, BYTEBUF)) { return false; }
+ return SI_equals_bytes(self, twin->buf, twin->size);
+}
+
+bool_t
+BB_equals_bytes(ByteBuf *self, const void *bytes, size_t size) {
+ return SI_equals_bytes(self, bytes, size);
+}
+
+int32_t
+BB_hash_sum(ByteBuf *self) {
+ uint32_t sum = 5381;
+ uint8_t *const buf = (uint8_t*)self->buf;
+
+ for (size_t i = 0, max = self->size; i < max; i++) {
+ sum = ((sum << 5) + sum) ^ buf[i];
+ }
+
+ return (int32_t)sum;
+}
+
+static INLINE void
+SI_mimic_bytes(ByteBuf *self, const void *bytes, size_t size) {
+ if (size > self->cap) { S_grow(self, size); }
+ memmove(self->buf, bytes, size);
+ self->size = size;
+}
+
+void
+BB_mimic_bytes(ByteBuf *self, const void *bytes, size_t size) {
+ SI_mimic_bytes(self, bytes, size);
+}
+
+void
+BB_mimic(ByteBuf *self, Obj *other) {
+ ByteBuf *twin = (ByteBuf*)CERTIFY(other, BYTEBUF);
+ SI_mimic_bytes(self, twin->buf, twin->size);
+}
+
+static INLINE void
+SI_cat_bytes(ByteBuf *self, const void *bytes, size_t size) {
+ const size_t new_size = self->size + size;
+ if (new_size > self->cap) {
+ S_grow(self, Memory_oversize(new_size, sizeof(char)));
+ }
+ memcpy((self->buf + self->size), bytes, size);
+ self->size = new_size;
+}
+
+void
+BB_cat_bytes(ByteBuf *self, const void *bytes, size_t size) {
+ SI_cat_bytes(self, bytes, size);
+}
+
+void
+BB_cat(ByteBuf *self, const ByteBuf *other) {
+ SI_cat_bytes(self, other->buf, other->size);
+}
+
+static void
+S_grow(ByteBuf *self, size_t size) {
+ if (size > self->cap) {
+ size_t amount = size;
+ size_t remainder = amount % sizeof(int64_t);
+ if (remainder) {
+ amount += sizeof(int64_t);
+ amount -= remainder;
+ }
+ self->buf = (char*)REALLOCATE(self->buf, amount);
+ self->cap = amount;
+ }
+}
+
+char*
+BB_grow(ByteBuf *self, size_t size) {
+ if (size > self->cap) { S_grow(self, size); }
+ return self->buf;
+}
+
+void
+BB_serialize(ByteBuf *self, OutStream *target) {
+ OutStream_Write_C32(target, self->size);
+ OutStream_Write_Bytes(target, self->buf, self->size);
+}
+
+ByteBuf*
+BB_deserialize(ByteBuf *self, InStream *instream) {
+ const size_t size = InStream_Read_C32(instream);
+ const size_t capacity = size ? size : sizeof(int64_t);
+ self = self ? self : (ByteBuf*)VTable_Make_Obj(BYTEBUF);
+ if (capacity > self->cap) { S_grow(self, capacity); }
+ self->size = size;
+ InStream_Read_Bytes(instream, self->buf, size);
+ return self;
+}
+
+int
+BB_compare(const void *va, const void *vb) {
+ const ByteBuf *a = *(const ByteBuf**)va;
+ const ByteBuf *b = *(const ByteBuf**)vb;
+ const size_t size = a->size < b->size ? a->size : b->size;
+
+ int32_t comparison = memcmp(a->buf, b->buf, size);
+
+ if (comparison == 0 && a->size != b->size) {
+ comparison = a->size < b->size ? -1 : 1;
+ }
+
+ return comparison;
+}
+
+int32_t
+BB_compare_to(ByteBuf *self, Obj *other) {
+ CERTIFY(other, BYTEBUF);
+ return BB_compare(&self, &other);
+}
+
+/******************************************************************/
+
+ViewByteBuf*
+ViewBB_new(char *buf, size_t size) {
+ ViewByteBuf *self = (ViewByteBuf*)VTable_Make_Obj(VIEWBYTEBUF);
+ return ViewBB_init(self, buf, size);
+}
+
+ViewByteBuf*
+ViewBB_init(ViewByteBuf *self, char *buf, size_t size) {
+ self->cap = 0;
+ self->buf = buf;
+ self->size = size;
+ return self;
+}
+
+void
+ViewBB_destroy(ViewByteBuf *self) {
+ Obj_destroy((Obj*)self);
+}
+
+void
+ViewBB_assign_bytes(ViewByteBuf *self, char*buf, size_t size) {
+ self->buf = buf;
+ self->size = size;
+}
+
+void
+ViewBB_assign(ViewByteBuf *self, const ByteBuf *other) {
+ self->buf = other->buf;
+ self->size = other->size;
+}
+
+
diff --git a/core/Lucy/Object/ByteBuf.cfh b/core/Lucy/Object/ByteBuf.cfh
new file mode 100644
index 0000000..ea8f06b
--- /dev/null
+++ b/core/Lucy/Object/ByteBuf.cfh
@@ -0,0 +1,159 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Growable buffer holding arbitrary bytes.
+ */
+
+class Lucy::Object::ByteBuf cnick BB inherits Lucy::Object::Obj {
+
+ char *buf;
+ size_t size; /* number of valid bytes */
+ size_t cap; /* allocated bytes, including terminating null */
+
+ /**
+ * @param capacity initial capacity of the ByteBuf, in bytes.
+ */
+ inert incremented ByteBuf*
+ new(size_t capacity);
+
+ inert ByteBuf*
+ init(ByteBuf *self, size_t capacity);
+
+ /** Return a pointer to a new ByteBuf which holds a copy of the passed-in
+ * string.
+ */
+ inert incremented ByteBuf*
+ new_bytes(const void *bytes, size_t size);
+
+ /** Return a pointer to a new ByteBuf which assumes ownership of the
+ * passed-in string.
+ */
+ inert incremented ByteBuf*
+ new_steal_bytes(void *bytes, size_t size, size_t capacity);
+
+ /** Lexical comparison of two ByteBufs, with level of indirection set to
+ * please qsort and friends.
+ */
+ inert int
+ compare(const void *va, const void *vb);
+
+ /** Set the object's size member. If greater than the object's capacity,
+ * throws an error.
+ */
+ void
+ Set_Size(ByteBuf *self, size_t size);
+
+ /** Accessor for "size" member.
+ */
+ size_t
+ Get_Size(ByteBuf *self);
+
+ /** Accessor for raw internal buffer.
+ */
+ nullable char*
+ Get_Buf(ByteBuf *self);
+
+ /** Return the number of bytes in the Object's allocation.
+ */
+ size_t
+ Get_Capacity(ByteBuf *self);
+
+ public void
+ Mimic(ByteBuf *self, Obj *other);
+
+ void
+ Mimic_Bytes(ByteBuf *self, const void *bytes, size_t size);
+
+ /** Concatenate the passed-in bytes onto the end of the ByteBuf. Allocate
+ * more memory as needed.
+ */
+ void
+ Cat_Bytes(ByteBuf *self, const void *bytes, size_t size);
+
+ /** Concatenate the contents of <code>other</code> onto the end of the
+ * original ByteBuf. Allocate more memory as needed.
+ */
+ void
+ Cat(ByteBuf *self, const ByteBuf *other);
+
+ /** Assign more memory to the ByteBuf, if it doesn't already have enough
+ * room to hold <code>size</code> bytes. Cannot shrink the allocation.
+ *
+ * @return a pointer to the raw buffer.
+ */
+ nullable char*
+ Grow(ByteBuf *self, size_t size);
+
+ /** Test whether the ByteBuf matches the passed-in bytes.
+ */
+ bool_t
+ Equals_Bytes(ByteBuf *self, const void *bytes, size_t size);
+
+ public int32_t
+ Compare_To(ByteBuf *self, Obj *other);
+
+ public incremented ByteBuf*
+ Clone(ByteBuf *self);
+
+ public void
+ Destroy(ByteBuf *self);
+
+ public bool_t
+ Equals(ByteBuf *self, Obj *other);
+
+ public int32_t
+ Hash_Sum(ByteBuf *self);
+
+ public void
+ Serialize(ByteBuf *self, OutStream *outstream);
+
+ public incremented ByteBuf*
+ Deserialize(ByteBuf *self, InStream *instream);
+}
+
+/**
+ * A ByteBuf that doesn't own its own string.
+ */
+class Lucy::Object::ViewByteBuf cnick ViewBB
+ inherits Lucy::Object::ByteBuf {
+
+ /** Return a pointer to a new "view" ByteBuf, offing a persective on the
+ * passed-in string.
+ */
+ inert incremented ViewByteBuf*
+ new(char *buf, size_t size);
+
+ inert incremented ViewByteBuf*
+ init(ViewByteBuf *self, char *buf, size_t size);
+
+ /** Assign buf and size members to the passed in values.
+ */
+ void
+ Assign_Bytes(ViewByteBuf *self, char *buf, size_t size);
+
+ /** Assign buf and size members from the passed-in ByteBuf.
+ */
+ void
+ Assign(ViewByteBuf *self, const ByteBuf *other);
+
+ public void
+ Destroy(ViewByteBuf *self);
+}
+
+
diff --git a/core/Lucy/Object/CharBuf.c b/core/Lucy/Object/CharBuf.c
new file mode 100644
index 0000000..ecff852
--- /dev/null
+++ b/core/Lucy/Object/CharBuf.c
@@ -0,0 +1,957 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_CHARBUF
+#define C_LUCY_VIEWCHARBUF
+#define C_LUCY_ZOMBIECHARBUF
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Object/CharBuf.h"
+
+#include "Lucy/Object/Err.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/StringHelper.h"
+
+// Helper function for throwing invalid UTF-8 error. Since THROW uses
+// a CharBuf internally, calling THROW with invalid UTF-8 would create an
+// infinite loop -- so we fwrite some of the bogus text to stderr and
+// invoke THROW with a generic message.
+#define DIE_INVALID_UTF8(text, size) \
+ S_die_invalid_utf8(text, size, __FILE__, __LINE__, CFISH_ERR_FUNC_MACRO)
+static void
+S_die_invalid_utf8(const char *text, size_t size, const char *file, int line,
+ const char *func);
+
+// Helper function for throwing invalid pattern error.
+static void
+S_die_invalid_pattern(const char *pattern);
+
+ZombieCharBuf EMPTY = { ZOMBIECHARBUF, {1}, "", 0, 0 };
+
+CharBuf*
+CB_new(size_t size) {
+ CharBuf *self = (CharBuf*)VTable_Make_Obj(CHARBUF);
+ return CB_init(self, size);
+}
+
+CharBuf*
+CB_init(CharBuf *self, size_t size) {
+ // Derive.
+ self->ptr = (char*)MALLOCATE(size + 1);
+
+ // Init.
+ *self->ptr = '\0'; // Empty string.
+
+ // Assign.
+ self->size = 0;
+ self->cap = size + 1;
+
+ return self;
+}
+
+CharBuf*
+CB_new_from_utf8(const char *ptr, size_t size) {
+ if (!StrHelp_utf8_valid(ptr, size)) {
+ DIE_INVALID_UTF8(ptr, size);
+ }
+ return CB_new_from_trusted_utf8(ptr, size);
+}
+
+CharBuf*
+CB_new_from_trusted_utf8(const char *ptr, size_t size) {
+ CharBuf *self = (CharBuf*)VTable_Make_Obj(CHARBUF);
+
+ // Derive.
+ self->ptr = (char*)MALLOCATE(size + 1);
+
+ // Copy.
+ memcpy(self->ptr, ptr, size);
+
+ // Assign.
+ self->size = size;
+ self->cap = size + 1;
+ self->ptr[size] = '\0'; // Null terminate.
+
+ return self;
+}
+
+CharBuf*
+CB_new_steal_from_trusted_str(char *ptr, size_t size, size_t cap) {
+ CharBuf *self = (CharBuf*)VTable_Make_Obj(CHARBUF);
+ self->ptr = ptr;
+ self->size = size;
+ self->cap = cap;
+ return self;
+}
+
+CharBuf*
+CB_new_steal_str(char *ptr, size_t size, size_t cap) {
+ StrHelp_utf8_valid(ptr, size);
+ return CB_new_steal_from_trusted_str(ptr, size, cap);
+}
+
+CharBuf*
+CB_newf(const char *pattern, ...) {
+ CharBuf *self = CB_new(strlen(pattern));
+ va_list args;
+ va_start(args, pattern);
+ CB_VCatF(self, pattern, args);
+ va_end(args);
+ return self;
+}
+
+void
+CB_destroy(CharBuf *self) {
+ FREEMEM(self->ptr);
+ SUPER_DESTROY(self, CHARBUF);
+}
+
+int32_t
+CB_hash_sum(CharBuf *self) {
+ uint32_t hashvalue = 5381;
+ ZombieCharBuf *iterator = ZCB_WRAP(self);
+
+ {
+ const CB_nip_one_t nip_one
+ = (CB_nip_one_t)METHOD(iterator->vtable, CB, Nip_One);
+ while (iterator->size) {
+ uint32_t code_point = (uint32_t)nip_one((CharBuf*)iterator);
+ hashvalue = ((hashvalue << 5) + hashvalue) ^ code_point;
+ }
+ }
+
+ return (int32_t) hashvalue;
+}
+
+static void
+S_grow(CharBuf *self, size_t size) {
+ if (size >= self->cap) {
+ CB_Grow(self, size);
+ }
+}
+
+char*
+CB_grow(CharBuf *self, size_t size) {
+ if (size >= self->cap) {
+ self->cap = size + 1;
+ self->ptr = (char*)REALLOCATE(self->ptr, self->cap);
+ }
+ return self->ptr;
+}
+
+static void
+S_die_invalid_utf8(const char *text, size_t size, const char *file, int line,
+ const char *func) {
+ fprintf(stderr, "Invalid UTF-8, aborting: '");
+ fwrite(text, sizeof(char), size < 200 ? size : 200, stderr);
+ if (size > 200) { fwrite("[...]", sizeof(char), 5, stderr); }
+ fprintf(stderr, "' (length %lu)\n", (unsigned long)size);
+ Err_throw_at(ERR, file, line, func, "Invalid UTF-8");
+}
+
+static void
+S_die_invalid_pattern(const char *pattern) {
+ size_t pattern_len = strlen(pattern);
+ fprintf(stderr, "Invalid pattern, aborting: '");
+ fwrite(pattern, sizeof(char), pattern_len, stderr);
+ fprintf(stderr, "'\n");
+ THROW(ERR, "Invalid pattern.");
+}
+
+void
+CB_setf(CharBuf *self, const char *pattern, ...) {
+ va_list args;
+ CB_Set_Size(self, 0);
+ va_start(args, pattern);
+ CB_VCatF(self, pattern, args);
+ va_end(args);
+}
+
+void
+CB_catf(CharBuf *self, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ CB_VCatF(self, pattern, args);
+ va_end(args);
+}
+
+void
+CB_vcatf(CharBuf *self, const char *pattern, va_list args) {
+ size_t pattern_len = strlen(pattern);
+ const char *pattern_start = pattern;
+ const char *pattern_end = pattern + pattern_len;
+ char buf[64];
+
+ for (; pattern < pattern_end; pattern++) {
+ const char *slice_end = pattern;
+
+ // Consume all characters leading up to a '%'.
+ while (slice_end < pattern_end && *slice_end != '%') { slice_end++; }
+ if (pattern != slice_end) {
+ size_t size = slice_end - pattern;
+ CB_Cat_Trusted_Str(self, pattern, size);
+ pattern = slice_end;
+ }
+
+ if (pattern < pattern_end) {
+ pattern++; // Move past '%'.
+
+ switch (*pattern) {
+ case '%': {
+ CB_Cat_Trusted_Str(self, "%", 1);
+ }
+ break;
+ case 'o': {
+ Obj *obj = va_arg(args, Obj*);
+ if (!obj) {
+ CB_Cat_Trusted_Str(self, "[NULL]", 6);
+ }
+ else if (Obj_Is_A(obj, CHARBUF)) {
+ CB_Cat(self, (CharBuf*)obj);
+ }
+ else {
+ CharBuf *string = Obj_To_String(obj);
+ CB_Cat(self, string);
+ DECREF(string);
+ }
+ }
+ break;
+ case 'i': {
+ int64_t val = 0;
+ size_t size;
+ if (pattern[1] == '8') {
+ val = va_arg(args, int32_t);
+ pattern++;
+ }
+ else if (pattern[1] == '3' && pattern[2] == '2') {
+ val = va_arg(args, int32_t);
+ pattern += 2;
+ }
+ else if (pattern[1] == '6' && pattern[2] == '4') {
+ val = va_arg(args, int64_t);
+ pattern += 2;
+ }
+ else {
+ S_die_invalid_pattern(pattern_start);
+ }
+ size = sprintf(buf, "%" I64P, val);
+ CB_Cat_Trusted_Str(self, buf, size);
+ }
+ break;
+ case 'u': {
+ uint64_t val = 0;
+ size_t size;
+ if (pattern[1] == '8') {
+ val = va_arg(args, uint32_t);
+ pattern += 1;
+ }
+ else if (pattern[1] == '3' && pattern[2] == '2') {
+ val = va_arg(args, uint32_t);
+ pattern += 2;
+ }
+ else if (pattern[1] == '6' && pattern[2] == '4') {
+ val = va_arg(args, uint64_t);
+ pattern += 2;
+ }
+ else {
+ S_die_invalid_pattern(pattern_start);
+ }
+ size = sprintf(buf, "%" U64P, val);
+ CB_Cat_Trusted_Str(self, buf, size);
+ }
+ break;
+ case 'f': {
+ if (pattern[1] == '6' && pattern[2] == '4') {
+ double num = va_arg(args, double);
+ char bigbuf[512];
+ size_t size = sprintf(bigbuf, "%g", num);
+ CB_Cat_Trusted_Str(self, bigbuf, size);
+ pattern += 2;
+ }
+ else {
+ S_die_invalid_pattern(pattern_start);
+ }
+ }
+ break;
+ case 'x': {
+ if (pattern[1] == '3' && pattern[2] == '2') {
+ unsigned long val = va_arg(args, uint32_t);
+ size_t size = sprintf(buf, "%.8lx", val);
+ CB_Cat_Trusted_Str(self, buf, size);
+ pattern += 2;
+ }
+ else {
+ S_die_invalid_pattern(pattern_start);
+ }
+ }
+ break;
+ case 's': {
+ char *string = va_arg(args, char*);
+ if (string == NULL) {
+ CB_Cat_Trusted_Str(self, "[NULL]", 6);
+ }
+ else {
+ size_t size = strlen(string);
+ if (StrHelp_utf8_valid(string, size)) {
+ CB_Cat_Trusted_Str(self, string, size);
+ }
+ else {
+ CB_Cat_Trusted_Str(self, "[INVALID UTF8]", 14);
+ }
+ }
+ }
+ break;
+ default: {
+ // Assume NULL-terminated pattern string, which
+ // eliminates the need for bounds checking if '%' is
+ // the last visible character.
+ S_die_invalid_pattern(pattern_start);
+ }
+ }
+ }
+ }
+}
+
+CharBuf*
+CB_to_string(CharBuf *self) {
+ return CB_new_from_trusted_utf8(self->ptr, self->size);
+}
+
+void
+CB_cat_char(CharBuf *self, uint32_t code_point) {
+ const size_t MAX_UTF8_BYTES = 4;
+ if (self->size + MAX_UTF8_BYTES >= self->cap) {
+ S_grow(self, Memory_oversize(self->size + MAX_UTF8_BYTES,
+ sizeof(char)));
+ }
+ char *end = self->ptr + self->size;
+ size_t count = StrHelp_encode_utf8_char(code_point, (uint8_t*)end);
+ self->size += count;
+ *(end + count) = '\0';
+}
+
+int32_t
+CB_swap_chars(CharBuf *self, uint32_t match, uint32_t replacement) {
+ int32_t num_swapped = 0;
+
+ if (match > 127) {
+ THROW(ERR, "match point too high: %u32", match);
+ }
+ else if (replacement > 127) {
+ THROW(ERR, "replacement code point too high: %u32", replacement);
+ }
+ else {
+ char *ptr = self->ptr;
+ char *const limit = ptr + self->size;
+ for (; ptr < limit; ptr++) {
+ if (*ptr == (char)match) {
+ *ptr = (char)replacement;
+ num_swapped++;
+ }
+ }
+ }
+
+ return num_swapped;
+}
+
+int64_t
+CB_to_i64(CharBuf *self) {
+ return CB_BaseX_To_I64(self, 10);
+}
+
+int64_t
+CB_basex_to_i64(CharBuf *self, uint32_t base) {
+ ZombieCharBuf *iterator = ZCB_WRAP(self);
+ int64_t retval = 0;
+ bool_t is_negative = false;
+
+ // Advance past minus sign.
+ if (ZCB_Code_Point_At(iterator, 0) == '-') {
+ ZCB_Nip_One(iterator);
+ is_negative = true;
+ }
+
+ // Accumulate.
+ while (iterator->size) {
+ int32_t code_point = ZCB_Nip_One(iterator);
+ if (isalnum(code_point)) {
+ int32_t addend = isdigit(code_point)
+ ? code_point - '0'
+ : tolower(code_point) - 'a' + 10;
+ if (addend > (int32_t)base) { break; }
+ retval *= base;
+ retval += addend;
+ }
+ else {
+ break;
+ }
+ }
+
+ // Apply minus sign.
+ if (is_negative) { retval = 0 - retval; }
+
+ return retval;
+}
+
+static double
+S_safe_to_f64(CharBuf *self) {
+ size_t amount = self->size < 511 ? self->size : 511;
+ char buf[512];
+ memcpy(buf, self->ptr, amount);
+ buf[amount] = 0; // NULL-terminate.
+ return strtod(buf, NULL);
+}
+
+double
+CB_to_f64(CharBuf *self) {
+ char *end;
+ double value = strtod(self->ptr, &end);
+ size_t consumed = end - self->ptr;
+ if (consumed > self->size) { // strtod overran
+ value = S_safe_to_f64(self);
+ }
+ return value;
+}
+
+CharBuf*
+CB_to_cb8(CharBuf *self) {
+ return CB_new_from_trusted_utf8(self->ptr, self->size);
+}
+
+CharBuf*
+CB_clone(CharBuf *self) {
+ return CB_new_from_trusted_utf8(self->ptr, self->size);
+}
+
+CharBuf*
+CB_load(CharBuf *self, Obj *dump) {
+ CharBuf *source = (CharBuf*)CERTIFY(dump, CHARBUF);
+ UNUSED_VAR(self);
+ return CB_Clone(source);
+}
+
+void
+CB_serialize(CharBuf *self, OutStream *target) {
+ OutStream_Write_C32(target, self->size);
+ OutStream_Write_Bytes(target, self->ptr, self->size);
+}
+
+CharBuf*
+CB_deserialize(CharBuf *self, InStream *instream) {
+ size_t size = InStream_Read_C32(instream);
+ self = self ? self : (CharBuf*)VTable_Make_Obj(CHARBUF);
+ if (size >= self->cap) { S_grow(self, size); }
+ InStream_Read_Bytes(instream, self->ptr, size);
+ self->size = size;
+ self->ptr[size] = '\0';
+ if (!StrHelp_utf8_valid(self->ptr, size)) {
+ DIE_INVALID_UTF8(self->ptr, size);
+ }
+ return self;
+}
+
+void
+CB_mimic_str(CharBuf *self, const char* ptr, size_t size) {
+ if (!StrHelp_utf8_valid(ptr, size)) {
+ DIE_INVALID_UTF8(ptr, size);
+ }
+ if (size >= self->cap) { S_grow(self, size); }
+ memmove(self->ptr, ptr, size);
+ self->size = size;
+ self->ptr[size] = '\0';
+}
+
+void
+CB_mimic(CharBuf *self, Obj *other) {
+ CharBuf *twin = (CharBuf*)CERTIFY(other, CHARBUF);
+ if (twin->size >= self->cap) { S_grow(self, twin->size); }
+ memmove(self->ptr, twin->ptr, twin->size);
+ self->size = twin->size;
+ self->ptr[twin->size] = '\0';
+}
+
+void
+CB_cat_str(CharBuf *self, const char* ptr, size_t size) {
+ if (!StrHelp_utf8_valid(ptr, size)) {
+ DIE_INVALID_UTF8(ptr, size);
+ }
+ CB_cat_trusted_str(self, ptr, size);
+}
+
+void
+CB_cat_trusted_str(CharBuf *self, const char* ptr, size_t size) {
+ const size_t new_size = self->size + size;
+ if (new_size >= self->cap) {
+ size_t amount = Memory_oversize(new_size, sizeof(char));
+ S_grow(self, amount);
+ }
+ memcpy((self->ptr + self->size), ptr, size);
+ self->size = new_size;
+ self->ptr[new_size] = '\0';
+}
+
+void
+CB_cat(CharBuf *self, const CharBuf *other) {
+ const size_t new_size = self->size + other->size;
+ if (new_size >= self->cap) {
+ size_t amount = Memory_oversize(new_size, sizeof(char));
+ S_grow(self, amount);
+ }
+ memcpy((self->ptr + self->size), other->ptr, other->size);
+ self->size = new_size;
+ self->ptr[new_size] = '\0';
+}
+
+bool_t
+CB_starts_with(CharBuf *self, const CharBuf *prefix) {
+ return CB_starts_with_str(self, prefix->ptr, prefix->size);
+}
+
+bool_t
+CB_starts_with_str(CharBuf *self, const char *prefix, size_t size) {
+ if (size <= self->size
+ && (memcmp(self->ptr, prefix, size) == 0)
+ ) {
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+bool_t
+CB_equals(CharBuf *self, Obj *other) {
+ CharBuf *const twin = (CharBuf*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, CHARBUF)) { return false; }
+ return CB_equals_str(self, twin->ptr, twin->size);
+}
+
+int32_t
+CB_compare_to(CharBuf *self, Obj *other) {
+ CERTIFY(other, CHARBUF);
+ return CB_compare(&self, &other);
+}
+
+bool_t
+CB_equals_str(CharBuf *self, const char *ptr, size_t size) {
+ if (self->size != size) {
+ return false;
+ }
+ return (memcmp(self->ptr, ptr, self->size) == 0);
+}
+
+bool_t
+CB_ends_with(CharBuf *self, const CharBuf *postfix) {
+ return CB_ends_with_str(self, postfix->ptr, postfix->size);
+}
+
+bool_t
+CB_ends_with_str(CharBuf *self, const char *postfix, size_t postfix_len) {
+ if (postfix_len <= self->size) {
+ char *start = self->ptr + self->size - postfix_len;
+ if (memcmp(start, postfix, postfix_len) == 0) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+int64_t
+CB_find(CharBuf *self, const CharBuf *substring) {
+ return CB_Find_Str(self, substring->ptr, substring->size);
+}
+
+int64_t
+CB_find_str(CharBuf *self, const char *ptr, size_t size) {
+ ZombieCharBuf *iterator = ZCB_WRAP(self);
+ int64_t location = 0;
+
+ while (iterator->size) {
+ if (ZCB_Starts_With_Str(iterator, ptr, size)) {
+ return location;
+ }
+ ZCB_Nip(iterator, 1);
+ location++;
+ }
+
+ return -1;
+}
+
+uint32_t
+CB_trim(CharBuf *self) {
+ return CB_Trim_Top(self) + CB_Trim_Tail(self);
+}
+
+uint32_t
+CB_trim_top(CharBuf *self) {
+ char *ptr = self->ptr;
+ char *end = ptr + self->size;
+ uint32_t count = 0;
+
+ while (ptr < end) {
+ uint32_t code_point = StrHelp_decode_utf8_char(ptr);
+ if (!StrHelp_is_whitespace(code_point)) { break; }
+ ptr += StrHelp_UTF8_COUNT[*(uint8_t*)ptr];
+ count++;
+ }
+
+ if (count) {
+ // Copy string backwards.
+ self->size = end - ptr;
+ memmove(self->ptr, ptr, self->size);
+ }
+
+ return count;
+}
+
+uint32_t
+CB_trim_tail(CharBuf *self) {
+ uint32_t count = 0;
+ char *const top = self->ptr;
+ const char *ptr = top + self->size;
+ size_t new_size = self->size;
+
+ while (NULL != (ptr = StrHelp_back_utf8_char(ptr, top))) {
+ uint32_t code_point = StrHelp_decode_utf8_char(ptr);
+ if (!StrHelp_is_whitespace(code_point)) { break; }
+ new_size = ptr - top;
+ count++;
+ }
+ self->size = new_size;
+
+ return count;
+}
+
+size_t
+CB_nip(CharBuf *self, size_t count) {
+ size_t num_nipped = 0;
+ char *ptr = self->ptr;
+ char *const end = ptr + self->size;
+ for (; ptr < end && count--; ptr += StrHelp_UTF8_COUNT[*(uint8_t*)ptr]) {
+ num_nipped++;
+ }
+ self->size = end - ptr;
+ memmove(self->ptr, ptr, self->size);
+ return num_nipped;
+}
+
+int32_t
+CB_nip_one(CharBuf *self) {
+ if (self->size == 0) {
+ return 0;
+ }
+ else {
+ int32_t retval = (int32_t)StrHelp_decode_utf8_char(self->ptr);
+ size_t consumed = StrHelp_UTF8_COUNT[*(uint8_t*)self->ptr];
+ char *ptr = self->ptr + StrHelp_UTF8_COUNT[*(uint8_t*)self->ptr];
+ self->size -= consumed;
+ memmove(self->ptr, ptr, self->size);
+ return retval;
+ }
+}
+
+size_t
+CB_chop(CharBuf *self, size_t count) {
+ size_t num_chopped = 0;
+ char *top = self->ptr;
+ const char *ptr = top + self->size;
+ for (num_chopped = 0; num_chopped < count; num_chopped++) {
+ const char *end = ptr;
+ if (NULL == (ptr = StrHelp_back_utf8_char(ptr, top))) { break; }
+ self->size -= (end - ptr);
+ }
+ return num_chopped;
+}
+
+size_t
+CB_length(CharBuf *self) {
+ size_t len = 0;
+ char *ptr = self->ptr;
+ char *end = ptr + self->size;
+ while (ptr < end) {
+ ptr += StrHelp_UTF8_COUNT[*(uint8_t*)ptr];
+ len++;
+ }
+ return len;
+}
+
+size_t
+CB_truncate(CharBuf *self, size_t count) {
+ uint32_t num_code_points;
+ ZombieCharBuf *iterator = ZCB_WRAP(self);
+ num_code_points = ZCB_Nip(iterator, count);
+ self->size -= iterator->size;
+ return num_code_points;
+}
+
+uint32_t
+CB_code_point_at(CharBuf *self, size_t tick) {
+ size_t count = 0;
+ char *ptr = self->ptr;
+ char *const end = ptr + self->size;
+
+ for (; ptr < end; ptr += StrHelp_UTF8_COUNT[*(uint8_t*)ptr]) {
+ if (count == tick) { return StrHelp_decode_utf8_char(ptr); }
+ count++;
+ }
+
+ return 0;
+}
+
+uint32_t
+CB_code_point_from(CharBuf *self, size_t tick) {
+ size_t count = 0;
+ char *top = self->ptr;
+ const char *ptr = top + self->size;
+
+ for (count = 0; count < tick; count++) {
+ if (NULL == (ptr = StrHelp_back_utf8_char(ptr, top))) { return 0; }
+ }
+ return StrHelp_decode_utf8_char(ptr);
+}
+
+CharBuf*
+CB_substring(CharBuf *self, size_t offset, size_t len) {
+ ZombieCharBuf *iterator = ZCB_WRAP(self);
+ char *sub_start;
+ size_t byte_len;
+
+ ZCB_Nip(iterator, offset);
+ sub_start = iterator->ptr;
+ ZCB_Nip(iterator, len);
+ byte_len = iterator->ptr - sub_start;
+
+ return CB_new_from_trusted_utf8(sub_start, byte_len);
+}
+
+int
+CB_compare(const void *va, const void *vb) {
+ const CharBuf *a = *(const CharBuf**)va;
+ const CharBuf *b = *(const CharBuf**)vb;
+ ZombieCharBuf *iterator_a = ZCB_WRAP(a);
+ ZombieCharBuf *iterator_b = ZCB_WRAP(b);
+ while (iterator_a->size && iterator_b->size) {
+ int32_t code_point_a = ZCB_Nip_One(iterator_a);
+ int32_t code_point_b = ZCB_Nip_One(iterator_b);
+ const int32_t comparison = code_point_a - code_point_b;
+ if (comparison != 0) { return comparison; }
+ }
+ if (iterator_a->size != iterator_b->size) {
+ return iterator_a->size < iterator_b->size ? -1 : 1;
+ }
+ return 0;
+}
+
+bool_t
+CB_less_than(const void *va, const void *vb) {
+ return CB_compare(va, vb) < 0 ? 1 : 0;
+}
+
+void
+CB_set_size(CharBuf *self, size_t size) {
+ self->size = size;
+}
+
+size_t
+CB_get_size(CharBuf *self) {
+ return self->size;
+}
+
+uint8_t*
+CB_get_ptr8(CharBuf *self) {
+ return (uint8_t*)self->ptr;
+}
+
+/*****************************************************************/
+
+ViewCharBuf*
+ViewCB_new_from_utf8(const char *utf8, size_t size) {
+ if (!StrHelp_utf8_valid(utf8, size)) {
+ DIE_INVALID_UTF8(utf8, size);
+ }
+ return ViewCB_new_from_trusted_utf8(utf8, size);
+}
+
+ViewCharBuf*
+ViewCB_new_from_trusted_utf8(const char *utf8, size_t size) {
+ ViewCharBuf *self = (ViewCharBuf*)VTable_Make_Obj(VIEWCHARBUF);
+ return ViewCB_init(self, utf8, size);
+}
+
+ViewCharBuf*
+ViewCB_init(ViewCharBuf *self, const char *utf8, size_t size) {
+ self->ptr = (char*)utf8;
+ self->size = size;
+ self->cap = 0;
+ return self;
+}
+
+void
+ViewCB_destroy(ViewCharBuf *self) {
+ // Note that we do not free self->ptr, and that we invoke the
+ // SUPER_DESTROY with CHARBUF instead of VIEWCHARBUF.
+ SUPER_DESTROY(self, CHARBUF);
+}
+
+void
+ViewCB_assign(ViewCharBuf *self, const CharBuf *other) {
+ self->ptr = other->ptr;
+ self->size = other->size;
+}
+
+void
+ViewCB_assign_str(ViewCharBuf *self, const char *utf8, size_t size) {
+ if (!StrHelp_utf8_valid(utf8, size)) {
+ DIE_INVALID_UTF8(utf8, size);
+ }
+ self->ptr = (char*)utf8;
+ self->size = size;
+}
+
+void
+ViewCB_assign_trusted_str(ViewCharBuf *self, const char *utf8, size_t size) {
+ self->ptr = (char*)utf8;
+ self->size = size;
+}
+
+uint32_t
+ViewCB_trim_top(ViewCharBuf *self) {
+ uint32_t count = 0;
+ char *ptr = self->ptr;
+ char *end = ptr + self->size;
+
+ while (ptr < end) {
+ uint32_t code_point = StrHelp_decode_utf8_char(ptr);
+ if (!StrHelp_is_whitespace(code_point)) { break; }
+ ptr += StrHelp_UTF8_COUNT[*(uint8_t*)ptr];
+ count++;
+ }
+
+ if (count) {
+ self->size = end - ptr;
+ self->ptr = ptr;
+ }
+
+ return count;
+}
+
+size_t
+ViewCB_nip(ViewCharBuf *self, size_t count) {
+ size_t num_nipped;
+ char *ptr = self->ptr;
+ char *end = ptr + self->size;
+ for (num_nipped = 0;
+ ptr < end && count--;
+ ptr += StrHelp_UTF8_COUNT[*(uint8_t*)ptr]
+ ) {
+ num_nipped++;
+ }
+ self->size = end - ptr;
+ self->ptr = ptr;
+ return num_nipped;
+}
+
+int32_t
+ViewCB_nip_one(ViewCharBuf *self) {
+ if (self->size == 0) {
+ return 0;
+ }
+ else {
+ int32_t retval = (int32_t)StrHelp_decode_utf8_char(self->ptr);
+ size_t consumed = StrHelp_UTF8_COUNT[*(uint8_t*)self->ptr];
+ self->ptr += consumed;
+ self->size -= consumed;
+ return retval;
+ }
+}
+
+char*
+ViewCB_grow(ViewCharBuf *self, size_t size) {
+ UNUSED_VAR(size);
+ THROW(ERR, "Can't grow a ViewCharBuf ('%o')", self);
+ UNREACHABLE_RETURN(char*);
+}
+
+/*****************************************************************/
+
+ZombieCharBuf*
+ZCB_new(void *allocation) {
+ static char empty_string[] = "";
+ ZombieCharBuf *self = (ZombieCharBuf*)allocation;
+ self->ref.count = 1;
+ self->vtable = ZOMBIECHARBUF;
+ self->cap = 0;
+ self->size = 0;
+ self->ptr = empty_string;
+ return self;
+}
+
+ZombieCharBuf*
+ZCB_newf(void *allocation, size_t alloc_size, const char *pattern, ...) {
+ ZombieCharBuf *self = (ZombieCharBuf*)allocation;
+
+ self->ref.count = 1;
+ self->vtable = ZOMBIECHARBUF;
+ self->cap = alloc_size - sizeof(ZombieCharBuf);
+ self->size = 0;
+ self->ptr = ((char*)allocation) + sizeof(ZombieCharBuf);
+
+ va_list args;
+ va_start(args, pattern);
+ ZCB_VCatF(self, pattern, args);
+ va_end(args);
+
+ return self;
+}
+
+ZombieCharBuf*
+ZCB_wrap_str(void *allocation, const char *ptr, size_t size) {
+ ZombieCharBuf *self = (ZombieCharBuf*)allocation;
+ self->ref.count = 1;
+ self->vtable = ZOMBIECHARBUF;
+ self->cap = 0;
+ self->size = size;
+ self->ptr = (char*)ptr;
+ return self;
+}
+
+ZombieCharBuf*
+ZCB_wrap(void *allocation, const CharBuf *source) {
+ return ZCB_wrap_str(allocation, source->ptr, source->size);
+}
+
+size_t
+ZCB_size() {
+ return sizeof(ZombieCharBuf);
+}
+
+void
+ZCB_destroy(ZombieCharBuf *self) {
+ THROW(ERR, "Can't destroy a ZombieCharBuf ('%o')", self);
+}
+
+
diff --git a/core/Lucy/Object/CharBuf.cfh b/core/Lucy/Object/CharBuf.cfh
new file mode 100644
index 0000000..10820a9
--- /dev/null
+++ b/core/Lucy/Object/CharBuf.cfh
@@ -0,0 +1,416 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+#include <stdarg.h>
+__END_C__
+
+/**
+ * Growable buffer holding Unicode characters.
+ */
+
+class Lucy::Object::CharBuf cnick CB
+ inherits Lucy::Object::Obj {
+
+ char *ptr;
+ size_t size;
+ size_t cap; /* allocated bytes, including terminating null */
+
+ inert incremented CharBuf*
+ new(size_t size);
+
+ inert CharBuf*
+ init(CharBuf *self, size_t size);
+
+ /** Return a new CharBuf which holds a copy of the passed-in string.
+ * Check for UTF-8 validity.
+ */
+ inert incremented CharBuf*
+ new_from_utf8(const char *utf8, size_t size);
+
+ /** Return a new CharBuf which holds a copy of the passed-in string. No
+ * validity checking is performed.
+ */
+ inert incremented CharBuf*
+ new_from_trusted_utf8(const char *utf8, size_t size);
+
+ /** Return a pointer to a new CharBuf which assumes ownership of the
+ * passed-in string. Check validity of supplied UTF-8.
+ */
+ inert incremented CharBuf*
+ new_steal_str(char *ptr, size_t size, size_t cap);
+
+ /** Return a pointer to a new CharBuf which assumes ownership of the
+ * passed-in string. Do not check validity of supplied UTF-8.
+ */
+ inert incremented CharBuf*
+ new_steal_from_trusted_str(char *ptr, size_t size, size_t cap);
+
+ /** Return a pointer to a new CharBuf which contains formatted data
+ * expanded according to CB_VCatF.
+ *
+ * Note: a user-supplied <code>pattern</code> string is a security hole
+ * and must not be allowed.
+ */
+ inert incremented CharBuf*
+ newf(const char *pattern, ...);
+
+ /** Perform lexical comparison of two CharBufs, with level of indirection
+ * set to please qsort and friends.
+ */
+ inert int
+ compare(const void *va, const void *vb);
+
+ /** Perform lexical comparison of two CharBufs, with level of indirection
+ * set to please qsort and friends, and return true if <code>a</code> is
+ * less than <code>b</code>.
+ */
+ inert bool_t
+ less_than(const void *va, const void *vb);
+
+ public void
+ Mimic(CharBuf *self, Obj *other);
+
+ void
+ Mimic_Str(CharBuf *self, const char *ptr, size_t size);
+
+ /** Concatenate the passed-in string onto the end of the CharBuf.
+ */
+ void
+ Cat_Str(CharBuf *self, const char *ptr, size_t size);
+
+ /** Concatenate the contents of <code>other</code> onto the end of the
+ * caller.
+ */
+ void
+ Cat(CharBuf *self, const CharBuf *other);
+
+ /** Concatenate formatted arguments. Similar to the printf family, but
+ * only accepts minimal options (just enough for decent error messages).
+ *
+ * Objects: %o
+ * char*: %s
+ * integers: %i8 %i32 %i64 %u8 %u32 %u64
+ * floats: %f64
+ * hex: %x32
+ *
+ * Note that all Clownfish Objects, including CharBufs, are printed via
+ * %o (which invokes Obj_To_String()).
+ */
+ void
+ VCatF(CharBuf *self, const char *pattern, va_list args);
+
+ /** Invokes CB_VCatF to concatenate formatted arguments. Note that this
+ * is only a function and not a method.
+ */
+ inert void
+ catf(CharBuf *self, const char *pattern, ...);
+
+ /** Replaces the contents of the CharBuf using formatted arguments.
+ */
+ inert void
+ setf(CharBuf *self, const char *pattern, ...);
+
+ /** Concatenate one Unicode character onto the end of the CharBuf.
+ */
+ void
+ Cat_Char(CharBuf *self, uint32_t code_point);
+
+ /** Replace all instances of one character for the other. For now, both
+ * the source and replacement code points must be ASCII.
+ */
+ int32_t
+ Swap_Chars(CharBuf *self, uint32_t match, uint32_t replacement);
+
+ public int64_t
+ To_I64(CharBuf *self);
+
+ /** Extract a 64-bit integer from a variable-base stringified version.
+ */
+ int64_t
+ BaseX_To_I64(CharBuf *self, uint32_t base);
+
+ public double
+ To_F64(CharBuf *self);
+
+ /** Assign more memory to the CharBuf, if it doesn't already have enough
+ * room to hold a string of <code>size</code> bytes. Cannot shrink the
+ * allocation.
+ *
+ * @return a pointer to the raw buffer.
+ */
+ char*
+ Grow(CharBuf *self, size_t size);
+
+ /** Test whether the CharBuf starts with the content of another.
+ */
+ bool_t
+ Starts_With(CharBuf *self, const CharBuf *prefix);
+
+ /** Test whether the CharBuf starts with the passed-in string.
+ */
+ bool_t
+ Starts_With_Str(CharBuf *self, const char *prefix, size_t size);
+
+ /** Test whether the CharBuf ends with the content of another.
+ */
+ bool_t
+ Ends_With(CharBuf *self, const CharBuf *postfix);
+
+ /** Test whether the CharBuf ends with the passed-in string.
+ */
+ bool_t
+ Ends_With_Str(CharBuf *self, const char *postfix, size_t size);
+
+ /** Return the location of the substring within the CharBuf (measured in
+ * code points), or -1 if the substring does not match.
+ */
+ int64_t
+ Find(CharBuf *self, const CharBuf *substring);
+
+ int64_t
+ Find_Str(CharBuf *self, const char *ptr, size_t size);
+
+ /** Test whether the CharBuf matches the passed-in string.
+ */
+ bool_t
+ Equals_Str(CharBuf *self, const char *ptr, size_t size);
+
+ /** Return the number of Unicode code points in the object's string.
+ */
+ size_t
+ Length(CharBuf *self);
+
+ /** Set the CharBuf's <code>size</code> attribute.
+ */
+ void
+ Set_Size(CharBuf *self, size_t size);
+
+ /** Get the CharBuf's <code>size</code> attribute.
+ */
+ size_t
+ Get_Size(CharBuf *self);
+
+ /** Return the internal backing array for the CharBuf if its internal
+ * encoding is UTF-8. If it is not encoded as UTF-8 throw an exception.
+ */
+ uint8_t*
+ Get_Ptr8(CharBuf *self);
+
+ /** Return a fresh copy of the string data in a CharBuf with an internal
+ * encoding of UTF-8.
+ */
+ CharBuf*
+ To_CB8(CharBuf *self);
+
+ public incremented CharBuf*
+ Clone(CharBuf *self);
+
+ public void
+ Destroy(CharBuf *self);
+
+ public bool_t
+ Equals(CharBuf *self, Obj *other);
+
+ public int32_t
+ Compare_To(CharBuf *self, Obj *other);
+
+ public int32_t
+ Hash_Sum(CharBuf *self);
+
+ public incremented CharBuf*
+ To_String(CharBuf *self);
+
+ public incremented CharBuf*
+ Load(CharBuf *self, Obj *dump);
+
+ public void
+ Serialize(CharBuf *self, OutStream *outstream);
+
+ public incremented CharBuf*
+ Deserialize(CharBuf *self, InStream *instream);
+
+ /** Remove Unicode whitespace characters from both top and tail.
+ */
+ uint32_t
+ Trim(CharBuf *self);
+
+ /** Remove leading Unicode whitespace.
+ */
+ uint32_t
+ Trim_Top(CharBuf *self);
+
+ /** Remove trailing Unicode whitespace.
+ */
+ uint32_t
+ Trim_Tail(CharBuf *self);
+
+ /** Remove characters (measured in code points) from the top of the
+ * CharBuf. Returns the number nipped.
+ */
+ size_t
+ Nip(CharBuf *self, size_t count);
+
+ /** Remove one character from the top of the CharBuf. Returns the code
+ * point, or 0 if the string was empty.
+ */
+ int32_t
+ Nip_One(CharBuf *self);
+
+ /** Remove characters (measured in code points) from the end of the
+ * CharBuf. Returns the number chopped.
+ */
+ size_t
+ Chop(CharBuf *self, size_t count);
+
+ /** Truncate the CharBuf so that it contains no more than
+ * <code>count</code>characters.
+ *
+ * @param count Maximum new length, in Unicode code points.
+ * @return The number of code points left in the string after truncation.
+ */
+ size_t
+ Truncate(CharBuf *self, size_t count);
+
+ /** Return the Unicode code point at the specified number of code points
+ * in. Return 0 if the string length is exceeded. (XXX It would be
+ * better to throw an exception, but that's not practical with UTF-8 and
+ * no cached length.)
+ */
+ uint32_t
+ Code_Point_At(CharBuf *self, size_t tick);
+
+ /** Return the Unicode code point at the specified number of code points
+ * counted backwards from the end of the string. Return 0 if outside the
+ * string.
+ */
+ uint32_t
+ Code_Point_From(CharBuf *self, size_t tick);
+
+ /** Return a newly allocated CharBuf containing a copy of the indicated
+ * substring.
+ * @param offset Offset from the top, in code points.
+ * @param len The desired length of the substring, in code points.
+ */
+ incremented CharBuf*
+ SubString(CharBuf *self, size_t offset, size_t len);
+
+ /** Concatenate the supplied text onto the end of the CharBuf. Don't
+ * check for UTF-8 validity.
+ */
+ void
+ Cat_Trusted_Str(CharBuf *self, const char *ptr, size_t size);
+}
+
+class Lucy::Object::ViewCharBuf cnick ViewCB
+ inherits Lucy::Object::CharBuf {
+
+ inert incremented ViewCharBuf*
+ new_from_utf8(const char *utf8, size_t size);
+
+ inert incremented ViewCharBuf*
+ new_from_trusted_utf8(const char *utf8, size_t size);
+
+ inert ViewCharBuf*
+ init(ViewCharBuf *self, const char *utf8, size_t size);
+
+ void
+ Assign(ViewCharBuf *self, const CharBuf *other);
+
+ void
+ Assign_Str(ViewCharBuf *self, const char *utf8, size_t size);
+
+ void
+ Assign_Trusted_Str(ViewCharBuf *self, const char *utf8, size_t size);
+
+ uint32_t
+ Trim_Top(ViewCharBuf *self);
+
+ size_t
+ Nip(ViewCharBuf *self, size_t count);
+
+ int32_t
+ Nip_One(ViewCharBuf *self);
+
+ /** Throws an error. */
+ char*
+ Grow(ViewCharBuf *self, size_t size);
+
+ public void
+ Destroy(ViewCharBuf *self);
+}
+
+class Lucy::Object::ZombieCharBuf cnick ZCB
+ inherits Lucy::Object::ViewCharBuf {
+
+ /** Return a ZombieCharBuf with a blank string.
+ */
+ inert incremented ZombieCharBuf*
+ new(void *allocation);
+
+ /**
+ * @param allocation A single block of memory which will be used for both
+ * the ZombieCharBuf object and its buffer.
+ * @param alloc_size The size of the allocation.
+ * @param pattern A format pattern.
+ */
+ inert incremented ZombieCharBuf*
+ newf(void *allocation, size_t alloc_size, const char *pattern, ...);
+
+ inert incremented ZombieCharBuf*
+ wrap(void *allocation, const CharBuf *source);
+
+ inert incremented ZombieCharBuf*
+ wrap_str(void *allocation, const char *ptr, size_t size);
+
+ /** Return the size for a ZombieCharBuf struct.
+ */
+ inert size_t
+ size();
+
+ /** Throws an error.
+ */
+ public void
+ Destroy(ZombieCharBuf *self);
+}
+
+__C__
+
+#define CFISH_ZCB_BLANK() lucy_ZCB_new(alloca(lucy_ZCB_size()))
+
+#define CFISH_ZCB_LITERAL(string) \
+ { LUCY_ZOMBIECHARBUF, {1}, string "", sizeof(string) -1, sizeof(string) }
+
+#define CFISH_ZCB_WRAP(source) \
+ lucy_ZCB_wrap(alloca(lucy_ZCB_size()), source)
+
+#define CFISH_ZCB_WRAP_STR(ptr, size) \
+ lucy_ZCB_wrap_str(alloca(lucy_ZCB_size()), ptr, size)
+
+extern lucy_ZombieCharBuf CFISH_ZCB_EMPTY;
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define ZCB_BLANK CFISH_ZCB_BLANK
+ #define ZCB_LITERAL(_string) CFISH_ZCB_LITERAL(_string)
+ #define EMPTY CFISH_ZCB_EMPTY
+ #define ZCB_WRAP CFISH_ZCB_WRAP
+ #define ZCB_WRAP_STR CFISH_ZCB_WRAP_STR
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Object/Err.c b/core/Lucy/Object/Err.c
new file mode 100644
index 0000000..2fae051
--- /dev/null
+++ b/core/Lucy/Object/Err.c
@@ -0,0 +1,263 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ERR
+#define C_LUCY_OBJ
+#define C_LUCY_VTABLE
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <string.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Util/Memory.h"
+
+Err*
+Err_new(CharBuf *mess) {
+ Err *self = (Err*)VTable_Make_Obj(ERR);
+ return Err_init(self, mess);
+}
+
+Err*
+Err_init(Err *self, CharBuf *mess) {
+ self->mess = mess;
+ return self;
+}
+
+void
+Err_destroy(Err *self) {
+ DECREF(self->mess);
+ SUPER_DESTROY(self, ERR);
+}
+
+Err*
+Err_make(Err *self) {
+ UNUSED_VAR(self);
+ return Err_new(CB_new(0));
+}
+
+CharBuf*
+Err_to_string(Err *self) {
+ return (CharBuf*)INCREF(self->mess);
+}
+
+void
+Err_cat_mess(Err *self, const CharBuf *mess) {
+ CB_Cat(self->mess, mess);
+}
+
+// Fallbacks in case variadic macros aren't available.
+#ifndef CHY_HAS_VARIADIC_MACROS
+void
+THROW(VTable *vtable, char *pattern, ...) {
+ va_list args;
+ Err_make_t make
+ = (Err_make_t)METHOD(CERTIFY(vtable, VTABLE), Err, Make);
+ Err *err = (Err*)CERTIFY(make(NULL), ERR);
+ CharBuf *mess = Err_Get_Mess(err);
+
+ va_start(args, pattern);
+ CB_VCatF(mess, pattern, args);
+ va_end(args);
+
+ Err_do_throw(err);
+}
+void
+CFISH_WARN(char *pattern, ...) {
+ va_list args;
+ CharBuf *const message = CB_new(strlen(pattern) + 10);
+
+ va_start(args, pattern);
+ CB_VCatF(message, pattern, args);
+ va_end(args);
+
+ Err_warn_mess(message);
+}
+CharBuf*
+CFISH_MAKE_MESS(char *pattern, ...) {
+ va_list args;
+ CharBuf *const message = CB_new(strlen(pattern) + 10);
+
+ va_start(args, pattern);
+ CB_VCatF(message, pattern, args);
+ va_end(args);
+
+ return message;
+}
+#endif
+
+
+static void
+S_vcat_mess(CharBuf *message, const char *file, int line, const char *func,
+ const char *pattern, va_list args) {
+ size_t guess_len = strlen(file)
+ + func ? strlen(func) : 0
+ + strlen(pattern)
+ + 30;
+ CB_Grow(message, guess_len);
+ CB_VCatF(message, pattern, args);
+ if (func != NULL) {
+ CB_catf(message, "\n\t%s at %s line %i32\n", func, file, (int32_t)line);
+ }
+ else {
+ CB_catf(message, "\n\t%s line %i32\n", file, (int32_t)line);
+ }
+}
+
+CharBuf*
+Err_make_mess(const char *file, int line, const char *func,
+ const char *pattern, ...) {
+ va_list args;
+ size_t guess_len = strlen(pattern) + strlen(file) + 20;
+ CharBuf *message = CB_new(guess_len);
+ va_start(args, pattern);
+ S_vcat_mess(message, file, line, func, pattern, args);
+ va_end(args);
+ return message;
+}
+
+void
+Err_warn_at(const char *file, int line, const char *func,
+ const char *pattern, ...) {
+ va_list args;
+ CharBuf *message = CB_new(0);
+ va_start(args, pattern);
+ S_vcat_mess(message, file, line, func, pattern, args);
+ va_end(args);
+ Err_warn_mess(message);
+}
+
+CharBuf*
+Err_get_mess(Err *self) {
+ return self->mess;
+}
+
+void
+Err_add_frame(Err *self, const char *file, int line, const char *func) {
+ if (CB_Ends_With_Str(self->mess, "\n", 1)) { CB_Chop(self->mess, 1); }
+
+ if (func != NULL) {
+ CB_catf(self->mess, "\n\t%s at %s line %i32\n", func, file,
+ (int32_t)line);
+ }
+ else {
+ CB_catf(self->mess, "\n\tat %s line %i32\n", file, (int32_t)line);
+ }
+}
+
+void
+Err_rethrow(Err *self, const char *file, int line, const char *func) {
+ Err_add_frame(self, file, line, func);
+ Err_do_throw(self);
+}
+
+void
+Err_throw_at(VTable *vtable, const char *file, int line,
+ const char *func, const char *pattern, ...) {
+ va_list args;
+ Err_make_t make
+ = (Err_make_t)METHOD(CERTIFY(vtable, VTABLE), Err, Make);
+ Err *err = (Err*)CERTIFY(make(NULL), ERR);
+ CharBuf *mess = Err_Get_Mess(err);
+
+ va_start(args, pattern);
+ S_vcat_mess(mess, file, line, func, pattern, args);
+ va_end(args);
+
+ Err_do_throw(err);
+}
+
+// Inlined, slightly optimized version of Obj_is_a.
+static INLINE bool_t
+SI_obj_is_a(Obj *obj, VTable *target_vtable) {
+ VTable *vtable = obj->vtable;
+
+ while (vtable != NULL) {
+ if (vtable == target_vtable) {
+ return true;
+ }
+ vtable = vtable->parent;
+ }
+
+ return false;
+}
+
+Obj*
+Err_downcast(Obj *obj, VTable *vtable, const char *file, int line,
+ const char *func) {
+ if (obj && !SI_obj_is_a(obj, vtable)) {
+ Err_throw_at(ERR, file, line, func, "Can't downcast from %o to %o",
+ Obj_Get_Class_Name(obj), VTable_Get_Name(vtable));
+ }
+ return obj;
+}
+
+Obj*
+Err_certify(Obj *obj, VTable *vtable, const char *file, int line,
+ const char *func) {
+ if (!obj) {
+ Err_throw_at(ERR, file, line, func, "Object isn't a %o, it's NULL",
+ VTable_Get_Name(vtable));
+ }
+ else if (!SI_obj_is_a(obj, vtable)) {
+ Err_throw_at(ERR, file, line, func, "Can't downcast from %o to %o",
+ Obj_Get_Class_Name(obj), VTable_Get_Name(vtable));
+ }
+ return obj;
+}
+
+#ifdef CHY_HAS_WINDOWS_H
+
+#include <windows.h>
+
+char*
+Err_win_error() {
+ size_t buf_size = 256;
+ char *buf = (char*)MALLOCATE(buf_size);
+ size_t message_len = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
+ NULL, // message source table
+ GetLastError(),
+ 0, // language id
+ buf,
+ buf_size,
+ NULL // empty va_list
+ );
+ if (message_len == 0) {
+ char unknown[] = "Unknown error";
+ size_t len = sizeof(unknown);
+ strncpy(buf, unknown, len);
+ }
+ else if (message_len > 1) {
+ // Kill stupid newline.
+ buf[message_len - 2] = '\0';
+ }
+ return buf;
+}
+
+#else
+
+char*
+Err_win_error() {
+ return NULL; // Never called.
+}
+
+#endif // CHY_HAS_WINDOWS_H
+
+
diff --git a/core/Lucy/Object/Err.cfh b/core/Lucy/Object/Err.cfh
new file mode 100644
index 0000000..c8c16f5
--- /dev/null
+++ b/core/Lucy/Object/Err.cfh
@@ -0,0 +1,239 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Exception.
+ *
+ * Most of the time when Lucy encounters an error, it tries to raise a
+ * Lucy::Object::Err exception with an error message and context
+ * information.
+ *
+ * At present, it is only safe to catch exceptions which are specifically
+ * documented as catchable; most times when an Err is raised, Lucy leaks
+ * memory.
+ *
+ * The Err module also provides access to a per-thread Err shared variable via
+ * set_error() and get_error(). It may be used to store an Err object
+ * temporarily, so that calling code may choose how to handle a particular
+ * error condition.
+ */
+class Lucy::Object::Err inherits Lucy::Object::Obj {
+
+ CharBuf *mess;
+
+ inert incremented Err*
+ new(decremented CharBuf *mess);
+
+ inert Err*
+ init(Err *self, decremented CharBuf *mess);
+
+ public void
+ Destroy(Err *self);
+
+ public incremented CharBuf*
+ To_String(Err *self);
+
+ void*
+ To_Host(Err *self);
+
+ /** Concatenate the supplied argument onto the internal "mess".
+ */
+ public void
+ Cat_Mess(Err *self, const CharBuf *mess);
+
+ public CharBuf*
+ Get_Mess(Err *self);
+
+ /** Add information about the current stack frame onto <code>mess</code>.
+ */
+ void
+ Add_Frame(Err *self, const char *file, int line, const char *func);
+
+ public incremented Err*
+ Make(Err *self);
+
+ /** Set the value of "error", a per-thread Err shared variable.
+ */
+ public inert void
+ set_error(decremented Err *error);
+
+ /** Retrieve per-thread Err shared variable "error".
+ */
+ public inert nullable Err*
+ get_error();
+
+ /** Print an error message to stderr with some C contextual information.
+ * Usually invoked via the WARN(pattern, ...) macro.
+ */
+ inert void
+ warn_at(const char *file, int line, const char *func,
+ const char *pattern, ...);
+
+ /** Raise an exception. Usually invoked via the THROW macro.
+ */
+ inert void
+ throw_at(VTable *vtable, const char *file, int line, const char *func,
+ const char *pattern, ...);
+
+ /** Throw an existing exception after tacking on additional context data.
+ */
+ inert void
+ rethrow(Err *error, const char *file, int line, const char *func);
+
+ /** Raise an exception. Clean up the supplied message by decrementing its
+ * refcount.
+ *
+ * @param vtable The vtable for the Err class to throw.
+ * @param message Error message, to be output verbatim.
+ */
+ inert void
+ throw_mess(VTable *vtable, decremented CharBuf *message);
+
+ /** Invoke host exception handling.
+ */
+ inert void
+ do_throw(decremented Err *self);
+
+ /** Invoke host warning mechanism. Clean up the supplied message by
+ * decrementing its refcount.
+ *
+ * @param message Error message, to be output verbatim.
+ */
+ inert void
+ warn_mess(decremented CharBuf *message);
+
+ /** Create a formatted error message. Ususally invoked via the MAKE_MESS
+ * macro.
+ */
+ inert CharBuf*
+ make_mess(const char *file, int line, const char *func,
+ const char *pattern, ...);
+
+ /** Verify that <code>obj</code> is either NULL or inherits from
+ * the class represented by <code>vtable</code>.
+ *
+ * @return the object.
+ */
+ inert nullable Obj*
+ downcast(Obj *obj, VTable *vtable, const char *file, int line,
+ const char *func);
+
+ /** Verify that <code>obj</code> is not NULL and inherits from the class
+ * represented by <code>vtable</code>.
+ *
+ * @return the object.
+ */
+ inert Obj*
+ certify(Obj *obj, VTable *vtable, const char *file, int line,
+ const char *func);
+
+ /** Verify that an object belongs to a subclass and not an abstract class.
+ */
+ inert inline void
+ abstract_class_check(Obj *obj, VTable *vtable);
+
+ /** On Windows, return a newly allocated buffer containing the string
+ * description for the the last error in the thread.
+ */
+ inert char*
+ win_error();
+}
+
+__C__
+#ifdef CHY_HAS_FUNC_MACRO
+ #define CFISH_ERR_FUNC_MACRO CHY_FUNC_MACRO
+#else
+ #define CFISH_ERR_FUNC_MACRO NULL
+#endif
+
+#define CFISH_ERR_ADD_FRAME(_error) \
+ Lucy_Err_Add_Frame(_error, __FILE__, __LINE__, \
+ CFISH_ERR_FUNC_MACRO)
+
+#define CFISH_RETHROW(_error) \
+ lucy_Err_rethrow((lucy_Err*)_error, __FILE__, __LINE__, \
+ CFISH_ERR_FUNC_MACRO)
+
+/** Macro version of lucy_Err_throw_at which inserts contextual information
+ * automatically, provided that the compiler supports the necessary features.
+ */
+#ifdef CHY_HAS_VARIADIC_MACROS
+ #ifdef CHY_HAS_ISO_VARIADIC_MACROS
+ #define CFISH_THROW(_vtable, ...) \
+ lucy_Err_throw_at(_vtable, __FILE__, __LINE__, CFISH_ERR_FUNC_MACRO, \
+ __VA_ARGS__)
+ #define CFISH_WARN(...) \
+ lucy_Err_warn_at(__FILE__, __LINE__, CFISH_ERR_FUNC_MACRO, __VA_ARGS__)
+ #define CFISH_MAKE_MESS(...) \
+ lucy_Err_make_mess(__FILE__, __LINE__, CFISH_ERR_FUNC_MACRO, \
+ __VA_ARGS__)
+ #elif defined(CHY_HAS_GNUC_VARIADIC_MACROS)
+ #define CFISH_THROW(_vtable, args...) \
+ lucy_Err_throw_at(_vtable, __FILE__, __LINE__, \
+ CFISH_ERR_FUNC_MACRO, ##args)
+ #define CFISH_WARN(args...) \
+ lucy_Err_warn_at(__FILE__, __LINE__, CFISH_ERR_FUNC_MACRO, ##args)
+ #define CFISH_MAKE_MESS(args...) \
+ lucy_Err_make_mess(__FILE__, __LINE__, CFISH_ERR_FUNC_MACRO, ##args)
+ #endif
+#else
+ void
+ CFISH_THROW(lucy_VTable *vtable, char* format, ...);
+ void
+ CFISH_WARN(char* format, ...);
+ lucy_CharBuf*
+ CFISH_MAKE_MESS(char* format, ...);
+#endif
+
+#define CFISH_DOWNCAST(_obj, _vtable) \
+ lucy_Err_downcast((lucy_Obj*)(_obj), (_vtable), \
+ __FILE__, __LINE__, CFISH_ERR_FUNC_MACRO)
+
+
+#define CFISH_CERTIFY(_obj, _vtable) \
+ lucy_Err_certify((lucy_Obj*)(_obj), (_vtable), \
+ __FILE__, __LINE__, CFISH_ERR_FUNC_MACRO)
+
+static CHY_INLINE void
+lucy_Err_abstract_class_check(lucy_Obj *obj, lucy_VTable *vtable) {
+ lucy_VTable *const my_vtable = *(lucy_VTable**)obj;
+ if (my_vtable == vtable) {
+ lucy_CharBuf *mess = CFISH_MAKE_MESS("%o is an abstract class",
+ Lucy_Obj_Get_Class_Name(obj));
+ Lucy_Obj_Dec_RefCount(obj);
+ lucy_Err_throw_mess(LUCY_ERR, mess);
+ }
+}
+
+#define CFISH_ABSTRACT_CLASS_CHECK(_obj, _vtable) \
+ lucy_Err_abstract_class_check(((lucy_Obj*)_obj), _vtable)
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define THROW CFISH_THROW
+ #define RETHROW CFISH_RETHROW
+ #define WARN CFISH_WARN
+ #define MAKE_MESS CFISH_MAKE_MESS
+ #define ERR_ADD_FRAME CFISH_ERR_ADD_FRAME
+ #define ERR_FUNC_MACRO CFISH_ERR_FUNC_MACRO
+ #define DOWNCAST CFISH_DOWNCAST
+ #define CERTIFY CFISH_CERTIFY
+ #define ABSTRACT_CLASS_CHECK CFISH_ABSTRACT_CLASS_CHECK
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Object/Hash.c b/core/Lucy/Object/Hash.c
new file mode 100644
index 0000000..44b8e09
--- /dev/null
+++ b/core/Lucy/Object/Hash.c
@@ -0,0 +1,501 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HASH
+#define C_LUCY_HASHTOMBSTONE
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <string.h>
+#include <stdlib.h>
+
+#include "Lucy/Object/VTable.h"
+
+#include "Lucy/Object/Hash.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/VArray.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+#include "Lucy/Util/Memory.h"
+
+static HashTombStone TOMBSTONE = {
+ HASHTOMBSTONE,
+ {1}
+};
+
+#define HashEntry lucy_HashEntry
+
+typedef struct HashEntry {
+ Obj *key;
+ Obj *value;
+ int32_t hash_sum;
+} HashEntry;
+
+// Reset the iterator. Hash_Iterate must be called to restart iteration.
+static INLINE void
+SI_kill_iter(Hash *self);
+
+// Return the entry associated with the key, if any.
+static INLINE HashEntry*
+SI_fetch_entry(Hash *self, const Obj *key, int32_t hash_sum);
+
+// Double the number of buckets and redistribute all entries.
+static INLINE HashEntry*
+SI_rebuild_hash(Hash *self);
+
+Hash*
+Hash_new(uint32_t capacity) {
+ Hash *self = (Hash*)VTable_Make_Obj(HASH);
+ return Hash_init(self, capacity);
+}
+
+Hash*
+Hash_init(Hash *self, uint32_t capacity) {
+ // Allocate enough space to hold the requested number of elements without
+ // triggering a rebuild.
+ uint32_t requested_capacity = capacity < I32_MAX ? capacity : I32_MAX;
+ uint32_t threshold;
+ capacity = 16;
+ while (1) {
+ threshold = (capacity / 3) * 2;
+ if (threshold > requested_capacity) { break; }
+ capacity *= 2;
+ }
+
+ // Init.
+ self->size = 0;
+ self->iter_tick = -1;
+
+ // Derive.
+ self->capacity = capacity;
+ self->entries = (HashEntry*)CALLOCATE(capacity, sizeof(HashEntry));
+ self->threshold = threshold;
+
+ return self;
+}
+
+void
+Hash_destroy(Hash *self) {
+ if (self->entries) {
+ Hash_Clear(self);
+ FREEMEM(self->entries);
+ }
+ SUPER_DESTROY(self, HASH);
+}
+
+Hash*
+Hash_dump(Hash *self) {
+ Hash *dump = Hash_new(self->size);
+ Obj *key;
+ Obj *value;
+
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &value)) {
+ // Since JSON only supports text hash keys, Dump() can only support
+ // text hash keys.
+ CERTIFY(key, CHARBUF);
+ Hash_Store(dump, key, Obj_Dump(value));
+ }
+
+ return dump;
+}
+
+Obj*
+Hash_load(Hash *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *class_name = (CharBuf*)Hash_Fetch_Str(source, "_class", 6);
+ UNUSED_VAR(self);
+
+ // Assume that the presence of the "_class" key paired with a valid class
+ // name indicates the output of a Dump rather than an ordinary Hash. */
+ if (class_name && CB_Is_A(class_name, CHARBUF)) {
+ VTable *vtable = VTable_fetch_vtable(class_name);
+
+ if (!vtable) {
+ CharBuf *parent_class = VTable_find_parent_class(class_name);
+ if (parent_class) {
+ VTable *parent = VTable_singleton(parent_class, NULL);
+ vtable = VTable_singleton(class_name, parent);
+ DECREF(parent_class);
+ }
+ else {
+ // TODO: Fix Hash_Load() so that it works with ordinary hash
+ // keys named "_class".
+ THROW(ERR, "Can't find class '%o'", class_name);
+ }
+ }
+
+ // Dispatch to an alternate Load() method.
+ if (vtable) {
+ Obj_load_t load = (Obj_load_t)METHOD(vtable, Obj, Load);
+ if (load == Obj_load) {
+ THROW(ERR, "Abstract method Load() not defined for %o",
+ VTable_Get_Name(vtable));
+ }
+ else if (load != (Obj_load_t)Hash_load) { // stop inf loop
+ return load(NULL, dump);
+ }
+ }
+ }
+
+ // It's an ordinary Hash.
+ {
+ Hash *loaded = Hash_new(source->size);
+ Obj *key;
+ Obj *value;
+
+ Hash_Iterate(source);
+ while (Hash_Next(source, &key, &value)) {
+ Hash_Store(loaded, key, Obj_Load(value, value));
+ }
+
+ return (Obj*)loaded;
+ }
+}
+
+void
+Hash_serialize(Hash *self, OutStream *outstream) {
+ Obj *key;
+ Obj *val;
+ uint32_t charbuf_count = 0;
+ OutStream_Write_C32(outstream, self->size);
+
+ // Write CharBuf keys first. CharBuf keys are the common case; grouping
+ // them together is a form of run-length-encoding and saves space, since
+ // we omit the per-key class name.
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &val)) {
+ if (Obj_Is_A(key, CHARBUF)) { charbuf_count++; }
+ }
+ OutStream_Write_C32(outstream, charbuf_count);
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &val)) {
+ if (Obj_Is_A(key, CHARBUF)) {
+ Obj_Serialize(key, outstream);
+ FREEZE(val, outstream);
+ }
+ }
+
+ // Punt on the classes of the remaining keys.
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &val)) {
+ if (!Obj_Is_A(key, CHARBUF)) {
+ FREEZE(key, outstream);
+ FREEZE(val, outstream);
+ }
+ }
+}
+
+Hash*
+Hash_deserialize(Hash *self, InStream *instream) {
+ uint32_t size = InStream_Read_C32(instream);
+ uint32_t num_charbufs = InStream_Read_C32(instream);
+ uint32_t num_other = size - num_charbufs;
+ CharBuf *key = num_charbufs ? CB_new(0) : NULL;
+
+ if (self) { Hash_init(self, size); }
+ else { self = Hash_new(size); }
+
+ // Read key-value pairs with CharBuf keys.
+ while (num_charbufs--) {
+ uint32_t len = InStream_Read_C32(instream);
+ char *key_buf = CB_Grow(key, len);
+ InStream_Read_Bytes(instream, key_buf, len);
+ key_buf[len] = '\0';
+ CB_Set_Size(key, len);
+ Hash_Store(self, (Obj*)key, THAW(instream));
+ }
+ DECREF(key);
+
+ // Read remaining key/value pairs.
+ while (num_other--) {
+ Obj *k = THAW(instream);
+ Hash_Store(self, k, THAW(instream));
+ DECREF(k);
+ }
+
+ return self;
+}
+
+void
+Hash_clear(Hash *self) {
+ HashEntry *entry = (HashEntry*)self->entries;
+ HashEntry *const limit = entry + self->capacity;
+
+ // Iterate through all entries.
+ for (; entry < limit; entry++) {
+ if (!entry->key) { continue; }
+ DECREF(entry->key);
+ DECREF(entry->value);
+ entry->key = NULL;
+ entry->value = NULL;
+ entry->hash_sum = 0;
+ }
+
+ self->size = 0;
+}
+
+void
+Hash_do_store(Hash *self, Obj *key, Obj *value,
+ int32_t hash_sum, bool_t use_this_key) {
+ HashEntry *entries = self->size >= self->threshold
+ ? SI_rebuild_hash(self)
+ : (HashEntry*)self->entries;
+ uint32_t tick = hash_sum;
+ const uint32_t mask = self->capacity - 1;
+
+ while (1) {
+ tick &= mask;
+ HashEntry *entry = entries + tick;
+ if (entry->key == (Obj*)&TOMBSTONE || !entry->key) {
+ if (entry->key == (Obj*)&TOMBSTONE) {
+ // Take note of diminished tombstone clutter.
+ self->threshold++;
+ }
+ entry->key = use_this_key
+ ? key
+ : Hash_Make_Key(self, key, hash_sum);
+ entry->value = value;
+ entry->hash_sum = hash_sum;
+ self->size++;
+ break;
+ }
+ else if (entry->hash_sum == hash_sum
+ && Obj_Equals(key, entry->key)
+ ) {
+ DECREF(entry->value);
+ entry->value = value;
+ break;
+ }
+ tick++; // linear scan
+ }
+}
+
+void
+Hash_store(Hash *self, Obj *key, Obj *value) {
+ Hash_do_store(self, key, value, Obj_Hash_Sum(key), false);
+}
+
+void
+Hash_store_str(Hash *self, const char *key, size_t key_len, Obj *value) {
+ ZombieCharBuf *key_buf = ZCB_WRAP_STR((char*)key, key_len);
+ Hash_do_store(self, (Obj*)key_buf, value,
+ ZCB_Hash_Sum(key_buf), false);
+}
+
+Obj*
+Hash_make_key(Hash *self, Obj *key, int32_t hash_sum) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(hash_sum);
+ return Obj_Clone(key);
+}
+
+Obj*
+Hash_fetch_str(Hash *self, const char *key, size_t key_len) {
+ ZombieCharBuf *key_buf = ZCB_WRAP_STR(key, key_len);
+ return Hash_fetch(self, (Obj*)key_buf);
+}
+
+static INLINE HashEntry*
+SI_fetch_entry(Hash *self, const Obj *key, int32_t hash_sum) {
+ uint32_t tick = hash_sum;
+ HashEntry *const entries = (HashEntry*)self->entries;
+ HashEntry *entry;
+
+ while (1) {
+ tick &= self->capacity - 1;
+ entry = entries + tick;
+ if (!entry->key) {
+ // Failed to find the key, so return NULL.
+ return NULL;
+ }
+ else if (entry->hash_sum == hash_sum
+ && Obj_Equals(key, entry->key)
+ ) {
+ return entry;
+ }
+ tick++;
+ }
+}
+
+Obj*
+Hash_fetch(Hash *self, const Obj *key) {
+ HashEntry *entry = SI_fetch_entry(self, key, Obj_Hash_Sum(key));
+ return entry ? entry->value : NULL;
+}
+
+Obj*
+Hash_delete(Hash *self, const Obj *key) {
+ HashEntry *entry = SI_fetch_entry(self, key, Obj_Hash_Sum(key));
+ if (entry) {
+ Obj *value = entry->value;
+ DECREF(entry->key);
+ entry->key = (Obj*)&TOMBSTONE;
+ entry->value = NULL;
+ entry->hash_sum = 0;
+ self->size--;
+ self->threshold--; // limit number of tombstones
+ return value;
+ }
+ else {
+ return NULL;
+ }
+}
+
+Obj*
+Hash_delete_str(Hash *self, const char *key, size_t key_len) {
+ ZombieCharBuf *key_buf = ZCB_WRAP_STR(key, key_len);
+ return Hash_delete(self, (Obj*)key_buf);
+}
+
+uint32_t
+Hash_iterate(Hash *self) {
+ SI_kill_iter(self);
+ return self->size;
+}
+
+static INLINE void
+SI_kill_iter(Hash *self) {
+ self->iter_tick = -1;
+}
+
+bool_t
+Hash_next(Hash *self, Obj **key, Obj **value) {
+ while (1) {
+ if (++self->iter_tick >= (int32_t)self->capacity) {
+ // Bail since we've completed the iteration.
+ --self->iter_tick;
+ *key = NULL;
+ *value = NULL;
+ return false;
+ }
+ else {
+ HashEntry *const entry
+ = (HashEntry*)self->entries + self->iter_tick;
+ if (entry->key && entry->key != (Obj*)&TOMBSTONE) {
+ // Success!
+ *key = entry->key;
+ *value = entry->value;
+ return true;
+ }
+ }
+ }
+}
+
+Obj*
+Hash_find_key(Hash *self, const Obj *key, int32_t hash_sum) {
+ HashEntry *entry = SI_fetch_entry(self, key, hash_sum);
+ return entry ? entry->key : NULL;
+}
+
+VArray*
+Hash_keys(Hash *self) {
+ Obj *key;
+ Obj *val;
+ VArray *keys = VA_new(self->size);
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &val)) {
+ VA_push(keys, INCREF(key));
+ }
+ return keys;
+}
+
+VArray*
+Hash_values(Hash *self) {
+ Obj *key;
+ Obj *val;
+ VArray *values = VA_new(self->size);
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &val)) { VA_push(values, INCREF(val)); }
+ return values;
+}
+
+bool_t
+Hash_equals(Hash *self, Obj *other) {
+ Hash *twin = (Hash*)other;
+ Obj *key;
+ Obj *val;
+
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, HASH)) { return false; }
+ if (self->size != twin->size) { return false; }
+
+ Hash_Iterate(self);
+ while (Hash_Next(self, &key, &val)) {
+ Obj *other_val = Hash_Fetch(twin, key);
+ if (!other_val || !Obj_Equals(other_val, val)) { return false; }
+ }
+
+ return true;
+}
+
+uint32_t
+Hash_get_capacity(Hash *self) {
+ return self->capacity;
+}
+
+uint32_t
+Hash_get_size(Hash *self) {
+ return self->size;
+}
+
+static INLINE HashEntry*
+SI_rebuild_hash(Hash *self) {
+ HashEntry *old_entries = (HashEntry*)self->entries;
+ HashEntry *entry = old_entries;
+ HashEntry *limit = old_entries + self->capacity;
+
+ SI_kill_iter(self);
+ self->capacity *= 2;
+ self->threshold = (self->capacity / 3) * 2;
+ self->entries = (HashEntry*)CALLOCATE(self->capacity, sizeof(HashEntry));
+ self->size = 0;
+
+ for (; entry < limit; entry++) {
+ if (!entry->key || entry->key == (Obj*)&TOMBSTONE) {
+ continue;
+ }
+ Hash_do_store(self, entry->key, entry->value,
+ entry->hash_sum, true);
+ }
+
+ FREEMEM(old_entries);
+
+ return (HashEntry*)self->entries;
+}
+
+/***************************************************************************/
+
+uint32_t
+HashTombStone_get_refcount(HashTombStone* self) {
+ CHY_UNUSED_VAR(self);
+ return 1;
+}
+
+HashTombStone*
+HashTombStone_inc_refcount(HashTombStone* self) {
+ return self;
+}
+
+uint32_t
+HashTombStone_dec_refcount(HashTombStone* self) {
+ UNUSED_VAR(self);
+ return 1;
+}
+
+
diff --git a/core/Lucy/Object/Hash.cfh b/core/Lucy/Object/Hash.cfh
new file mode 100644
index 0000000..fbf6185
--- /dev/null
+++ b/core/Lucy/Object/Hash.cfh
@@ -0,0 +1,162 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Hashtable.
+ *
+ * Values are stored by reference and may be any kind of Obj. By default, keys
+ * are cloned and so must belong to a class that implements Clone(); however,
+ * this behavior can be changed by overridding Make_Key(), e.g. to implement
+ * efficient hash sets.
+ */
+class Lucy::Object::Hash inherits Lucy::Object::Obj {
+
+ void *entries;
+ uint32_t capacity;
+ uint32_t size;
+ uint32_t threshold; /* rehashing trigger point */
+ int32_t iter_tick; /* used when iterating */
+
+ inert incremented Hash*
+ new(uint32_t capacity = 0);
+
+ /**
+ * @param capacity The number of elements that the hash will be asked to
+ * hold initially.
+ */
+ inert Hash*
+ init(Hash *self, uint32_t capacity = 0);
+
+ /** Empty the hash of all key-value pairs.
+ */
+ void
+ Clear(Hash *self);
+
+ /** Store a key-value pair. If <code>key</code> is not already present,
+ * Make_Key() will be called to manufacture the internally used key.
+ */
+ void
+ Store(Hash *self, Obj *key, decremented Obj *value);
+
+ void
+ Store_Str(Hash *self, const char *str, size_t len,
+ decremented Obj *value);
+
+ /** Fetch the value associated with <code>key</code>.
+ *
+ * @return the value, or NULL if <code>key</code> is not present.
+ */
+ nullable Obj*
+ Fetch(Hash *self, const Obj *key);
+
+ nullable Obj*
+ Fetch_Str(Hash *self, const char *key, size_t key_len);
+
+ /** Attempt to delete a key-value pair from the hash.
+ *
+ * @return the value if <code>key</code> exists and thus deletion
+ * succeeds; otherwise NULL.
+ */
+ incremented nullable Obj*
+ Delete(Hash *self, const Obj *key);
+
+ incremented nullable Obj*
+ Delete_Str(Hash *self, const char *key, size_t key_ley);
+
+ /** Prepare to iterate over all the key-value pairs in the hash.
+ *
+ * @return the number of pairs which will be iterated over.
+ */
+ uint32_t
+ Iterate(Hash *self);
+
+ /** Retrieve the next key-value pair from the hash, setting the supplied
+ * pointers to point at them.
+ *
+ * @return true while iterating, false when the iterator has been
+ * exhausted.
+ */
+ bool_t
+ Next(Hash *self, Obj **key, Obj **value);
+
+ /** Search for a key which Equals the key supplied, and return the key
+ * rather than its value.
+ */
+ nullable Obj*
+ Find_Key(Hash *self, const Obj *key, int32_t hash_sum);
+
+ /** Return an VArray of pointers to the hash's keys.
+ */
+ incremented VArray*
+ Keys(Hash *self);
+
+ /** Return an VArray of pointers to the hash's values.
+ */
+ incremented VArray*
+ Values(Hash *self);
+
+ /** Create a key to be stored within the hash entry. Implementations must
+ * supply an object which produces the same Hash_Sum() value and tests
+ * true for Equals(). By default, calls Clone().
+ */
+ public incremented Obj*
+ Make_Key(Hash *self, Obj *key, int32_t hash_sum);
+
+ uint32_t
+ Get_Capacity(Hash *self);
+
+ /** Accessor for Hash's "size" member.
+ *
+ * @return the number of key-value pairs.
+ */
+ public uint32_t
+ Get_Size(Hash *self);
+
+ public bool_t
+ Equals(Hash *self, Obj *other);
+
+ public incremented Hash*
+ Dump(Hash *self);
+
+ public incremented Obj*
+ Load(Hash *self, Obj *dump);
+
+ public void
+ Serialize(Hash *self, OutStream *outstream);
+
+ public incremented Hash*
+ Deserialize(Hash *self, InStream *instream);
+
+ public void
+ Destroy(Hash *self);
+}
+
+class Lucy::Object::Hash::HashTombStone
+ inherits Lucy::Object::Obj {
+
+ uint32_t
+ Get_RefCount(HashTombStone* self);
+
+ incremented HashTombStone*
+ Inc_RefCount(HashTombStone* self);
+
+ uint32_t
+ Dec_RefCount(HashTombStone* self);
+}
+
+
diff --git a/core/Lucy/Object/Host.cfh b/core/Lucy/Object/Host.cfh
new file mode 100644
index 0000000..b1d580e
--- /dev/null
+++ b/core/Lucy/Object/Host.cfh
@@ -0,0 +1,108 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+#define CFISH_HOST_ARGTYPE_I32 (int32_t)0x00000001
+#define CFISH_HOST_ARGTYPE_I64 (int32_t)0x00000002
+#define CFISH_HOST_ARGTYPE_F32 (int32_t)0x00000003
+#define CFISH_HOST_ARGTYPE_F64 (int32_t)0x00000004
+#define CFISH_HOST_ARGTYPE_STR (int32_t)0x00000006
+#define CFISH_HOST_ARGTYPE_OBJ (int32_t)0x00000007
+#define CFISH_HOST_ARGTYPE_MASK 0x00000007
+
+#define CFISH_ARG_I32(_label, _value) \
+ CFISH_HOST_ARGTYPE_I32, (_label), ((int32_t)_value)
+#define CFISH_ARG_I64(_label, _value) \
+ CFISH_HOST_ARGTYPE_I64, (_label), ((int64_t)_value)
+#define CFISH_ARG_I(_type, _label, _value) \
+ (sizeof(_type) <= 4 ? CFISH_HOST_ARGTYPE_I32 : CFISH_HOST_ARGTYPE_I64), \
+ (_label), (sizeof(_type) <= 4 ? (int32_t)_value : (int64_t)_value)
+#define CFISH_ARG_F32(_label, _value) \
+ CFISH_HOST_ARGTYPE_F32, (_label), ((double)_value)
+#define CFISH_ARG_F64(_label, _value) \
+ CFISH_HOST_ARGTYPE_F64, (_label), ((double)_value)
+#define CFISH_ARG_STR(_label, _value) \
+ CFISH_HOST_ARGTYPE_STR, (_label), ((lucy_CharBuf*)_value)
+#define CFISH_ARG_OBJ(_label, _value) \
+ CFISH_HOST_ARGTYPE_OBJ, (_label), ((lucy_Obj*)_value)
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define ARG_I32 CFISH_ARG_I32
+ #define ARG_I64 CFISH_ARG_I64
+ #define ARG_F32 CFISH_ARG_F32
+ #define ARG_F64 CFISH_ARG_F64
+ #define ARG_STR CFISH_ARG_STR
+ #define ARG_OBJ CFISH_ARG_OBJ
+#endif
+
+__END_C__
+
+/** Callbacks to the host environment.
+ *
+ * All the callback functions are variadic, and all are designed to take a
+ * series of arguments using the ARG_XXX macros.
+ *
+ * int32_t area = (int32_t)Host_callback_i64(self, "calc_area", 2,
+ * ARG_I32("length", len), ARG_I32("width", width));
+ *
+ * The first argument is void* to avoid the need for tiresome casting to Obj*,
+ * but must always be a Clownfish object.
+ *
+ * If the invoker is a VTable, it will be used to make a class
+ * callback rather than an object callback.
+ */
+inert class Lucy::Object::Host {
+
+ /** Invoke an object method in a void context.
+ */
+ inert void
+ callback(void *self, char *method, uint32_t num_args, ...);
+
+ /** Invoke an object method, expecting an integer.
+ */
+ inert int64_t
+ callback_i64(void *self, char *method, uint32_t num_args, ...);
+
+ /** Invoke an object method, expecting a 64-bit floating point return
+ * value.
+ */
+ inert double
+ callback_f64(void *self, char *method, uint32_t num_args, ...);
+
+ /** Invoke an object method, expecting a Obj-derived object back, or
+ * possibly NULL. In order to ensure that the host environment doesn't
+ * reclaim the return value, it's refcount is increased by one, which the
+ * caller will have to deal with.
+ */
+ inert incremented nullable Obj*
+ callback_obj(void *self, char *method, uint32_t num_args, ...);
+
+ /** Invoke an object method, expecting a host string of some kind back,
+ * which will be converted into a newly allocated CharBuf.
+ */
+ inert incremented nullable CharBuf*
+ callback_str(void *self, char *method, uint32_t num_args, ...);
+
+ /** Invoke an object method, expecting a host data structure back. It's
+ * up to the caller to know how to process it.
+ */
+ inert nullable void*
+ callback_host(void *self, char *method, uint32_t num_args, ...);
+}
+
+
diff --git a/core/Lucy/Object/I32Array.c b/core/Lucy/Object/I32Array.c
new file mode 100644
index 0000000..57609a1
--- /dev/null
+++ b/core/Lucy/Object/I32Array.c
@@ -0,0 +1,77 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_I32ARRAY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Object/I32Array.h"
+
+I32Array*
+I32Arr_new(int32_t *ints, uint32_t size) {
+ I32Array *self = (I32Array*)VTable_Make_Obj(I32ARRAY);
+ int32_t *ints_copy = (int32_t*)MALLOCATE(size * sizeof(int32_t));
+ memcpy(ints_copy, ints, size * sizeof(int32_t));
+ return I32Arr_init(self, ints_copy, size);
+}
+
+I32Array*
+I32Arr_new_blank(uint32_t size) {
+ I32Array *self = (I32Array*)VTable_Make_Obj(I32ARRAY);
+ int32_t *ints = (int32_t*)CALLOCATE(size, sizeof(int32_t));
+ return I32Arr_init(self, ints, size);
+}
+
+I32Array*
+I32Arr_new_steal(int32_t *ints, uint32_t size) {
+ I32Array *self = (I32Array*)VTable_Make_Obj(I32ARRAY);
+ return I32Arr_init(self, ints, size);
+}
+
+I32Array*
+I32Arr_init(I32Array *self, int32_t *ints, uint32_t size) {
+ self->ints = ints;
+ self->size = size;
+ return self;
+}
+
+void
+I32Arr_destroy(I32Array *self) {
+ FREEMEM(self->ints);
+ SUPER_DESTROY(self, I32ARRAY);
+}
+
+void
+I32Arr_set(I32Array *self, uint32_t tick, int32_t value) {
+ if (tick >= self->size) {
+ THROW(ERR, "Out of bounds: %u32 >= %u32", tick, self->size);
+ }
+ self->ints[tick] = value;
+}
+
+int32_t
+I32Arr_get(I32Array *self, uint32_t tick) {
+ if (tick >= self->size) {
+ THROW(ERR, "Out of bounds: %u32 >= %u32", tick, self->size);
+ }
+ return self->ints[tick];
+}
+
+uint32_t
+I32Arr_get_size(I32Array *self) {
+ return self->size;
+}
+
+
diff --git a/core/Lucy/Object/I32Array.cfh b/core/Lucy/Object/I32Array.cfh
new file mode 100644
index 0000000..117b53f
--- /dev/null
+++ b/core/Lucy/Object/I32Array.cfh
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Object::I32Array cnick I32Arr inherits Lucy::Object::Obj {
+ int32_t *ints;
+ uint32_t size;
+
+ inert incremented I32Array*
+ new(int32_t *ints, uint32_t size);
+
+ inert incremented I32Array*
+ new_steal(int32_t *ints, uint32_t size);
+
+ inert incremented I32Array*
+ new_blank(uint32_t size);
+
+ inert I32Array*
+ init(I32Array *self, int32_t *ints, uint32_t size);
+
+ /** Set the value at <code>tick</code>, or throw an error if
+ * <code>tick</code> is out of bounds.
+ */
+ void
+ Set(I32Array *self, uint32_t tick, int32_t value);
+
+ /** Return the value at <code>tick</code>, or throw an error if
+ * <code>tick</code> is out of bounds.
+ */
+ int32_t
+ Get(I32Array *self, uint32_t tick);
+
+ /** Accessor for 'size' member.
+ */
+ uint32_t
+ Get_Size(I32Array *self);
+
+ public void
+ Destroy(I32Array *self);
+}
+
+
diff --git a/core/Lucy/Object/LockFreeRegistry.c b/core/Lucy/Object/LockFreeRegistry.c
new file mode 100644
index 0000000..7bbcad4
--- /dev/null
+++ b/core/Lucy/Object/LockFreeRegistry.c
@@ -0,0 +1,130 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LOCKFREEREGISTRY
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "Lucy/Object/LockFreeRegistry.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Util/Atomic.h"
+#include "Lucy/Util/Memory.h"
+
+typedef struct lucy_LFRegEntry {
+ Obj *key;
+ Obj *value;
+ int32_t hash_sum;
+ struct lucy_LFRegEntry *volatile next;
+} lucy_LFRegEntry;
+#define LFRegEntry lucy_LFRegEntry
+
+LockFreeRegistry*
+LFReg_new(size_t capacity) {
+ LockFreeRegistry *self
+ = (LockFreeRegistry*)VTable_Make_Obj(LOCKFREEREGISTRY);
+ return LFReg_init(self, capacity);
+}
+
+LockFreeRegistry*
+LFReg_init(LockFreeRegistry *self, size_t capacity) {
+ self->capacity = capacity;
+ self->entries = CALLOCATE(capacity, sizeof(void*));
+ return self;
+}
+
+bool_t
+LFReg_register(LockFreeRegistry *self, Obj *key, Obj *value) {
+ LFRegEntry *new_entry = NULL;
+ int32_t hash_sum = Obj_Hash_Sum(key);
+ size_t bucket = (uint32_t)hash_sum % self->capacity;
+ LFRegEntry *volatile *entries = (LFRegEntry*volatile*)self->entries;
+ LFRegEntry *volatile *slot = &(entries[bucket]);
+
+ // Proceed through the linked list. Bail out if the key has already been
+ // registered.
+FIND_END_OF_LINKED_LIST:
+ while (*slot) {
+ LFRegEntry *entry = *slot;
+ if (entry->hash_sum == hash_sum) {
+ if (Obj_Equals(key, entry->key)) {
+ return false;
+ }
+ }
+ slot = &(entry->next);
+ }
+
+ // We've found an empty slot. Create the new entry.
+ if (!new_entry) {
+ new_entry = (LFRegEntry*)MALLOCATE(sizeof(LFRegEntry));
+ new_entry->hash_sum = hash_sum;
+ new_entry->key = INCREF(key);
+ new_entry->value = INCREF(value);
+ new_entry->next = NULL;
+ }
+
+ /* Attempt to append the new node onto the end of the linked list.
+ * However, if another thread filled the slot since we found it (perhaps
+ * while we were allocating that new node), the compare-and-swap will
+ * fail. If that happens, we have to go back and find the new end of the
+ * linked list, then try again. */
+ if (!Atomic_cas_ptr((void*volatile*)slot, NULL, new_entry)) {
+ goto FIND_END_OF_LINKED_LIST;
+ }
+
+ return true;
+}
+
+Obj*
+LFReg_fetch(LockFreeRegistry *self, Obj *key) {
+ int32_t hash_sum = Obj_Hash_Sum(key);
+ size_t bucket = (uint32_t)hash_sum % self->capacity;
+ LFRegEntry **entries = (LFRegEntry**)self->entries;
+ LFRegEntry *entry = entries[bucket];
+
+ while (entry) {
+ if (entry->hash_sum == hash_sum) {
+ if (Obj_Equals(key, entry->key)) {
+ return entry->value;
+ }
+ }
+ entry = entry->next;
+ }
+
+ return NULL;
+}
+
+void
+LFReg_destroy(LockFreeRegistry *self) {
+ size_t i;
+ LFRegEntry **entries = (LFRegEntry**)self->entries;
+
+ for (i = 0; i < self->capacity; i++) {
+ LFRegEntry *entry = entries[i];
+ while (entry) {
+ LFRegEntry *next_entry = entry->next;
+ DECREF(entry->key);
+ DECREF(entry->value);
+ FREEMEM(entry);
+ entry = next_entry;
+ }
+ }
+ FREEMEM(self->entries);
+
+ SUPER_DESTROY(self, LOCKFREEREGISTRY);
+}
+
+
diff --git a/core/Lucy/Object/LockFreeRegistry.cfh b/core/Lucy/Object/LockFreeRegistry.cfh
new file mode 100644
index 0000000..28a8f10
--- /dev/null
+++ b/core/Lucy/Object/LockFreeRegistry.cfh
@@ -0,0 +1,45 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Specialized lock free hash table for storing VTables.
+ */
+class Lucy::Object::LockFreeRegistry cnick LFReg inherits Lucy::Object::Obj {
+
+ size_t capacity;
+ void *entries;
+
+ inert incremented LockFreeRegistry*
+ new(size_t capacity);
+
+ inert LockFreeRegistry*
+ init(LockFreeRegistry *self, size_t capacity);
+
+ public void
+ Destroy(LockFreeRegistry *self);
+
+ bool_t
+ Register(LockFreeRegistry *self, Obj *key, Obj *value);
+
+ nullable Obj*
+ Fetch(LockFreeRegistry *self, Obj *key);
+
+ void*
+ To_Host(LockFreeRegistry *self);
+}
+
+
diff --git a/core/Lucy/Object/Num.c b/core/Lucy/Object/Num.c
new file mode 100644
index 0000000..0ad86bf
--- /dev/null
+++ b/core/Lucy/Object/Num.c
@@ -0,0 +1,357 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NUM
+#define C_LUCY_INTNUM
+#define C_LUCY_FLOATNUM
+#define C_LUCY_INTEGER32
+#define C_LUCY_INTEGER64
+#define C_LUCY_FLOAT32
+#define C_LUCY_FLOAT64
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "Lucy/Object/Num.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+Num*
+Num_init(Num *self) {
+ ABSTRACT_CLASS_CHECK(self, NUM);
+ return self;
+}
+
+bool_t
+Num_equals(Num *self, Obj *other) {
+ Num *twin = (Num*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, NUM)) { return false; }
+ if (Num_To_F64(self) != Num_To_F64(twin)) { return false; }
+ if (Num_To_I64(self) != Num_To_I64(twin)) { return false; }
+ return true;
+}
+
+int32_t
+Num_compare_to(Num *self, Obj *other) {
+ Num *twin = (Num*)CERTIFY(other, NUM);
+ double f64_diff = Num_To_F64(self) - Num_To_F64(twin);
+ if (f64_diff) {
+ if (f64_diff < 0) { return -1; }
+ else if (f64_diff > 0) { return 1; }
+ }
+ else {
+ int64_t i64_diff = Num_To_I64(self) - Num_To_I64(twin);
+ if (i64_diff) {
+ if (i64_diff < 0) { return -1; }
+ else if (i64_diff > 0) { return 1; }
+ }
+ }
+ return 0;
+}
+
+/***************************************************************************/
+
+FloatNum*
+FloatNum_init(FloatNum *self) {
+ ABSTRACT_CLASS_CHECK(self, FLOATNUM);
+ return (FloatNum*)Num_init((Num*)self);
+}
+
+CharBuf*
+FloatNum_to_string(FloatNum *self) {
+ return CB_newf("%f64", FloatNum_To_F64(self));
+}
+
+/***************************************************************************/
+
+IntNum*
+IntNum_init(IntNum *self) {
+ ABSTRACT_CLASS_CHECK(self, INTNUM);
+ return (IntNum*)Num_init((Num*)self);
+}
+
+CharBuf*
+IntNum_to_string(IntNum *self) {
+ return CB_newf("%i64", IntNum_To_I64(self));
+}
+
+/***************************************************************************/
+
+Float32*
+Float32_new(float value) {
+ Float32 *self = (Float32*)VTable_Make_Obj(FLOAT32);
+ return Float32_init(self, value);
+}
+
+Float32*
+Float32_init(Float32 *self, float value) {
+ self->value = value;
+ return (Float32*)FloatNum_init((FloatNum*)self);
+}
+
+float
+Float32_get_value(Float32 *self) {
+ return self->value;
+}
+
+void
+Float32_set_value(Float32 *self, float value) {
+ self->value = value;
+}
+
+double
+Float32_to_f64(Float32 *self) {
+ return self->value;
+}
+
+int64_t
+Float32_to_i64(Float32 *self) {
+ return (int64_t)self->value;
+}
+
+int32_t
+Float32_hash_sum(Float32 *self) {
+ return *(int32_t*)&self->value;
+}
+
+Float32*
+Float32_clone(Float32 *self) {
+ return Float32_new(self->value);
+}
+
+void
+Float32_mimic(Float32 *self, Obj *other) {
+ Float32 *twin = (Float32*)CERTIFY(other, FLOAT32);
+ self->value = twin->value;
+}
+
+void
+Float32_serialize(Float32 *self, OutStream *outstream) {
+ OutStream_Write_F32(outstream, self->value);
+}
+
+Float32*
+Float32_deserialize(Float32 *self, InStream *instream) {
+ float value = InStream_Read_F32(instream);
+ return self ? Float32_init(self, value) : Float32_new(value);
+}
+
+/***************************************************************************/
+
+Float64*
+Float64_new(double value) {
+ Float64 *self = (Float64*)VTable_Make_Obj(FLOAT64);
+ return Float64_init(self, value);
+}
+
+Float64*
+Float64_init(Float64 *self, double value) {
+ self->value = value;
+ return (Float64*)FloatNum_init((FloatNum*)self);
+}
+
+double
+Float64_get_value(Float64 *self) {
+ return self->value;
+}
+
+void
+Float64_set_value(Float64 *self, double value) {
+ self->value = value;
+}
+
+double
+Float64_to_f64(Float64 *self) {
+ return self->value;
+}
+
+int64_t
+Float64_to_i64(Float64 *self) {
+ return (int64_t)self->value;
+}
+
+Float64*
+Float64_clone(Float64 *self) {
+ return Float64_new(self->value);
+}
+
+void
+Float64_mimic(Float64 *self, Obj *other) {
+ Float64 *twin = (Float64*)CERTIFY(other, FLOAT64);
+ self->value = twin->value;
+}
+
+int32_t
+Float64_hash_sum(Float64 *self) {
+ int32_t *ints = (int32_t*)&self->value;
+ return ints[0] ^ ints[1];
+}
+
+void
+Float64_serialize(Float64 *self, OutStream *outstream) {
+ OutStream_Write_F64(outstream, self->value);
+}
+
+Float64*
+Float64_deserialize(Float64 *self, InStream *instream) {
+ double value = InStream_Read_F64(instream);
+ return self ? Float64_init(self, value) : Float64_new(value);
+}
+
+/***************************************************************************/
+
+Integer32*
+Int32_new(int32_t value) {
+ Integer32 *self = (Integer32*)VTable_Make_Obj(INTEGER32);
+ return Int32_init(self, value);
+}
+
+Integer32*
+Int32_init(Integer32 *self, int32_t value) {
+ self->value = value;
+ return (Integer32*)IntNum_init((IntNum*)self);
+}
+
+int32_t
+Int32_get_value(Integer32 *self) {
+ return self->value;
+}
+
+void
+Int32_set_value(Integer32 *self, int32_t value) {
+ self->value = value;
+}
+
+double
+Int32_to_f64(Integer32 *self) {
+ return self->value;
+}
+
+int64_t
+Int32_to_i64(Integer32 *self) {
+ return self->value;
+}
+
+Integer32*
+Int32_clone(Integer32 *self) {
+ return Int32_new(self->value);
+}
+
+void
+Int32_mimic(Integer32 *self, Obj *other) {
+ Integer32 *twin = (Integer32*)CERTIFY(other, INTEGER32);
+ self->value = twin->value;
+}
+
+int32_t
+Int32_hash_sum(Integer32 *self) {
+ return self->value;
+}
+
+void
+Int32_serialize(Integer32 *self, OutStream *outstream) {
+ OutStream_Write_C32(outstream, (uint32_t)self->value);
+}
+
+Integer32*
+Int32_deserialize(Integer32 *self, InStream *instream) {
+ int32_t value = (int32_t)InStream_Read_C32(instream);
+ return self ? Int32_init(self, value) : Int32_new(value);
+}
+
+/***************************************************************************/
+
+Integer64*
+Int64_new(int64_t value) {
+ Integer64 *self = (Integer64*)VTable_Make_Obj(INTEGER64);
+ return Int64_init(self, value);
+}
+
+Integer64*
+Int64_init(Integer64 *self, int64_t value) {
+ self->value = value;
+ return (Integer64*)FloatNum_init((FloatNum*)self);
+}
+
+int64_t
+Int64_get_value(Integer64 *self) {
+ return self->value;
+}
+
+void
+Int64_set_value(Integer64 *self, int64_t value) {
+ self->value = value;
+}
+
+double
+Int64_to_f64(Integer64 *self) {
+ return (double)self->value;
+}
+
+int64_t
+Int64_to_i64(Integer64 *self) {
+ return self->value;
+}
+
+Integer64*
+Int64_clone(Integer64 *self) {
+ return Int64_new(self->value);
+}
+
+void
+Int64_mimic(Integer64 *self, Obj *other) {
+ Integer64 *twin = (Integer64*)CERTIFY(other, INTEGER64);
+ self->value = twin->value;
+}
+
+int32_t
+Int64_hash_sum(Integer64 *self) {
+ int32_t *ints = (int32_t*)&self->value;
+ return ints[0] ^ ints[1];
+}
+
+bool_t
+Int64_equals(Integer64 *self, Obj *other) {
+ Num *twin = (Num*)other;
+ if (twin == (Num*)self) { return true; }
+ if (!Obj_Is_A(other, NUM)) { return false; }
+ if (Num_Is_A(twin, FLOATNUM)) {
+ double floating_val = Num_To_F64(twin);
+ int64_t int_val = (int64_t)floating_val;
+ if ((double)int_val != floating_val) { return false; }
+ if (int_val != self->value) { return false; }
+ }
+ else {
+ if (self->value != Num_To_I64(twin)) { return false; }
+ }
+ return true;
+}
+
+void
+Int64_serialize(Integer64 *self, OutStream *outstream) {
+ OutStream_Write_C64(outstream, (uint64_t)self->value);
+}
+
+Integer64*
+Int64_deserialize(Integer64 *self, InStream *instream) {
+ int64_t value = (int64_t)InStream_Read_C64(instream);
+ return self ? Int64_init(self, value) : Int64_new(value);
+}
+
+
diff --git a/core/Lucy/Object/Num.cfh b/core/Lucy/Object/Num.cfh
new file mode 100644
index 0000000..c18cd5f
--- /dev/null
+++ b/core/Lucy/Object/Num.cfh
@@ -0,0 +1,235 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Abstract base class for numbers.
+ */
+abstract class Lucy::Object::Num inherits Lucy::Object::Obj {
+
+ inert Num*
+ init(Num *self);
+
+ public bool_t
+ Equals(Num *self, Obj *other);
+
+ public int32_t
+ Compare_To(Num *self, Obj *other);
+}
+
+/** Abstract base class for floating point numbers.
+ */
+abstract class Lucy::Object::FloatNum inherits Lucy::Object::Num {
+
+ inert FloatNum*
+ init(FloatNum *self);
+
+ public incremented CharBuf*
+ To_String(FloatNum *self);
+}
+
+
+/** Abstract base class for Integers.
+ */
+abstract class Lucy::Object::IntNum inherits Lucy::Object::Num {
+
+ inert IntNum*
+ init(IntNum *self);
+
+ public incremented CharBuf*
+ To_String(IntNum *self);
+}
+
+
+/** Single precision floating point number.
+ */
+class Lucy::Object::Float32 inherits Lucy::Object::FloatNum {
+
+ float value;
+
+ /**
+ * @param value Initial value.
+ */
+ inert Float32*
+ init(Float32* self, float value);
+
+ inert Float32*
+ new(float value);
+
+ void
+ Set_Value(Float32 *self, float value);
+
+ float
+ Get_Value(Float32 *self);
+
+ public int64_t
+ To_I64(Float32 *self);
+
+ public double
+ To_F64(Float32 *self);
+
+ public int32_t
+ Hash_Sum(Float32 *self);
+
+ public void
+ Serialize(Float32 *self, OutStream *outstream);
+
+ public incremented Float32*
+ Deserialize(Float32 *self, InStream *instream);
+
+ public incremented Float32*
+ Clone(Float32 *self);
+
+ public void
+ Mimic(Float32 *self, Obj *other);
+}
+
+/** Double precision floating point number.
+ */
+class Lucy::Object::Float64 inherits Lucy::Object::FloatNum {
+
+ double value;
+
+ /**
+ * @param value Initial value.
+ */
+ inert Float64*
+ init(Float64* self, double value);
+
+ inert Float64*
+ new(double value);
+
+ void
+ Set_Value(Float64 *self, double value);
+
+ double
+ Get_Value(Float64 *self);
+
+ public int64_t
+ To_I64(Float64 *self);
+
+ public double
+ To_F64(Float64 *self);
+
+ public int32_t
+ Hash_Sum(Float64 *self);
+
+ public void
+ Serialize(Float64 *self, OutStream *outstream);
+
+ public incremented Float64*
+ Deserialize(Float64 *self, InStream *instream);
+
+ public incremented Float64*
+ Clone(Float64 *self);
+
+ public void
+ Mimic(Float64 *self, Obj *other);
+}
+
+/** 32-bit signed integer.
+ */
+class Lucy::Object::Integer32 cnick Int32
+ inherits Lucy::Object::IntNum {
+
+ int32_t value;
+
+ /**
+ * @param value Initial value.
+ */
+ inert Integer32*
+ init(Integer32* self, int32_t value);
+
+ inert Integer32*
+ new(int32_t value);
+
+ void
+ Set_Value(Integer32 *self, int32_t value);
+
+ int32_t
+ Get_Value(Integer32 *self);
+
+ public int64_t
+ To_I64(Integer32 *self);
+
+ public double
+ To_F64(Integer32 *self);
+
+ public int32_t
+ Hash_Sum(Integer32 *self);
+
+ public void
+ Serialize(Integer32 *self, OutStream *outstream);
+
+ public incremented Integer32*
+ Deserialize(Integer32 *self, InStream *instream);
+
+ public incremented Integer32*
+ Clone(Integer32 *self);
+
+ public void
+ Mimic(Integer32 *self, Obj *other);
+}
+
+/**
+ * 64-bit signed integer.
+ */
+class Lucy::Object::Integer64 cnick Int64
+ inherits Lucy::Object::IntNum {
+
+ int64_t value;
+
+ /**
+ * @param value Initial value.
+ */
+ inert Integer64*
+ init(Integer64* self, int64_t value);
+
+ inert Integer64*
+ new(int64_t value);
+
+ void
+ Set_Value(Integer64 *self, int64_t value);
+
+ int64_t
+ Get_Value(Integer64 *self);
+
+ public int64_t
+ To_I64(Integer64 *self);
+
+ public double
+ To_F64(Integer64 *self);
+
+ public int32_t
+ Hash_Sum(Integer64 *self);
+
+ public bool_t
+ Equals(Integer64 *self, Obj *other);
+
+ public void
+ Serialize(Integer64 *self, OutStream *outstream);
+
+ public incremented Integer64*
+ Deserialize(Integer64 *self, InStream *instream);
+
+ public incremented Integer64*
+ Clone(Integer64 *self);
+
+ public void
+ Mimic(Integer64 *self, Obj *other);
+}
+
+
diff --git a/core/Lucy/Object/Obj.c b/core/Lucy/Object/Obj.c
new file mode 100644
index 0000000..e0538c2
--- /dev/null
+++ b/core/Lucy/Object/Obj.c
@@ -0,0 +1,125 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_OBJ
+#define C_LUCY_VTABLE
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "Lucy/Object/Obj.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/Hash.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Memory.h"
+
+Obj*
+Obj_init(Obj *self) {
+ ABSTRACT_CLASS_CHECK(self, OBJ);
+ return self;
+}
+
+void
+Obj_destroy(Obj *self) {
+ FREEMEM(self);
+}
+
+int32_t
+Obj_hash_sum(Obj *self) {
+ int64_t hash_sum = PTR_TO_I64(self);
+ return (int32_t)hash_sum;
+}
+
+bool_t
+Obj_is_a(Obj *self, VTable *ancestor) {
+ VTable *vtable = self ? self->vtable : NULL;
+
+ while (vtable != NULL) {
+ if (vtable == ancestor) {
+ return true;
+ }
+ vtable = vtable->parent;
+ }
+
+ return false;
+}
+
+bool_t
+Obj_equals(Obj *self, Obj *other) {
+ return (self == other);
+}
+
+void
+Obj_serialize(Obj *self, OutStream *outstream) {
+ CharBuf *class_name = Obj_Get_Class_Name(self);
+ CB_Serialize(class_name, outstream);
+}
+
+Obj*
+Obj_deserialize(Obj *self, InStream *instream) {
+ CharBuf *class_name = CB_deserialize(NULL, instream);
+ if (!self) {
+ VTable *vtable = VTable_singleton(class_name, OBJ);
+ self = VTable_Make_Obj(vtable);
+ }
+ else {
+ CharBuf *my_class = VTable_Get_Name(self->vtable);
+ if (!CB_Equals(class_name, (Obj*)my_class)) {
+ THROW(ERR, "Class mismatch: %o %o", class_name, my_class);
+ }
+ }
+ DECREF(class_name);
+ return Obj_init(self);
+}
+
+CharBuf*
+Obj_to_string(Obj *self) {
+#if (SIZEOF_PTR == 4)
+ return CB_newf("%o@0x%x32", Obj_Get_Class_Name(self), self);
+#elif (SIZEOF_PTR == 8)
+ int64_t iaddress = PTR_TO_I64(self);
+ uint64_t address = (uint64_t)iaddress;
+ uint32_t address_hi = address >> 32;
+ uint32_t address_lo = address & 0xFFFFFFFF;
+ return CB_newf("%o@0x%x32%x32", Obj_Get_Class_Name(self), address_hi,
+ address_lo);
+#else
+ #error "Unexpected pointer size."
+#endif
+}
+
+Obj*
+Obj_dump(Obj *self) {
+ return (Obj*)Obj_To_String(self);
+}
+
+VTable*
+Obj_get_vtable(Obj *self) {
+ return self->vtable;
+}
+
+CharBuf*
+Obj_get_class_name(Obj *self) {
+ return VTable_Get_Name(self->vtable);
+}
+
+
diff --git a/core/Lucy/Object/Obj.cfh b/core/Lucy/Object/Obj.cfh
new file mode 100644
index 0000000..d8d4c58
--- /dev/null
+++ b/core/Lucy/Object/Obj.cfh
@@ -0,0 +1,223 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Base class for all Lucy objects.
+ */
+
+class Lucy::Object::Obj {
+
+ VTable *vtable;
+ lucy_ref_t ref;
+
+ /** Abstract initializer.
+ */
+ public inert Obj*
+ init(Obj* self);
+
+ /** Zero argument factory constructor.
+ */
+ public abstract incremented Obj*
+ Make(Obj *self);
+
+ /** Return an object's refcount.
+ */
+ uint32_t
+ Get_RefCount(Obj *self);
+
+ /** Increment an object's refcount.
+ *
+ * @return The object, allowing an assignment idiom.
+ */
+ incremented Obj*
+ Inc_RefCount(Obj *self);
+
+ /** NULL-safe invocation of Inc_RefCount().
+ *
+ * @return NULL if <code>self</code> is NULL, otherwise the return value
+ * of Inc_RefCount().
+ */
+ inert inline incremented Obj*
+ incref(Obj *self);
+
+ /** Decrement an object's refcount, calling Destroy() if it hits 0.
+ *
+ * @return the modified refcount.
+ */
+ uint32_t
+ Dec_RefCount(Obj *self);
+
+ /** NULL-safe invocation of Dec_RefCount().
+ *
+ * @return NULL if <code>self</code> is NULL, otherwise the return value
+ * of Dec_RefCount().
+ */
+ inert inline uint32_t
+ decref(Obj *self);
+
+ /** Return a host-language object wrapper for this object.
+ */
+ void*
+ To_Host(Obj *self);
+
+ /** Return a clone of the object.
+ */
+ public abstract incremented Obj*
+ Clone(Obj *self);
+
+ /** Generic destructor. Frees the struct itself but not any complex
+ * member elements.
+ */
+ public void
+ Destroy(Obj *self);
+
+ /** Invoke the Destroy() method found in <code>vtable</code> on
+ * <code>self</code>.
+ *
+ * TODO: Eliminate this function if we can arrive at a proper SUPER syntax.
+ */
+ inert inline void
+ super_destroy(Obj *self, VTable *vtable);
+
+ /** Indicate whether two objects are the same. By default, compares the
+ * memory address.
+ *
+ * @param other Another Obj.
+ */
+ public bool_t
+ Equals(Obj *self, Obj *other);
+
+ /** Indicate whether one object is less than, equal to, or greater than
+ * another.
+ *
+ * @param other Another Obj.
+ * @return 0 if the objects are equal, a negative number if
+ * <code>self</code> is less than <code>other</code>, and a positive
+ * number if <code>self</code> is greater than <code>other</code>.
+ */
+ public abstract int32_t
+ Compare_To(Obj *self, Obj *other);
+
+ /** Return a hash code for the object -- by default, the memory address.
+ */
+ public int32_t
+ Hash_Sum(Obj *self);
+
+ /** Return the object's VTable.
+ */
+ public VTable*
+ Get_VTable(Obj *self);
+
+ /** Return the name of the class that the object belongs to.
+ */
+ public CharBuf*
+ Get_Class_Name(Obj *self);
+
+ /** Indicate whether the object is a descendent of <code>ancestor</code>.
+ */
+ public bool_t
+ Is_A(Obj *self, VTable *ancestor);
+
+ /** Generic stringification: "ClassName@hex_mem_address".
+ */
+ public incremented CharBuf*
+ To_String(Obj *self);
+
+ /** Convert the object to a 64-bit integer.
+ */
+ public abstract int64_t
+ To_I64(Obj *self);
+
+ /** Convert the object to a double precision floating point number.
+ */
+ public abstract double
+ To_F64(Obj *self);
+
+ /** Serialize the object by writing to the supplied OutStream.
+ */
+ public void
+ Serialize(Obj *self, OutStream *outstream);
+
+ /** Inflate an object by reading the serialized form from the instream.
+ * The assumption is that the object has been allocated, assigned a
+ * refcount and a vtable, but that everything else is uninitialized.
+ *
+ * Implementations should also be prepared to handle NULL as an argument,
+ * in which case they should create the object from nothing.
+ */
+ public incremented Obj*
+ Deserialize(Obj *self, InStream *instream);
+
+ /** Return a representation of the object using only scalars, hashes, and
+ * arrays. Some implementations support JSON serialization via Dump() and
+ * its companion method, Load(); for others, Dump() is only a debugging
+ * aid. The default simply calls To_String().
+ */
+ public incremented Obj*
+ Dump(Obj *self);
+
+ /** Create an object from the output of a call to Dump(). Implementations
+ * must not reference the caller.
+ *
+ * @param dump The output of Dump().
+ */
+ public abstract incremented Obj*
+ Load(Obj *self, Obj *dump);
+
+ /** Update the internal state of the object to mimic that of
+ * <code>other</code>.
+ */
+ public abstract void
+ Mimic(Obj *self, Obj *other);
+}
+
+__C__
+static CHY_INLINE void
+lucy_Obj_super_destroy(lucy_Obj *self, lucy_VTable *vtable) {
+ lucy_Obj_destroy_t super_destroy
+ = (lucy_Obj_destroy_t)LUCY_SUPER_METHOD(vtable, Obj, Destroy);
+ super_destroy(self);
+}
+
+#define LUCY_SUPER_DESTROY(_self, _vtable) \
+ lucy_Obj_super_destroy((lucy_Obj*)_self, _vtable)
+
+static CHY_INLINE lucy_Obj*
+lucy_Obj_incref(lucy_Obj *self) {
+ if (self != NULL) { return Lucy_Obj_Inc_RefCount(self); }
+ else { return NULL; }
+}
+
+#define LUCY_INCREF(_self) lucy_Obj_incref((lucy_Obj*)_self)
+
+static CHY_INLINE uint32_t
+lucy_Obj_decref(lucy_Obj *self) {
+ if (self != NULL) { return Lucy_Obj_Dec_RefCount(self); }
+ else { return 0; }
+}
+
+#define LUCY_DECREF(_self) lucy_Obj_decref((lucy_Obj*)_self)
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define SUPER_DESTROY(_self, _vtable) LUCY_SUPER_DESTROY(_self, _vtable)
+ #define INCREF(_self) LUCY_INCREF(_self)
+ #define DECREF(_self) LUCY_DECREF(_self)
+#endif
+
+__END_C__
+
+
diff --git a/core/Lucy/Object/VArray.c b/core/Lucy/Object/VArray.c
new file mode 100644
index 0000000..b822263
--- /dev/null
+++ b/core/Lucy/Object/VArray.c
@@ -0,0 +1,366 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_VARRAY
+#include <string.h>
+#include <stdlib.h>
+
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Object/VArray.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/Freezer.h"
+#include "Lucy/Util/SortUtils.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+VArray*
+VA_new(uint32_t capacity) {
+ VArray *self = (VArray*)VTable_Make_Obj(VARRAY);
+ VA_init(self, capacity);
+ return self;
+}
+
+VArray*
+VA_init(VArray *self, uint32_t capacity) {
+ // Init.
+ self->size = 0;
+
+ // Assign.
+ self->cap = capacity;
+
+ // Derive.
+ self->elems = (Obj**)CALLOCATE(capacity, sizeof(Obj*));
+
+ return self;
+}
+
+void
+VA_destroy(VArray *self) {
+ if (self->elems) {
+ Obj **elems = self->elems;
+ Obj **const limit = elems + self->size;
+ for (; elems < limit; elems++) {
+ DECREF(*elems);
+ }
+ FREEMEM(self->elems);
+ }
+ SUPER_DESTROY(self, VARRAY);
+}
+
+VArray*
+VA_dump(VArray *self) {
+ VArray *dump = VA_new(self->size);
+ uint32_t i, max;
+ for (i = 0, max = self->size; i < max; i++) {
+ Obj *elem = VA_Fetch(self, i);
+ if (elem) { VA_Store(dump, i, Obj_Dump(elem)); }
+ }
+ return dump;
+}
+
+VArray*
+VA_load(VArray *self, Obj *dump) {
+ VArray *source = (VArray*)CERTIFY(dump, VARRAY);
+ VArray *loaded = VA_new(source->size);
+ uint32_t i, max;
+ UNUSED_VAR(self);
+
+ for (i = 0, max = source->size; i < max; i++) {
+ Obj *elem_dump = VA_Fetch(source, i);
+ if (elem_dump) {
+ VA_Store(loaded, i, Obj_Load(elem_dump, elem_dump));
+ }
+ }
+
+ return loaded;
+}
+
+void
+VA_serialize(VArray *self, OutStream *outstream) {
+ uint32_t i;
+ uint32_t last_valid_tick = 0;
+ OutStream_Write_C32(outstream, self->size);
+ for (i = 0; i < self->size; i++) {
+ Obj *elem = self->elems[i];
+ if (elem) {
+ OutStream_Write_C32(outstream, i - last_valid_tick);
+ FREEZE(elem, outstream);
+ last_valid_tick = i;
+ }
+ }
+ // Terminate.
+ OutStream_Write_C32(outstream, self->size - last_valid_tick);
+}
+
+VArray*
+VA_deserialize(VArray *self, InStream *instream) {
+ uint32_t tick;
+ uint32_t size = InStream_Read_C32(instream);
+ if (self) {
+ self->size = size;
+ self->cap = size + 1;
+ self->elems = (Obj**)CALLOCATE(self->cap, sizeof(Obj*));
+ }
+ else { self = VA_new(size); }
+ for (tick = InStream_Read_C32(instream);
+ tick < size;
+ tick += InStream_Read_C32(instream)
+ ) {
+ Obj *obj = THAW(instream);
+ self->elems[tick] = obj;
+ }
+ self->size = size;
+ return self;
+}
+
+VArray*
+VA_clone(VArray *self) {
+ uint32_t i;
+ VArray *twin = VA_new(self->size);
+
+ // Clone each element.
+ for (i = 0; i < self->size; i++) {
+ Obj *elem = self->elems[i];
+ if (elem) {
+ twin->elems[i] = Obj_Clone(elem);
+ }
+ }
+
+ // Ensure that size is the same if NULL elems at end.
+ twin->size = self->size;
+
+ return twin;
+}
+
+VArray*
+VA_shallow_copy(VArray *self) {
+ uint32_t i;
+ VArray *twin;
+ Obj **elems;
+
+ // Dupe, then increment refcounts.
+ twin = VA_new(self->size);
+ elems = twin->elems;
+ memcpy(elems, self->elems, self->size * sizeof(Obj*));
+ twin->size = self->size;
+ for (i = 0; i < self->size; i++) {
+ if (elems[i] != NULL) {
+ (void)INCREF(elems[i]);
+ }
+ }
+
+ return twin;
+}
+
+void
+VA_push(VArray *self, Obj *element) {
+ if (self->size == self->cap) {
+ VA_Grow(self, Memory_oversize(self->size + 1, sizeof(Obj*)));
+ }
+ self->elems[self->size] = element;
+ self->size++;
+}
+
+void
+VA_push_varray(VArray *self, VArray *other) {
+ uint32_t i;
+ uint32_t tick = self->size;
+ uint32_t new_size = self->size + other->size;
+ if (new_size > self->cap) {
+ VA_Grow(self, Memory_oversize(new_size, sizeof(Obj*)));
+ }
+ for (i = 0; i < other->size; i++, tick++) {
+ Obj *elem = VA_Fetch(other, i);
+ if (elem != NULL) {
+ self->elems[tick] = INCREF(elem);
+ }
+ }
+ self->size = new_size;
+}
+
+Obj*
+VA_pop(VArray *self) {
+ if (!self->size) {
+ return NULL;
+ }
+ self->size--;
+ return self->elems[self->size];
+}
+
+void
+VA_unshift(VArray *self, Obj *elem) {
+ if (self->size == self->cap) {
+ VA_Grow(self, Memory_oversize(self->size + 1, sizeof(Obj*)));
+ }
+ memmove(self->elems + 1, self->elems, self->size * sizeof(Obj*));
+ self->elems[0] = elem;
+ self->size++;
+}
+
+Obj*
+VA_shift(VArray *self) {
+ if (!self->size) {
+ return NULL;
+ }
+ else {
+ Obj *const return_val = self->elems[0];
+ self->size--;
+ if (self->size > 0) {
+ memmove(self->elems, self->elems + 1,
+ self->size * sizeof(Obj*));
+ }
+ return return_val;
+ }
+}
+
+Obj*
+VA_fetch(VArray *self, uint32_t num) {
+ if (num >= self->size) {
+ return NULL;
+ }
+
+ return self->elems[num];
+}
+
+void
+VA_store(VArray *self, uint32_t tick, Obj *elem) {
+ if (tick >= self->cap) {
+ VA_Grow(self, Memory_oversize(tick + 1, sizeof(Obj*)));
+ }
+ if (tick < self->size) { DECREF(self->elems[tick]); }
+ else { self->size = tick + 1; }
+ self->elems[tick] = elem;
+}
+
+void
+VA_grow(VArray *self, uint32_t capacity) {
+ if (capacity > self->cap) {
+ self->elems = (Obj**)REALLOCATE(self->elems, capacity * sizeof(Obj*));
+ self->cap = capacity;
+ memset(self->elems + self->size, 0,
+ (capacity - self->size) * sizeof(Obj*));
+ }
+}
+
+Obj*
+VA_delete(VArray *self, uint32_t num) {
+ Obj *elem = NULL;
+ if (num < self->size) {
+ elem = self->elems[num];
+ self->elems[num] = NULL;
+ }
+ return elem;
+}
+
+void
+VA_excise(VArray *self, uint32_t offset, uint32_t length) {
+ uint32_t i;
+ uint32_t num_to_move;
+
+ if (self->size <= offset) { return; }
+ else if (self->size < offset + length) { length = self->size - offset; }
+
+ for (i = 0; i < length; i++) {
+ DECREF(self->elems[offset + i]);
+ }
+
+ num_to_move = self->size - (offset + length);
+ memmove(self->elems + offset, self->elems + offset + length,
+ num_to_move * sizeof(Obj*));
+ self->size -= length;
+}
+
+void
+VA_clear(VArray *self) {
+ VA_excise(self, 0, self->size);
+}
+
+void
+VA_resize(VArray *self, uint32_t size) {
+ if (size < self->size) {
+ VA_Excise(self, size, self->size - size);
+ }
+ else if (size > self->size) {
+ VA_Grow(self, size);
+ }
+ self->size = size;
+}
+
+uint32_t
+VA_get_size(VArray *self) {
+ return self->size;
+}
+
+uint32_t
+VA_get_capacity(VArray *self) {
+ return self->cap;
+}
+
+static int
+S_default_compare(void *context, const void *va, const void *vb) {
+ Obj *a = *(Obj**)va;
+ Obj *b = *(Obj**)vb;
+ UNUSED_VAR(context);
+ if (a != NULL && b != NULL) { return Obj_Compare_To(a, b); }
+ else if (a == NULL && b == NULL) { return 0; }
+ else if (a == NULL) { return 1; } // NULL to the back
+ else /* b == NULL */ { return -1; } // NULL to the back
+}
+
+void
+VA_sort(VArray *self, lucy_Sort_compare_t compare, void *context) {
+ if (!compare) { compare = S_default_compare; }
+ Sort_quicksort(self->elems, self->size, sizeof(void*), compare, context);
+}
+
+bool_t
+VA_equals(VArray *self, Obj *other) {
+ VArray *twin = (VArray*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, VARRAY)) { return false; }
+ if (twin->size != self->size) {
+ return false;
+ }
+ else {
+ uint32_t i, max;
+ for (i = 0, max = self->size; i < max; i++) {
+ Obj *val = self->elems[i];
+ Obj *other_val = twin->elems[i];
+ if ((val && !other_val) || (other_val && !val)) { return false; }
+ if (val && !Obj_Equals(val, other_val)) { return false; }
+ }
+ }
+ return true;
+}
+
+VArray*
+VA_gather(VArray *self, lucy_VA_gather_test_t test, void *data) {
+ uint32_t i, max;
+ VArray *gathered = VA_new(self->size);
+ for (i = 0, max = self->size; i < max; i++) {
+ if (test(self, i, data)) {
+ Obj *elem = self->elems[i];
+ VA_Push(gathered, elem ? INCREF(elem) : NULL);
+ }
+ }
+ return gathered;
+}
+
+
diff --git a/core/Lucy/Object/VArray.cfh b/core/Lucy/Object/VArray.cfh
new file mode 100644
index 0000000..a3bd060
--- /dev/null
+++ b/core/Lucy/Object/VArray.cfh
@@ -0,0 +1,171 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+#include "Lucy/Util/SortUtils.h"
+
+typedef chy_bool_t
+(*lucy_VA_gather_test_t)(lucy_VArray *self, uint32_t tick, void *data);
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define lucy_VA_gather_test_t lucy_VA_gather_test_t
+#endif
+__END_C__
+
+/** Variable-sized array.
+ */
+class Lucy::Object::VArray cnick VA inherits Lucy::Object::Obj {
+
+ Obj **elems;
+ uint32_t size;
+ uint32_t cap;
+
+ inert incremented VArray*
+ new(uint32_t capacity);
+
+ /**
+ * @param capacity Initial number of elements that the object will be able
+ * to hold before reallocation.
+ */
+ inert VArray*
+ init(VArray *self, uint32_t capacity);
+
+ /** Push an item onto the end of a VArray.
+ */
+ void
+ Push(VArray *self, decremented Obj *element);
+
+ /** Push all the elements of another VArray onto the end of this one.
+ */
+ void
+ Push_VArray(VArray *self, VArray *other);
+
+ /** Pop an item off of the end of a VArray.
+ */
+ incremented nullable Obj*
+ Pop(VArray *self);
+
+ /** Unshift an item onto the front of a VArray.
+ */
+ void
+ Unshift(VArray *self, decremented Obj *element);
+
+ /** Shift an item off of the front of a VArray.
+ */
+ incremented nullable Obj*
+ Shift(VArray *self);
+
+ /** Ensure that the VArray has room for at least <code>capacity</code>
+ * elements.
+ */
+ void
+ Grow(VArray *self, uint32_t capacity);
+
+ /** Fetch the element at <code>tick</tick>.
+ */
+ nullable Obj*
+ Fetch(VArray *self, uint32_t tick);
+
+ /** Store an element at index <code>tick</code>, possibly displacing an
+ * existing element.
+ */
+ void
+ Store(VArray *self, uint32_t tick, decremented Obj *elem);
+
+ /** Replace an element in the VArray with NULL and return it.
+ *
+ * @return whatever was stored at <code>tick</code>.
+ */
+ incremented nullable Obj*
+ Delete(VArray *self, uint32_t tick);
+
+ /** Remove <code>length</code> elements from the array, starting at
+ * <code>offset</code>. Move elements over to fill in the gap.
+ */
+ void
+ Excise(VArray *self, uint32_t offset, uint32_t length);
+
+ /** Clone the VArray but merely increment the refcounts of its elements
+ * rather than clone them.
+ */
+ incremented VArray*
+ Shallow_Copy(VArray *self);
+
+ /** Dupe the VArray, cloning each internal element.
+ */
+ public incremented VArray*
+ Clone(VArray *self);
+
+ /** Quicksort the VArry using the supplied comparison routine. Safety
+ * checks are the responsibility of the caller.
+ *
+ * @param compare Comparison routine. The default uses Obj_Compare_To and
+ * sorts NULL elements towards the end.
+ * @param context Argument supplied to the comparison routine.
+ */
+ void
+ Sort(VArray *self, lucy_Sort_compare_t compare = NULL,
+ void *context = NULL);
+
+ /** Set the size for the VArray. If the new size is larger than the
+ * current size, grow the object to accommodate NULL elements; if smaller
+ * than the current size, decrement and discard truncated elements.
+ */
+ void
+ Resize(VArray *self, uint32_t size);
+
+ /** Empty the VArray.
+ */
+ void
+ Clear(VArray *self);
+
+ /** Accessor for <code>size</code> member.
+ */
+ public uint32_t
+ Get_Size(VArray *self);
+
+ /** Accessor for <code>capacity</code> member.
+ */
+ uint32_t
+ Get_Capacity(VArray *self);
+
+ /** Return all elements for which <code>test</code> returns true.
+ */
+ public incremented VArray*
+ Gather(VArray *self, lucy_VA_gather_test_t test, void *data);
+
+ public bool_t
+ Equals(VArray *self, Obj *other);
+
+ public incremented VArray*
+ Dump(VArray *self);
+
+ public incremented VArray*
+ Load(VArray *self, Obj *dump);
+
+ public void
+ Serialize(VArray *self, OutStream *outstream);
+
+ public incremented VArray*
+ Deserialize(VArray *self, InStream *instream);
+
+ public void
+ Destroy(VArray *self);
+}
+
+
diff --git a/core/Lucy/Object/VTable.c b/core/Lucy/Object/VTable.c
new file mode 100644
index 0000000..f5567e8
--- /dev/null
+++ b/core/Lucy/Object/VTable.c
@@ -0,0 +1,283 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_VTABLE
+#define C_LUCY_OBJ
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <string.h>
+#include <ctype.h>
+
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/Hash.h"
+#include "Lucy/Object/LockFreeRegistry.h"
+#include "Lucy/Object/VArray.h"
+#include "Lucy/Util/Atomic.h"
+#include "Lucy/Util/Memory.h"
+
+size_t VTable_offset_of_parent = offsetof(VTable, parent);
+
+// Remove spaces and underscores, convert to lower case.
+static void
+S_scrunch_charbuf(CharBuf *source, CharBuf *target);
+
+LockFreeRegistry *VTable_registry = NULL;
+
+void
+VTable_destroy(VTable *self) {
+ THROW(ERR, "Insane attempt to destroy VTable for class '%o'", self->name);
+}
+
+VTable*
+VTable_clone(VTable *self) {
+ VTable *twin
+ = (VTable*)Memory_wrapped_calloc(self->vt_alloc_size, 1);
+
+ memcpy(twin, self, self->vt_alloc_size);
+ twin->name = CB_Clone(self->name);
+ twin->ref.count = 1;
+
+ return twin;
+}
+
+Obj*
+VTable_inc_refcount(VTable *self) {
+ return (Obj*)self;
+}
+
+uint32_t
+VTable_dec_refcount(VTable *self) {
+ UNUSED_VAR(self);
+ return 1;
+}
+
+uint32_t
+VTable_get_refcount(VTable *self) {
+ UNUSED_VAR(self);
+ /* VTable_Get_RefCount() lies to other Lucy code about the refcount
+ * because we don't want to have to synchronize access to the cached host
+ * object to which we have delegated responsibility for keeping refcounts.
+ * It always returns 1 because 1 is a positive number, and thus other Lucy
+ * code will be fooled into believing it never needs to take action such
+ * as initiating a destructor.
+ *
+ * It's possible that the host has in fact increased the refcount of the
+ * cached host object if there are multiple refs to it on the other side
+ * of the Lucy/host border, but returning 1 is good enough to fool Lucy
+ * code.
+ */
+ return 1;
+}
+
+void
+VTable_override(VTable *self, lucy_method_t method, size_t offset) {
+ union { char *char_ptr; lucy_method_t *func_ptr; } pointer;
+ pointer.char_ptr = ((char*)self) + offset;
+ pointer.func_ptr[0] = method;
+}
+
+CharBuf*
+VTable_get_name(VTable *self) {
+ return self->name;
+}
+
+VTable*
+VTable_get_parent(VTable *self) {
+ return self->parent;
+}
+
+size_t
+VTable_get_obj_alloc_size(VTable *self) {
+ return self->obj_alloc_size;
+}
+
+void
+VTable_init_registry() {
+ LockFreeRegistry *reg = LFReg_new(256);
+ if (Atomic_cas_ptr((void*volatile*)&VTable_registry, NULL, reg)) {
+ return;
+ }
+ else {
+ DECREF(reg);
+ }
+}
+
+VTable*
+VTable_singleton(const CharBuf *subclass_name, VTable *parent) {
+ if (VTable_registry == NULL) {
+ VTable_init_registry();
+ }
+
+ VTable *singleton = (VTable*)LFReg_Fetch(VTable_registry, (Obj*)subclass_name);
+ if (singleton == NULL) {
+ VArray *novel_host_methods;
+ uint32_t num_novel;
+
+ if (parent == NULL) {
+ CharBuf *parent_class = VTable_find_parent_class(subclass_name);
+ if (parent_class == NULL) {
+ THROW(ERR, "Class '%o' doesn't descend from %o", subclass_name,
+ OBJ->name);
+ }
+ else {
+ parent = VTable_singleton(parent_class, NULL);
+ DECREF(parent_class);
+ }
+ }
+
+ // Copy source vtable.
+ singleton = VTable_Clone(parent);
+
+ // Turn clone into child.
+ singleton->parent = parent;
+ DECREF(singleton->name);
+ singleton->name = CB_Clone(subclass_name);
+
+ // Allow host methods to override.
+ novel_host_methods = VTable_novel_host_methods(subclass_name);
+ num_novel = VA_Get_Size(novel_host_methods);
+ if (num_novel) {
+ Hash *meths = Hash_new(num_novel);
+ uint32_t i;
+ CharBuf *scrunched = CB_new(0);
+ ZombieCharBuf *callback_name = ZCB_BLANK();
+ for (i = 0; i < num_novel; i++) {
+ CharBuf *meth = (CharBuf*)VA_fetch(novel_host_methods, i);
+ S_scrunch_charbuf(meth, scrunched);
+ Hash_Store(meths, (Obj*)scrunched, INCREF(&EMPTY));
+ }
+ cfish_Callback **callbacks
+ = (cfish_Callback**)singleton->callbacks;
+ for (i = 0; callbacks[i] != NULL; i++) {
+ cfish_Callback *const callback = callbacks[i];
+ ZCB_Assign_Str(callback_name, callback->name,
+ callback->name_len);
+ S_scrunch_charbuf((CharBuf*)callback_name, scrunched);
+ if (Hash_Fetch(meths, (Obj*)scrunched)) {
+ VTable_Override(singleton, callback->func,
+ callback->offset);
+ }
+ }
+ DECREF(scrunched);
+ DECREF(meths);
+ }
+ DECREF(novel_host_methods);
+
+ // Register the new class, both locally and with host.
+ if (VTable_add_to_registry(singleton)) {
+ // Doing this after registering is racy, but hard to fix. :(
+ VTable_register_with_host(singleton, parent);
+ }
+ else {
+ DECREF(singleton);
+ singleton = (VTable*)LFReg_Fetch(VTable_registry, (Obj*)subclass_name);
+ if (!singleton) {
+ THROW(ERR, "Failed to either insert or fetch VTable for '%o'",
+ subclass_name);
+ }
+ }
+ }
+
+ return singleton;
+}
+
+Obj*
+VTable_make_obj(VTable *self) {
+ Obj *obj = (Obj*)Memory_wrapped_calloc(self->obj_alloc_size, 1);
+ obj->vtable = self;
+ obj->ref.count = 1;
+ return obj;
+}
+
+Obj*
+VTable_init_obj(VTable *self, void *allocation) {
+ Obj *obj = (Obj*)allocation;
+ obj->vtable = self;
+ obj->ref.count = 1;
+ return obj;
+}
+
+Obj*
+VTable_load_obj(VTable *self, Obj *dump) {
+ Obj_load_t load = (Obj_load_t)METHOD(self, Obj, Load);
+ if (load == Obj_load) {
+ THROW(ERR, "Abstract method Load() not defined for %o", self->name);
+ }
+ return load(NULL, dump);
+}
+
+static void
+S_scrunch_charbuf(CharBuf *source, CharBuf *target) {
+ ZombieCharBuf *iterator = ZCB_WRAP(source);
+ CB_Set_Size(target, 0);
+ while (ZCB_Get_Size(iterator)) {
+ uint32_t code_point = ZCB_Nip_One(iterator);
+ if (code_point > 127) {
+ THROW(ERR, "Can't fold case for %o", source);
+ }
+ else if (code_point != '_') {
+ CB_Cat_Char(target, tolower(code_point));
+ }
+ }
+}
+
+bool_t
+VTable_add_to_registry(VTable *vtable) {
+ if (VTable_registry == NULL) {
+ VTable_init_registry();
+ }
+ if (LFReg_Fetch(VTable_registry, (Obj*)vtable->name)) {
+ return false;
+ }
+ else {
+ CharBuf *klass = CB_Clone(vtable->name);
+ bool_t retval
+ = LFReg_Register(VTable_registry, (Obj*)klass, (Obj*)vtable);
+ DECREF(klass);
+ return retval;
+ }
+}
+
+bool_t
+VTable_add_alias_to_registry(VTable *vtable, CharBuf *alias) {
+ if (VTable_registry == NULL) {
+ VTable_init_registry();
+ }
+ if (LFReg_Fetch(VTable_registry, (Obj*)alias)) {
+ return false;
+ }
+ else {
+ CharBuf *klass = CB_Clone(alias);
+ bool_t retval
+ = LFReg_Register(VTable_registry, (Obj*)klass, (Obj*)vtable);
+ DECREF(klass);
+ return retval;
+ }
+}
+
+VTable*
+VTable_fetch_vtable(const CharBuf *class_name) {
+ VTable *vtable = NULL;
+ if (VTable_registry != NULL) {
+ vtable = (VTable*)LFReg_Fetch(VTable_registry, (Obj*)class_name);
+ }
+ return vtable;
+}
+
+
diff --git a/core/Lucy/Object/VTable.cfh b/core/Lucy/Object/VTable.cfh
new file mode 100644
index 0000000..9c17b72
--- /dev/null
+++ b/core/Lucy/Object/VTable.cfh
@@ -0,0 +1,148 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Virtual method dispatch table.
+ *
+ * VTables, which are the first element in any Clownfish object, are actually
+ * objects themselves. (Their first element is a VTable which describes the
+ * behavior of VTables.)
+ */
+
+class Lucy::Object::VTable inherits Lucy::Object::Obj {
+
+ VTable *parent;
+ CharBuf *name;
+ uint32_t flags;
+ void *x; /* Reserved for future expansion */
+ size_t obj_alloc_size;
+ size_t vt_alloc_size;
+ void *callbacks;
+ lucy_method_t[1] methods; /* flexible array */
+
+ inert LockFreeRegistry *registry;
+ inert size_t offset_of_parent;
+
+ /** Return a singleton. If a VTable can be found in the registry based on
+ * the subclass name, it will be returned. Otherwise, a new VTable will
+ * be created using [parent] as a base.
+ *
+ * If [parent] is NULL, an attempt will be made to find it using
+ * VTable_find_parent_class(). If the attempt fails, an error will
+ * result.
+ */
+ inert VTable*
+ singleton(const CharBuf *subclass_name, VTable *parent);
+
+ /** Register a vtable, so that it can be retrieved by class name.
+ *
+ * TODO: Move this functionality to some kind of class loader.
+ *
+ * @return true on success, false if the class was already registered.
+ */
+ inert bool_t
+ add_to_registry(VTable *vtable);
+
+ inert bool_t
+ add_alias_to_registry(VTable *vtable, CharBuf *alias);
+
+ /** Initialize the registry.
+ */
+ inert void
+ init_registry();
+
+ /** Tell the host about the new class.
+ */
+ inert void
+ register_with_host(VTable *vtable, VTable *parent);
+
+ /** Find a registered class and return its vtable. May return NULL if the
+ * class is not registered.
+ */
+ inert nullable VTable*
+ fetch_vtable(const CharBuf *class_name);
+
+ /** Given a class name, return the name of a parent class which descends
+ * from Lucy::Object::Obj, or NULL if such a class can't be found.
+ */
+ inert nullable CharBuf*
+ find_parent_class(const CharBuf *class_name);
+
+ /** List all of the methods that a class has overridden via the host
+ * language.
+ */
+ inert incremented VArray*
+ novel_host_methods(const CharBuf *class_name);
+
+ /** Replace a function pointer in the VTable.
+ */
+ void
+ Override(VTable *self, lucy_method_t method_ptr, size_t offset);
+
+ /** Create an empty object of the type defined by the VTable: allocate,
+ * assign its vtable and give it an initial refcount of 1. The caller is
+ * responsible for initialization.
+ */
+ Obj*
+ Make_Obj(VTable *self);
+
+ /** Take a raw memory allocation which is presumed to be of adequate size,
+ * assign its vtable and give it an initial refcount of 1.
+ */
+ Obj*
+ Init_Obj(VTable *self, void *allocation);
+
+ /** Create a new object using the supplied dump, assuming that Load() has
+ * been defined for the class.
+ */
+ Obj*
+ Load_Obj(VTable *self, Obj *dump);
+
+ /** Create a new object to go with the supplied host object.
+ */
+ Obj*
+ Foster_Obj(VTable *self, void *host_obj);
+
+ CharBuf*
+ Get_Name(VTable *self);
+
+ VTable*
+ Get_Parent(VTable *self);
+
+ size_t
+ Get_Obj_Alloc_Size(VTable *self);
+
+ public incremented VTable*
+ Clone(VTable *self);
+
+ incremented Obj*
+ Inc_RefCount(VTable *self);
+
+ uint32_t
+ Dec_RefCount(VTable *self);
+
+ uint32_t
+ Get_RefCount(VTable *self);
+
+ void*
+ To_Host(VTable *self);
+
+ public void
+ Destroy(VTable *self);
+}
+
+
diff --git a/core/Lucy/Plan/Architecture.c b/core/Lucy/Plan/Architecture.c
new file mode 100644
index 0000000..808496e
--- /dev/null
+++ b/core/Lucy/Plan/Architecture.c
@@ -0,0 +1,277 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ARCHITECTURE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/DeletionsWriter.h"
+#include "Lucy/Index/DocReader.h"
+#include "Lucy/Index/DocWriter.h"
+#include "Lucy/Index/HighlightReader.h"
+#include "Lucy/Index/HighlightWriter.h"
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Index/LexiconWriter.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/PostingListWriter.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/SegWriter.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Index/SortReader.h"
+#include "Lucy/Index/SortWriter.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Store/Folder.h"
+
+Architecture*
+Arch_new() {
+ Architecture *self = (Architecture*)VTable_Make_Obj(ARCHITECTURE);
+ return Arch_init(self);
+}
+
+Architecture*
+Arch_init(Architecture *self) {
+ return self;
+}
+
+bool_t
+Arch_equals(Architecture *self, Obj *other) {
+ Architecture *twin = (Architecture*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, ARCHITECTURE)) { return false; }
+ return true;
+}
+
+void
+Arch_init_seg_writer(Architecture *self, SegWriter *writer) {
+ Arch_Register_Lexicon_Writer(self, writer);
+ Arch_Register_Posting_List_Writer(self, writer);
+ Arch_Register_Sort_Writer(self, writer);
+ Arch_Register_Doc_Writer(self, writer);
+ Arch_Register_Highlight_Writer(self, writer);
+ Arch_Register_Deletions_Writer(self, writer);
+}
+
+void
+Arch_register_lexicon_writer(Architecture *self, SegWriter *writer) {
+ Schema *schema = SegWriter_Get_Schema(writer);
+ Snapshot *snapshot = SegWriter_Get_Snapshot(writer);
+ Segment *segment = SegWriter_Get_Segment(writer);
+ PolyReader *polyreader = SegWriter_Get_PolyReader(writer);
+ LexiconWriter *lex_writer
+ = LexWriter_new(schema, snapshot, segment, polyreader);
+ UNUSED_VAR(self);
+ SegWriter_Register(writer, VTable_Get_Name(LEXICONWRITER),
+ (DataWriter*)lex_writer);
+}
+
+void
+Arch_register_posting_list_writer(Architecture *self, SegWriter *writer) {
+ Schema *schema = SegWriter_Get_Schema(writer);
+ Snapshot *snapshot = SegWriter_Get_Snapshot(writer);
+ Segment *segment = SegWriter_Get_Segment(writer);
+ PolyReader *polyreader = SegWriter_Get_PolyReader(writer);
+ LexiconWriter *lex_writer = (LexiconWriter*)SegWriter_Fetch(
+ writer, VTable_Get_Name(LEXICONWRITER));
+ UNUSED_VAR(self);
+ if (!lex_writer) {
+ THROW(ERR, "Can't fetch a LexiconWriter");
+ }
+ else {
+ PostingListWriter *plist_writer
+ = PListWriter_new(schema, snapshot, segment, polyreader,
+ lex_writer);
+ SegWriter_Register(writer, VTable_Get_Name(POSTINGLISTWRITER),
+ (DataWriter*)plist_writer);
+ SegWriter_Add_Writer(writer, (DataWriter*)INCREF(plist_writer));
+ }
+}
+
+void
+Arch_register_doc_writer(Architecture *self, SegWriter *writer) {
+ Schema *schema = SegWriter_Get_Schema(writer);
+ Snapshot *snapshot = SegWriter_Get_Snapshot(writer);
+ Segment *segment = SegWriter_Get_Segment(writer);
+ PolyReader *polyreader = SegWriter_Get_PolyReader(writer);
+ DocWriter *doc_writer
+ = DocWriter_new(schema, snapshot, segment, polyreader);
+ UNUSED_VAR(self);
+ SegWriter_Register(writer, VTable_Get_Name(DOCWRITER),
+ (DataWriter*)doc_writer);
+ SegWriter_Add_Writer(writer, (DataWriter*)INCREF(doc_writer));
+}
+
+void
+Arch_register_sort_writer(Architecture *self, SegWriter *writer) {
+ Schema *schema = SegWriter_Get_Schema(writer);
+ Snapshot *snapshot = SegWriter_Get_Snapshot(writer);
+ Segment *segment = SegWriter_Get_Segment(writer);
+ PolyReader *polyreader = SegWriter_Get_PolyReader(writer);
+ SortWriter *sort_writer
+ = SortWriter_new(schema, snapshot, segment, polyreader);
+ UNUSED_VAR(self);
+ SegWriter_Register(writer, VTable_Get_Name(SORTWRITER),
+ (DataWriter*)sort_writer);
+ SegWriter_Add_Writer(writer, (DataWriter*)INCREF(sort_writer));
+}
+
+void
+Arch_register_highlight_writer(Architecture *self, SegWriter *writer) {
+ Schema *schema = SegWriter_Get_Schema(writer);
+ Snapshot *snapshot = SegWriter_Get_Snapshot(writer);
+ Segment *segment = SegWriter_Get_Segment(writer);
+ PolyReader *polyreader = SegWriter_Get_PolyReader(writer);
+ HighlightWriter *hl_writer
+ = HLWriter_new(schema, snapshot, segment, polyreader);
+ UNUSED_VAR(self);
+ SegWriter_Register(writer, VTable_Get_Name(HIGHLIGHTWRITER),
+ (DataWriter*)hl_writer);
+ SegWriter_Add_Writer(writer, (DataWriter*)INCREF(hl_writer));
+}
+
+void
+Arch_register_deletions_writer(Architecture *self, SegWriter *writer) {
+ Schema *schema = SegWriter_Get_Schema(writer);
+ Snapshot *snapshot = SegWriter_Get_Snapshot(writer);
+ Segment *segment = SegWriter_Get_Segment(writer);
+ PolyReader *polyreader = SegWriter_Get_PolyReader(writer);
+ DefaultDeletionsWriter *del_writer
+ = DefDelWriter_new(schema, snapshot, segment, polyreader);
+ UNUSED_VAR(self);
+ SegWriter_Register(writer, VTable_Get_Name(DELETIONSWRITER),
+ (DataWriter*)del_writer);
+ SegWriter_Set_Del_Writer(writer, (DeletionsWriter*)del_writer);
+}
+
+void
+Arch_init_seg_reader(Architecture *self, SegReader *reader) {
+ Arch_Register_Doc_Reader(self, reader);
+ Arch_Register_Lexicon_Reader(self, reader);
+ Arch_Register_Posting_List_Reader(self, reader);
+ Arch_Register_Sort_Reader(self, reader);
+ Arch_Register_Highlight_Reader(self, reader);
+ Arch_Register_Deletions_Reader(self, reader);
+}
+
+void
+Arch_register_doc_reader(Architecture *self, SegReader *reader) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Folder *folder = SegReader_Get_Folder(reader);
+ VArray *segments = SegReader_Get_Segments(reader);
+ Snapshot *snapshot = SegReader_Get_Snapshot(reader);
+ int32_t seg_tick = SegReader_Get_Seg_Tick(reader);
+ DefaultDocReader *doc_reader
+ = DefDocReader_new(schema, folder, snapshot, segments, seg_tick);
+ UNUSED_VAR(self);
+ SegReader_Register(reader, VTable_Get_Name(DOCREADER),
+ (DataReader*)doc_reader);
+}
+
+void
+Arch_register_posting_list_reader(Architecture *self, SegReader *reader) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Folder *folder = SegReader_Get_Folder(reader);
+ VArray *segments = SegReader_Get_Segments(reader);
+ Snapshot *snapshot = SegReader_Get_Snapshot(reader);
+ int32_t seg_tick = SegReader_Get_Seg_Tick(reader);
+ LexiconReader *lex_reader = (LexiconReader*)SegReader_Obtain(
+ reader, VTable_Get_Name(LEXICONREADER));
+ DefaultPostingListReader *plist_reader
+ = DefPListReader_new(schema, folder, snapshot, segments, seg_tick,
+ lex_reader);
+ UNUSED_VAR(self);
+ SegReader_Register(reader, VTable_Get_Name(POSTINGLISTREADER),
+ (DataReader*)plist_reader);
+}
+
+void
+Arch_register_lexicon_reader(Architecture *self, SegReader *reader) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Folder *folder = SegReader_Get_Folder(reader);
+ VArray *segments = SegReader_Get_Segments(reader);
+ Snapshot *snapshot = SegReader_Get_Snapshot(reader);
+ int32_t seg_tick = SegReader_Get_Seg_Tick(reader);
+ DefaultLexiconReader *lex_reader
+ = DefLexReader_new(schema, folder, snapshot, segments, seg_tick);
+ UNUSED_VAR(self);
+ SegReader_Register(reader, VTable_Get_Name(LEXICONREADER),
+ (DataReader*)lex_reader);
+}
+
+void
+Arch_register_sort_reader(Architecture *self, SegReader *reader) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Folder *folder = SegReader_Get_Folder(reader);
+ VArray *segments = SegReader_Get_Segments(reader);
+ Snapshot *snapshot = SegReader_Get_Snapshot(reader);
+ int32_t seg_tick = SegReader_Get_Seg_Tick(reader);
+ DefaultSortReader *sort_reader
+ = DefSortReader_new(schema, folder, snapshot, segments, seg_tick);
+ UNUSED_VAR(self);
+ SegReader_Register(reader, VTable_Get_Name(SORTREADER),
+ (DataReader*)sort_reader);
+}
+
+void
+Arch_register_highlight_reader(Architecture *self, SegReader *reader) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Folder *folder = SegReader_Get_Folder(reader);
+ VArray *segments = SegReader_Get_Segments(reader);
+ Snapshot *snapshot = SegReader_Get_Snapshot(reader);
+ int32_t seg_tick = SegReader_Get_Seg_Tick(reader);
+ DefaultHighlightReader* hl_reader
+ = DefHLReader_new(schema, folder, snapshot, segments, seg_tick);
+ UNUSED_VAR(self);
+ SegReader_Register(reader, VTable_Get_Name(HIGHLIGHTREADER),
+ (DataReader*)hl_reader);
+}
+
+void
+Arch_register_deletions_reader(Architecture *self, SegReader *reader) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Folder *folder = SegReader_Get_Folder(reader);
+ VArray *segments = SegReader_Get_Segments(reader);
+ Snapshot *snapshot = SegReader_Get_Snapshot(reader);
+ int32_t seg_tick = SegReader_Get_Seg_Tick(reader);
+ DefaultDeletionsReader* del_reader
+ = DefDelReader_new(schema, folder, snapshot, segments, seg_tick);
+ UNUSED_VAR(self);
+ SegReader_Register(reader, VTable_Get_Name(DELETIONSREADER),
+ (DataReader*)del_reader);
+}
+
+Similarity*
+Arch_make_similarity(Architecture *self) {
+ UNUSED_VAR(self);
+ return Sim_new();
+}
+
+int32_t
+Arch_index_interval(Architecture *self) {
+ UNUSED_VAR(self);
+ return 128;
+}
+
+int32_t
+Arch_skip_interval(Architecture *self) {
+ UNUSED_VAR(self);
+ return 16;
+}
+
+
diff --git a/core/Lucy/Plan/Architecture.cfh b/core/Lucy/Plan/Architecture.cfh
new file mode 100644
index 0000000..a1d6c9e
--- /dev/null
+++ b/core/Lucy/Plan/Architecture.cfh
@@ -0,0 +1,160 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Configure major components of an index.
+ *
+ * By default, a Lucy index consists of several main parts: lexicon,
+ * postings, stored documents, deletions, and highlight data. The readers and
+ * writers for that data are spawned by Architecture. Each component operates
+ * at the segment level; Architecture's factory methods are used to build up
+ * L<SegWriter|Lucy::Index::SegWriter> and
+ * L<SegReader|Lucy::Index::SegReader>.
+ */
+class Lucy::Plan::Architecture cnick Arch inherits Lucy::Object::Obj {
+
+ public inert incremented Architecture*
+ new();
+
+ /** Constructor. Takes no arguments.
+ */
+ public inert Architecture*
+ init(Architecture *self);
+
+ /** Initialize a SegWriter, adding DataWriter components.
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Init_Seg_Writer(Architecture *self, SegWriter *writer);
+
+ /** Spawn a LexiconWriter and Register() it with the supplied SegWriter,
+ * but don't add it to the SegWriter's writer stack.
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Register_Lexicon_Writer(Architecture *self, SegWriter *writer);
+
+ /** Spawn a PostingListWriter and Register() it with the supplied
+ * SegWriter, adding it to the SegWriter's writer stack. The SegWriter
+ * must contain a previously registered LexiconWriter.
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Register_Posting_List_Writer(Architecture *self, SegWriter *writer);
+
+ /** Spawn a DataWriter and Register() it with the supplied SegWriter,
+ * adding it to the SegWriter's writer stack.
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Register_Doc_Writer(Architecture *self, SegWriter *writer);
+
+ /** Spawn a SortWriter and Register() it with the supplied SegWriter,
+ * adding it to the SegWriter's writer stack.
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Register_Sort_Writer(Architecture *self, SegWriter *writer);
+
+ /** Spawn a HighlightWriter and Register() it with the supplied SegWriter,
+ * adding it to the SegWriter's writer stack.
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Register_Highlight_Writer(Architecture *self, SegWriter *writer);
+
+ /** Spawn a DeletionsWriter and Register() it with the supplied SegWriter,
+ * also calling Set_Del_Writer().
+ *
+ * @param writer A SegWriter.
+ */
+ public void
+ Register_Deletions_Writer(Architecture *self, SegWriter *writer);
+
+ /** Initialize a SegReader, registering DataReaders.
+ */
+ public void
+ Init_Seg_Reader(Architecture *self, SegReader *reader);
+
+ /** Spawn a DocReader and Register() it with the supplied SegReader.
+ *
+ * @param reader A SegReader.
+ */
+ public void
+ Register_Doc_Reader(Architecture *self, SegReader *reader);
+
+ /** Spawn a PostingListReader and Register() it with the supplied SegReader.
+ *
+ * @param reader A SegReader.
+ */
+ public void
+ Register_Posting_List_Reader(Architecture *self, SegReader *reader);
+
+ /** Spawn a SortReader and Register() it with the supplied SegReader.
+ *
+ * @param reader A SegReader.
+ */
+ public void
+ Register_Sort_Reader(Architecture *self, SegReader *reader);
+
+ /** Spawn a HighlightReader and Register() it with the supplied
+ * SegReader.
+ *
+ * @param reader A SegReader.
+ */
+ public void
+ Register_Highlight_Reader(Architecture *self, SegReader *reader);
+
+ /** Spawn a LexiconReader and Register() it with the supplied SegReader.
+ *
+ * @param reader A SegReader.
+ */
+ public void
+ Register_Lexicon_Reader(Architecture *self, SegReader *reader);
+
+ /** Spawn a DeletionsReader and Register() it with the supplied SegReader.
+ *
+ * @param reader A SegReader.
+ */
+ public void
+ Register_Deletions_Reader(Architecture *self, SegReader *reader);
+
+ /** Factory method for creating a new Similarity object.
+ */
+ public Similarity*
+ Make_Similarity(Architecture *self);
+
+ public int32_t
+ Index_Interval(Architecture *self);
+
+ public int32_t
+ Skip_Interval(Architecture *self);
+
+ /** Returns true for any Architecture object. Subclasses should override
+ * this weak check.
+ */
+ public bool_t
+ Equals(Architecture *self, Obj *other);
+}
+
+
diff --git a/core/Lucy/Plan/BlobType.c b/core/Lucy/Plan/BlobType.c
new file mode 100644
index 0000000..8211aba
--- /dev/null
+++ b/core/Lucy/Plan/BlobType.c
@@ -0,0 +1,118 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BLOBTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/BlobType.h"
+
+BlobType*
+BlobType_new(bool_t stored) {
+ BlobType *self = (BlobType*)VTable_Make_Obj(BLOBTYPE);
+ return BlobType_init(self, stored);
+}
+
+BlobType*
+BlobType_init(BlobType *self, bool_t stored) {
+ FType_init((FieldType*)self);
+ self->stored = stored;
+ return self;
+}
+
+bool_t
+BlobType_binary(BlobType *self) {
+ UNUSED_VAR(self);
+ return true;
+}
+
+void
+BlobType_set_sortable(BlobType *self, bool_t sortable) {
+ UNUSED_VAR(self);
+ if (sortable) { THROW(ERR, "BlobType fields can't be sortable"); }
+}
+
+ViewByteBuf*
+BlobType_make_blank(BlobType *self) {
+ UNUSED_VAR(self);
+ return ViewBB_new(NULL, 0);
+}
+
+int8_t
+BlobType_primitive_id(BlobType *self) {
+ UNUSED_VAR(self);
+ return FType_BLOB;
+}
+
+bool_t
+BlobType_equals(BlobType *self, Obj *other) {
+ BlobType *twin = (BlobType*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, BLOBTYPE)) { return false; }
+ return FType_equals((FieldType*)self, other);
+}
+
+Hash*
+BlobType_dump_for_schema(BlobType *self) {
+ Hash *dump = Hash_new(0);
+ Hash_Store_Str(dump, "type", 4, (Obj*)CB_newf("blob"));
+
+ // Store attributes that override the defaults -- even if they're
+ // meaningless.
+ if (self->boost != 1.0) {
+ Hash_Store_Str(dump, "boost", 5, (Obj*)CB_newf("%f64", self->boost));
+ }
+ if (self->indexed) {
+ Hash_Store_Str(dump, "indexed", 7, (Obj*)CB_newf("1"));
+ }
+ if (self->stored) {
+ Hash_Store_Str(dump, "stored", 6, (Obj*)CB_newf("1"));
+ }
+
+ return dump;
+}
+
+Hash*
+BlobType_dump(BlobType *self) {
+ Hash *dump = BlobType_Dump_For_Schema(self);
+ Hash_Store_Str(dump, "_class", 6,
+ (Obj*)CB_Clone(BlobType_Get_Class_Name(self)));
+ DECREF(Hash_Delete_Str(dump, "type", 4));
+ return dump;
+}
+
+BlobType*
+BlobType_load(BlobType *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *class_name = (CharBuf*)Hash_Fetch_Str(source, "_class", 6);
+ VTable *vtable
+ = (class_name != NULL && Obj_Is_A((Obj*)class_name, CHARBUF))
+ ? VTable_singleton(class_name, NULL)
+ : BLOBTYPE;
+ BlobType *loaded = (BlobType*)VTable_Make_Obj(vtable);
+ Obj *boost_dump = Hash_Fetch_Str(source, "boost", 5);
+ Obj *indexed_dump = Hash_Fetch_Str(source, "indexed", 7);
+ Obj *stored_dump = Hash_Fetch_Str(source, "stored", 6);
+ UNUSED_VAR(self);
+
+ BlobType_init(loaded, false);
+ if (boost_dump) { loaded->boost = (float)Obj_To_F64(boost_dump); }
+ if (indexed_dump) { loaded->indexed = (bool_t)Obj_To_I64(indexed_dump); }
+ if (stored_dump) { loaded->stored = (bool_t)Obj_To_I64(stored_dump); }
+
+ return loaded;
+}
+
+
diff --git a/core/Lucy/Plan/BlobType.cfh b/core/Lucy/Plan/BlobType.cfh
new file mode 100644
index 0000000..3398e78
--- /dev/null
+++ b/core/Lucy/Plan/BlobType.cfh
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Default behaviors for binary fields.
+ *
+ * BlobType is an implementation of FieldType tuned for use with fields
+ * containing binary data, which cannot be indexed or searched -- only stored.
+ */
+class Lucy::Plan::BlobType inherits Lucy::Plan::FieldType
+ : dumpable {
+
+ /**
+ * @param stored boolean indicating whether the field should be stored.
+ */
+ public inert BlobType*
+ init(BlobType *self, bool_t stored);
+
+ public inert incremented BlobType*
+ new(bool_t stored);
+
+ /** Returns true.
+ */
+ public bool_t
+ Binary(BlobType *self);
+
+ /** Throws an error unless <code>sortable</code> is false.
+ */
+ public void
+ Set_Sortable(BlobType *self, bool_t sortable);
+
+ incremented ViewByteBuf*
+ Make_Blank(BlobType *self);
+
+ int8_t
+ Primitive_ID(BlobType *self);
+
+ incremented Hash*
+ Dump_For_Schema(BlobType *self);
+
+ public incremented Hash*
+ Dump(BlobType *self);
+
+ public incremented BlobType*
+ Load(BlobType *self, Obj *dump);
+
+ public bool_t
+ Equals(BlobType *self, Obj *other);
+}
+
+
diff --git a/core/Lucy/Plan/FieldType.c b/core/Lucy/Plan/FieldType.c
new file mode 100644
index 0000000..59976e8
--- /dev/null
+++ b/core/Lucy/Plan/FieldType.c
@@ -0,0 +1,112 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FIELDTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/Similarity.h"
+
+FieldType*
+FType_init(FieldType *self) {
+ return FType_init2(self, 1.0f, false, false, false);
+}
+
+FieldType*
+FType_init2(FieldType *self, float boost, bool_t indexed, bool_t stored,
+ bool_t sortable) {
+ self->boost = boost;
+ self->indexed = indexed;
+ self->stored = stored;
+ self->sortable = sortable;
+ ABSTRACT_CLASS_CHECK(self, FIELDTYPE);
+ return self;
+}
+
+void
+FType_set_boost(FieldType *self, float boost) {
+ self->boost = boost;
+}
+
+void
+FType_set_indexed(FieldType *self, bool_t indexed) {
+ self->indexed = !!indexed;
+}
+
+void
+FType_set_stored(FieldType *self, bool_t stored) {
+ self->stored = !!stored;
+}
+
+void
+FType_set_sortable(FieldType *self, bool_t sortable) {
+ self->sortable = !!sortable;
+}
+
+float
+FType_get_boost(FieldType *self) {
+ return self->boost;
+}
+
+bool_t
+FType_indexed(FieldType *self) {
+ return self->indexed;
+}
+
+bool_t
+FType_stored(FieldType *self) {
+ return self->stored;
+}
+
+bool_t
+FType_sortable(FieldType *self) {
+ return self->sortable;
+}
+
+bool_t
+FType_binary(FieldType *self) {
+ UNUSED_VAR(self);
+ return false;
+}
+
+Similarity*
+FType_similarity(FieldType *self) {
+ UNUSED_VAR(self);
+ return NULL;
+}
+
+int32_t
+FType_compare_values(FieldType *self, Obj *a, Obj *b) {
+ UNUSED_VAR(self);
+ return Obj_Compare_To(a, b);
+}
+
+bool_t
+FType_equals(FieldType *self, Obj *other) {
+ FieldType *twin = (FieldType*)other;
+ if (twin == self) { return true; }
+ if (FType_Get_VTable(self) != FType_Get_VTable(twin)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!!self->indexed != !!twin->indexed) { return false; }
+ if (!!self->stored != !!twin->stored) { return false; }
+ if (!!self->sortable != !!twin->sortable) { return false; }
+ if (!!FType_Binary(self) != !!FType_Binary(twin)) { return false; }
+ return true;
+}
+
+
diff --git a/core/Lucy/Plan/FieldType.cfh b/core/Lucy/Plan/FieldType.cfh
new file mode 100644
index 0000000..846e15f
--- /dev/null
+++ b/core/Lucy/Plan/FieldType.cfh
@@ -0,0 +1,187 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+
+#define lucy_FType_TEXT 1
+#define lucy_FType_BLOB 2
+#define lucy_FType_INT32 3
+#define lucy_FType_INT64 4
+#define lucy_FType_FLOAT32 5
+#define lucy_FType_FLOAT64 6
+#define lucy_FType_PRIMITIVE_ID_MASK 0x7
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define FType_TEXT lucy_FType_TEXT
+ #define FType_BLOB lucy_FType_BLOB
+ #define FType_INT32 lucy_FType_INT32
+ #define FType_INT64 lucy_FType_INT64
+ #define FType_FLOAT32 lucy_FType_FLOAT32
+ #define FType_FLOAT64 lucy_FType_FLOAT64
+ #define FType_PRIMITIVE_ID_MASK lucy_FType_PRIMITIVE_ID_MASK
+#endif
+
+__END_C__
+
+/** Define a field's behavior.
+ *
+ * FieldType is an abstract class defining a set of traits and behaviors which
+ * may be associated with one or more field names.
+ *
+ * Properties which are common to all field types include <code>boost</code>,
+ * <code>indexed</code>, <code>stored</code>, <code>sortable</code>,
+ * <code>binary</code>, and <code>similarity</code>.
+ *
+ * The <code>boost</code> property is a floating point scoring multiplier
+ * which defaults to 1.0. Values greater than 1.0 cause the field to
+ * contribute more to a document's score, lower values, less.
+ *
+ * The <code>indexed</code> property indicates whether the field should be
+ * indexed (so that it can be searched).
+ *
+ * The <code>stored</code> property indicates whether to store the raw field
+ * value, so that it can be retrieved when a document turns up in a search.
+ *
+ * The <code>sortable</code> property indicates whether search results should
+ * be sortable based on the contents of the field.
+ *
+ * The <code>binary</code> property indicates whether the field contains
+ * binary or text data. Unlike most other properties, <code>binary</code> is
+ * not settable.
+ *
+ * The <code>similarity</code> property is a
+ * L<Similarity|Lucy::Index::Similarity> object which defines matching
+ * and scoring behavior for the field. It is required if the field is
+ * <code>indexed</code>.
+ */
+abstract class Lucy::Plan::FieldType cnick FType
+ inherits Lucy::Object::Obj {
+
+ float boost;
+ bool_t indexed;
+ bool_t stored;
+ bool_t sortable;
+
+ inert FieldType*
+ init(FieldType *self);
+
+ inert FieldType*
+ init2(FieldType *self, float boost = 1.0, bool_t indexed = false,
+ bool_t stored = false, bool_t sortable = false);
+
+ /** Setter for <code>boost</code>.
+ */
+ public void
+ Set_Boost(FieldType *self, float boost);
+
+ /** Accessor for <code>boost</code>.
+ */
+ public float
+ Get_Boost(FieldType *self);
+
+ /** Setter for <code>indexed</code>.
+ */
+ public void
+ Set_Indexed(FieldType *self, bool_t indexed);
+
+ /** Accessor for <code>indexed</code>.
+ */
+ public bool_t
+ Indexed(FieldType *self);
+
+ /** Setter for <code>stored</code>.
+ */
+ public void
+ Set_Stored(FieldType *self, bool_t stored);
+
+ /** Accessor for <code>stored</code>.
+ */
+ public bool_t
+ Stored(FieldType *self);
+
+ /** Setter for <code>sortable</code>.
+ */
+ public void
+ Set_Sortable(FieldType *self, bool_t sortable);
+
+ /** Accessor for <code>sortable</code>.
+ */
+ public bool_t
+ Sortable(FieldType *self);
+
+ /** Indicate whether the field contains binary data.
+ */
+ public bool_t
+ Binary(FieldType *self);
+
+ /** Compare two values for the field. The default implementation
+ * dispatches to the Compare_To() method of argument <code>a</code>.
+ *
+ * @return a negative number if a is "less than" b, 0 if they are "equal",
+ * and a positive number if a is "greater than" b.
+ */
+ public int32_t
+ Compare_Values(FieldType *self, Obj *a, Obj *b);
+
+ /** NULL-safe comparison wrapper which sorts NULLs towards the back.
+ */
+ inert inline int32_t
+ null_back_compare_values(FieldType *self, Obj *a, Obj *b);
+
+ /** Produce a Stepper suitable for use by a Lexicon.
+ */
+ abstract incremented TermStepper*
+ Make_Term_Stepper(FieldType *self);
+
+ /** Internal id used for switch() ops. Unique for each primitive type.
+ */
+ abstract int8_t
+ Primitive_ID(FieldType *self);
+
+ /** Produce a special mimimal dump which does not include Similarity or
+ * Analyzer dumps. For exclusive internal use by Schema.
+ */
+ abstract incremented Hash*
+ Dump_For_Schema(FieldType *self);
+
+ /** Compares all common properties.
+ */
+ public bool_t
+ Equals(FieldType *self, Obj *other);
+}
+
+__C__
+
+static CHY_INLINE int32_t
+lucy_FType_null_back_compare_values(lucy_FieldType *self,
+ lucy_Obj *a, lucy_Obj *b) {
+ if (a == NULL) {
+ if (b == NULL) { return 0; }
+ else { return 1; }
+ }
+ else if (b == NULL) {
+ return -1;
+ }
+ else {
+ return Lucy_FType_Compare_Values(self, a, b);
+ }
+}
+
+__END_C__
+
+
diff --git a/core/Lucy/Plan/FullTextType.c b/core/Lucy/Plan/FullTextType.c
new file mode 100644
index 0000000..59a249b
--- /dev/null
+++ b/core/Lucy/Plan/FullTextType.c
@@ -0,0 +1,181 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FULLTEXTTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/Similarity.h"
+
+FullTextType*
+FullTextType_new(Analyzer *analyzer) {
+ FullTextType *self = (FullTextType*)VTable_Make_Obj(FULLTEXTTYPE);
+ return FullTextType_init(self, analyzer);
+}
+
+FullTextType*
+FullTextType_init(FullTextType *self, Analyzer *analyzer) {
+ return FullTextType_init2(self, analyzer, 1.0, true, true, false, false);
+}
+
+FullTextType*
+FullTextType_init2(FullTextType *self, Analyzer *analyzer, float boost,
+ bool_t indexed, bool_t stored, bool_t sortable,
+ bool_t highlightable) {
+ FType_init((FieldType*)self);
+
+ /* Assign */
+ self->boost = boost;
+ self->indexed = indexed;
+ self->stored = stored;
+ self->sortable = sortable;
+ self->highlightable = highlightable;
+ self->analyzer = (Analyzer*)INCREF(analyzer);
+
+ return self;
+}
+
+void
+FullTextType_destroy(FullTextType *self) {
+ DECREF(self->analyzer);
+ SUPER_DESTROY(self, FULLTEXTTYPE);
+}
+
+bool_t
+FullTextType_equals(FullTextType *self, Obj *other) {
+ FullTextType *twin = (FullTextType*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, FULLTEXTTYPE)) { return false; }
+ if (!FType_equals((FieldType*)self, other)) { return false; }
+ if (!!self->sortable != !!twin->sortable) { return false; }
+ if (!!self->highlightable != !!twin->highlightable) { return false; }
+ if (!Analyzer_Equals(self->analyzer, (Obj*)twin->analyzer)) {
+ return false;
+ }
+ return true;
+}
+
+Hash*
+FullTextType_dump_for_schema(FullTextType *self) {
+ Hash *dump = Hash_new(0);
+ Hash_Store_Str(dump, "type", 4, (Obj*)CB_newf("fulltext"));
+
+ // Store attributes that override the defaults.
+ if (self->boost != 1.0) {
+ Hash_Store_Str(dump, "boost", 5, (Obj*)CB_newf("%f64", self->boost));
+ }
+ if (!self->indexed) {
+ Hash_Store_Str(dump, "indexed", 7, (Obj*)CB_newf("0"));
+ }
+ if (!self->stored) {
+ Hash_Store_Str(dump, "stored", 6, (Obj*)CB_newf("0"));
+ }
+ if (self->sortable) {
+ Hash_Store_Str(dump, "sortable", 8, (Obj*)CB_newf("1"));
+ }
+ if (self->highlightable) {
+ Hash_Store_Str(dump, "highlightable", 13, (Obj*)CB_newf("1"));
+ }
+
+ return dump;
+}
+
+Hash*
+FullTextType_dump(FullTextType *self) {
+ Hash *dump = FullTextType_Dump_For_Schema(self);
+ Hash_Store_Str(dump, "_class", 6,
+ (Obj*)CB_Clone(FullTextType_Get_Class_Name(self)));
+ Hash_Store_Str(dump, "analyzer", 8,
+ (Obj*)Analyzer_Dump(self->analyzer));
+ DECREF(Hash_Delete_Str(dump, "type", 4));
+
+ return dump;
+}
+
+FullTextType*
+FullTextType_load(FullTextType *self, Obj *dump) {
+ UNUSED_VAR(self);
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *class_name = (CharBuf*)Hash_Fetch_Str(source, "_class", 6);
+ VTable *vtable
+ = (class_name != NULL && Obj_Is_A((Obj*)class_name, CHARBUF))
+ ? VTable_singleton(class_name, NULL)
+ : FULLTEXTTYPE;
+ FullTextType *loaded = (FullTextType*)VTable_Make_Obj(vtable);
+
+ // Extract boost.
+ Obj *boost_dump = Hash_Fetch_Str(source, "boost", 5);
+ float boost = boost_dump ? (float)Obj_To_F64(boost_dump) : 1.0f;
+
+ // Find boolean properties.
+ Obj *indexed_dump = Hash_Fetch_Str(source, "indexed", 7);
+ Obj *stored_dump = Hash_Fetch_Str(source, "stored", 6);
+ Obj *sort_dump = Hash_Fetch_Str(source, "sortable", 8);
+ Obj *hl_dump = Hash_Fetch_Str(source, "highlightable", 13);
+ bool_t indexed = indexed_dump ? (bool_t)Obj_To_I64(indexed_dump) : true;
+ bool_t stored = stored_dump ? (bool_t)Obj_To_I64(stored_dump) : true;
+ bool_t sortable = sort_dump ? (bool_t)Obj_To_I64(sort_dump) : false;
+ bool_t hl = hl_dump ? (bool_t)Obj_To_I64(hl_dump) : false;
+
+ // Extract an Analyzer.
+ Obj *analyzer_dump = Hash_Fetch_Str(source, "analyzer", 8);
+ Analyzer *analyzer = NULL;
+ if (analyzer_dump) {
+ if (Obj_Is_A(analyzer_dump, ANALYZER)) {
+ // Schema munged the dump and installed a shared analyzer.
+ analyzer = (Analyzer*)INCREF(analyzer_dump);
+ }
+ else if (Obj_Is_A((Obj*)analyzer_dump, HASH)) {
+ analyzer = (Analyzer*)Obj_Load(analyzer_dump, analyzer_dump);
+ }
+ }
+ CERTIFY(analyzer, ANALYZER);
+
+ FullTextType_init(loaded, analyzer);
+ DECREF(analyzer);
+ if (boost_dump) { loaded->boost = boost; }
+ if (indexed_dump) { loaded->indexed = indexed; }
+ if (stored_dump) { loaded->stored = stored; }
+ if (sort_dump) { loaded->sortable = sortable; }
+ if (hl_dump) { loaded->highlightable = hl; }
+
+ return loaded;
+}
+
+void
+FullTextType_set_highlightable(FullTextType *self, bool_t highlightable) {
+ self->highlightable = highlightable;
+}
+
+Analyzer*
+FullTextType_get_analyzer(FullTextType *self) {
+ return self->analyzer;
+}
+
+bool_t
+FullTextType_highlightable(FullTextType *self) {
+ return self->highlightable;
+}
+
+Similarity*
+FullTextType_make_similarity(FullTextType *self) {
+ UNUSED_VAR(self);
+ return Sim_new();
+}
+
+
diff --git a/core/Lucy/Plan/FullTextType.cfh b/core/Lucy/Plan/FullTextType.cfh
new file mode 100644
index 0000000..3138373
--- /dev/null
+++ b/core/Lucy/Plan/FullTextType.cfh
@@ -0,0 +1,91 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Full-text search field type.
+ *
+ * Lucy::Plan::FullTextType is an implementation of
+ * L<Lucy::Plan::FieldType> tuned for "full text search".
+ *
+ * Full text fields are associated with an
+ * L<Analyzer|Lucy::Analysis::Analyzer>, which is used to tokenize and
+ * normalize the text so that it can be searched for individual words.
+ *
+ * For an exact-match, single value field type using character data, see
+ * L<StringType|Lucy::Plan::StringType>.
+ */
+class Lucy::Plan::FullTextType
+ inherits Lucy::Plan::TextType : dumpable {
+
+ bool_t highlightable;
+ Analyzer *analyzer;
+
+ /**
+ * @param analyzer An Analyzer.
+ * @param boost floating point per-field boost.
+ * @param indexed boolean indicating whether the field should be indexed.
+ * @param stored boolean indicating whether the field should be stored.
+ * @param sortable boolean indicating whether the field should be sortable.
+ * @param highlightable boolean indicating whether the field should be
+ * highlightable.
+ */
+ public inert FullTextType*
+ init(FullTextType *self, Analyzer *analyzer);
+
+ inert FullTextType*
+ init2(FullTextType *self, Analyzer *analyzer, float boost = 1.0,
+ bool_t indexed = true, bool_t stored = true,
+ bool_t sortable = false, bool_t highlightable = false);
+
+ public inert incremented FullTextType*
+ new(Analyzer *analyzer);
+
+ /** Indicate whether to store data required by
+ * L<Lucy::Highlight::Highlighter> for excerpt selection and search
+ * term highlighting.
+ */
+ public void
+ Set_Highlightable(FullTextType *self, bool_t highlightable);
+
+ /** Accessor for "highlightable" property.
+ */
+ public bool_t
+ Highlightable(FullTextType *self);
+
+ public Analyzer*
+ Get_Analyzer(FullTextType *self);
+
+ public incremented Similarity*
+ Make_Similarity(FullTextType *self);
+
+ incremented Hash*
+ Dump_For_Schema(FullTextType *self);
+
+ public incremented Hash*
+ Dump(FullTextType *self);
+
+ public incremented FullTextType*
+ Load(FullTextType *self, Obj *dump);
+
+ public bool_t
+ Equals(FullTextType *self, Obj *other);
+
+ public void
+ Destroy(FullTextType *self);
+}
+
+
diff --git a/core/Lucy/Plan/NumericType.c b/core/Lucy/Plan/NumericType.c
new file mode 100644
index 0000000..518ca16
--- /dev/null
+++ b/core/Lucy/Plan/NumericType.c
@@ -0,0 +1,290 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NUMERICTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/NumericType.h"
+
+NumericType*
+NumType_init(NumericType *self) {
+ return NumType_init2(self, 1.0, true, true, false);
+}
+
+NumericType*
+NumType_init2(NumericType *self, float boost, bool_t indexed, bool_t stored,
+ bool_t sortable) {
+ FType_init((FieldType*)self);
+ self->boost = boost;
+ self->indexed = indexed;
+ self->stored = stored;
+ self->sortable = sortable;
+ return self;
+}
+
+bool_t
+NumType_binary(NumericType *self) {
+ UNUSED_VAR(self);
+ return true;
+}
+
+Hash*
+NumType_dump_for_schema(NumericType *self) {
+ Hash *dump = Hash_new(0);
+ Hash_Store_Str(dump, "type", 4, (Obj*)NumType_Specifier(self));
+
+ // Store attributes that override the defaults.
+ if (self->boost != 1.0) {
+ Hash_Store_Str(dump, "boost", 5, (Obj*)CB_newf("%f64", self->boost));
+ }
+ if (!self->indexed) {
+ Hash_Store_Str(dump, "indexed", 7, (Obj*)CB_newf("0"));
+ }
+ if (!self->stored) {
+ Hash_Store_Str(dump, "stored", 6, (Obj*)CB_newf("0"));
+ }
+ if (self->sortable) {
+ Hash_Store_Str(dump, "sortable", 8, (Obj*)CB_newf("1"));
+ }
+
+ return dump;
+}
+
+Hash*
+NumType_dump(NumericType *self) {
+ Hash *dump = NumType_Dump_For_Schema(self);
+ Hash_Store_Str(dump, "_class", 6,
+ (Obj*)CB_Clone(NumType_Get_Class_Name(self)));
+ DECREF(Hash_Delete_Str(dump, "type", 4));
+ return dump;
+}
+
+NumericType*
+NumType_load(NumericType *self, Obj *dump) {
+ UNUSED_VAR(self);
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+
+ // Get a VTable
+ CharBuf *class_name = (CharBuf*)Hash_Fetch_Str(source, "_class", 6);
+ CharBuf *type_spec = (CharBuf*)Hash_Fetch_Str(source, "type", 4);
+ VTable *vtable = NULL;
+ if (class_name != NULL && Obj_Is_A((Obj*)class_name, CHARBUF)) {
+ vtable = VTable_singleton(class_name, NULL);
+ }
+ else if (type_spec != NULL && Obj_Is_A((Obj*)type_spec, CHARBUF)) {
+ if (CB_Equals_Str(type_spec, "i32_t", 5)) {
+ vtable = INT32TYPE;
+ }
+ else if (CB_Equals_Str(type_spec, "i64_t", 5)) {
+ vtable = INT64TYPE;
+ }
+ else if (CB_Equals_Str(type_spec, "f32_t", 5)) {
+ vtable = FLOAT32TYPE;
+ }
+ else if (CB_Equals_Str(type_spec, "f64_t", 5)) {
+ vtable = FLOAT64TYPE;
+ }
+ else {
+ THROW(ERR, "Unrecognized type string: '%o'", type_spec);
+ }
+ }
+ CERTIFY(vtable, VTABLE);
+ NumericType *loaded = (NumericType*)VTable_Make_Obj(vtable);
+
+ // Extract boost.
+ Obj *boost_dump = Hash_Fetch_Str(source, "boost", 5);
+ float boost = boost_dump ? (float)Obj_To_F64(boost_dump) : 1.0f;
+
+ // Find boolean properties.
+ Obj *indexed_dump = Hash_Fetch_Str(source, "indexed", 7);
+ Obj *stored_dump = Hash_Fetch_Str(source, "stored", 6);
+ Obj *sort_dump = Hash_Fetch_Str(source, "sortable", 8);
+ bool_t indexed = indexed_dump ? (bool_t)Obj_To_I64(indexed_dump) : true;
+ bool_t stored = stored_dump ? (bool_t)Obj_To_I64(stored_dump) : true;
+ bool_t sortable = sort_dump ? (bool_t)Obj_To_I64(sort_dump) : false;
+
+ return NumType_init2(loaded, boost, indexed, stored, sortable);
+}
+
+/****************************************************************************/
+
+Float64Type*
+Float64Type_new() {
+ Float64Type *self = (Float64Type*)VTable_Make_Obj(FLOAT64TYPE);
+ return Float64Type_init(self);
+}
+
+Float64Type*
+Float64Type_init(Float64Type *self) {
+ return Float64Type_init2(self, 1.0, true, true, false);
+}
+
+Float64Type*
+Float64Type_init2(Float64Type *self, float boost, bool_t indexed,
+ bool_t stored, bool_t sortable) {
+ return (Float64Type*)NumType_init2((NumericType*)self, boost, indexed,
+ stored, sortable);
+}
+
+CharBuf*
+Float64Type_specifier(Float64Type *self) {
+ UNUSED_VAR(self);
+ return CB_newf("f64_t");
+}
+
+int8_t
+Float64Type_primitive_id(Float64Type *self) {
+ UNUSED_VAR(self);
+ return FType_FLOAT64;
+}
+
+bool_t
+Float64Type_equals(Float64Type *self, Obj *other) {
+ if (self == (Float64Type*)other) { return true; }
+ if (!other) { return false; }
+ if (!Obj_Is_A(other, FLOAT64TYPE)) { return false; }
+ Float64Type_equals_t super_equals = (Float64Type_equals_t)SUPER_METHOD(
+ FLOAT64TYPE, Float64Type, Equals);
+ return super_equals(self, other);
+}
+
+/****************************************************************************/
+
+Float32Type*
+Float32Type_new() {
+ Float32Type *self = (Float32Type*)VTable_Make_Obj(FLOAT32TYPE);
+ return Float32Type_init(self);
+}
+
+Float32Type*
+Float32Type_init(Float32Type *self) {
+ return Float32Type_init2(self, 1.0, true, true, false);
+}
+
+Float32Type*
+Float32Type_init2(Float32Type *self, float boost, bool_t indexed,
+ bool_t stored, bool_t sortable) {
+ return (Float32Type*)NumType_init2((NumericType*)self, boost, indexed,
+ stored, sortable);
+}
+
+CharBuf*
+Float32Type_specifier(Float32Type *self) {
+ UNUSED_VAR(self);
+ return CB_newf("f32_t");
+}
+
+int8_t
+Float32Type_primitive_id(Float32Type *self) {
+ UNUSED_VAR(self);
+ return FType_FLOAT32;
+}
+
+bool_t
+Float32Type_equals(Float32Type *self, Obj *other) {
+ if (self == (Float32Type*)other) { return true; }
+ if (!other) { return false; }
+ if (!Obj_Is_A(other, FLOAT32TYPE)) { return false; }
+ Float32Type_equals_t super_equals = (Float32Type_equals_t)SUPER_METHOD(
+ FLOAT32TYPE, Float32Type, Equals);
+ return super_equals(self, other);
+}
+
+/****************************************************************************/
+
+Int32Type*
+Int32Type_new() {
+ Int32Type *self = (Int32Type*)VTable_Make_Obj(INT32TYPE);
+ return Int32Type_init(self);
+}
+
+Int32Type*
+Int32Type_init(Int32Type *self) {
+ return Int32Type_init2(self, 1.0, true, true, false);
+}
+
+Int32Type*
+Int32Type_init2(Int32Type *self, float boost, bool_t indexed,
+ bool_t stored, bool_t sortable) {
+ return (Int32Type*)NumType_init2((NumericType*)self, boost, indexed,
+ stored, sortable);
+}
+
+CharBuf*
+Int32Type_specifier(Int32Type *self) {
+ UNUSED_VAR(self);
+ return CB_newf("i32_t");
+}
+
+int8_t
+Int32Type_primitive_id(Int32Type *self) {
+ UNUSED_VAR(self);
+ return FType_INT32;
+}
+
+bool_t
+Int32Type_equals(Int32Type *self, Obj *other) {
+ if (self == (Int32Type*)other) { return true; }
+ if (!other) { return false; }
+ if (!Obj_Is_A(other, INT32TYPE)) { return false; }
+ Int32Type_equals_t super_equals = (Int32Type_equals_t)SUPER_METHOD(
+ INT32TYPE, Int32Type, Equals);
+ return super_equals(self, other);
+}
+
+/****************************************************************************/
+
+Int64Type*
+Int64Type_new() {
+ Int64Type *self = (Int64Type*)VTable_Make_Obj(INT64TYPE);
+ return Int64Type_init(self);
+}
+
+Int64Type*
+Int64Type_init(Int64Type *self) {
+ return Int64Type_init2(self, 1.0, true, true, false);
+}
+
+Int64Type*
+Int64Type_init2(Int64Type *self, float boost, bool_t indexed,
+ bool_t stored, bool_t sortable) {
+ return (Int64Type*)NumType_init2((NumericType*)self, boost, indexed,
+ stored, sortable);
+}
+
+CharBuf*
+Int64Type_specifier(Int64Type *self) {
+ UNUSED_VAR(self);
+ return CB_newf("i64_t");
+}
+
+int8_t
+Int64Type_primitive_id(Int64Type *self) {
+ UNUSED_VAR(self);
+ return FType_INT64;
+}
+
+bool_t
+Int64Type_equals(Int64Type *self, Obj *other) {
+ if (self == (Int64Type*)other) { return true; }
+ if (!other) { return false; }
+ if (!Obj_Is_A(other, INT64TYPE)) { return false; }
+ Int64Type_equals_t super_equals = (Int64Type_equals_t)SUPER_METHOD(
+ INT64TYPE, Int64Type, Equals);
+ return super_equals(self, other);
+}
+
+
diff --git a/core/Lucy/Plan/NumericType.cfh b/core/Lucy/Plan/NumericType.cfh
new file mode 100644
index 0000000..316308f
--- /dev/null
+++ b/core/Lucy/Plan/NumericType.cfh
@@ -0,0 +1,148 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Plan::NumericType cnick NumType
+ inherits Lucy::Plan::FieldType : dumpable {
+
+ public inert NumericType*
+ init(NumericType *self);
+
+ inert NumericType*
+ init2(NumericType *self, float boost = 1.0, bool_t indexed = true,
+ bool_t stored = true, bool_t sortable = false);
+
+ /** Returns true.
+ */
+ public bool_t
+ Binary(NumericType *self);
+
+ /** Return the primitive type specifier for the object type, e.g.
+ * f64_t for Float64, uint32_t for UInteger32, etc.
+ */
+ abstract incremented CharBuf*
+ Specifier(NumericType *self);
+
+ incremented Hash*
+ Dump_For_Schema(NumericType *self);
+
+ public incremented Hash*
+ Dump(NumericType *self);
+
+ public incremented NumericType*
+ Load(NumericType *self, Obj *dump);
+}
+
+abstract class Lucy::Plan::FloatType
+ inherits Lucy::Plan::NumericType : dumpable { }
+
+class Lucy::Plan::Float64Type
+ inherits Lucy::Plan::FloatType : dumpable {
+
+ public inert Float64Type*
+ new();
+
+ public inert Float64Type*
+ init(Float64Type *self);
+
+ inert Float64Type*
+ init2(Float64Type *self, float boost = 1.0, bool_t indexed = true,
+ bool_t stored = true, bool_t sortable = true);
+
+ int8_t
+ Primitive_ID(Float64Type *self);
+
+ incremented CharBuf*
+ Specifier(Float64Type *self);
+
+ public bool_t
+ Equals(Float64Type *self, Obj *other);
+}
+
+class Lucy::Plan::Float32Type
+ inherits Lucy::Plan::FloatType : dumpable {
+
+ public inert Float32Type*
+ new();
+
+ public inert Float32Type*
+ init(Float32Type *self);
+
+ inert Float32Type*
+ init2(Float32Type *self, float boost = 1.0, bool_t indexed = true,
+ bool_t stored = true, bool_t sortable = false);
+
+ int8_t
+ Primitive_ID(Float32Type *self);
+
+ incremented CharBuf*
+ Specifier(Float32Type *self);
+
+ public bool_t
+ Equals(Float32Type *self, Obj *other);
+}
+
+abstract class Lucy::Plan::IntType
+ inherits Lucy::Plan::NumericType : dumpable { }
+
+class Lucy::Plan::Int32Type
+ inherits Lucy::Plan::IntType : dumpable {
+
+ public inert Int32Type*
+ new();
+
+ public inert Int32Type*
+ init(Int32Type *self);
+
+ inert Int32Type*
+ init2(Int32Type *self, float boost = 1.0, bool_t indexed = true,
+ bool_t stored = true, bool_t sortable = false);
+
+ int8_t
+ Primitive_ID(Int32Type *self);
+
+ incremented CharBuf*
+ Specifier(Int32Type *self);
+
+ public bool_t
+ Equals(Int32Type *self, Obj *other);
+}
+
+class Lucy::Plan::Int64Type
+ inherits Lucy::Plan::IntType : dumpable {
+
+ public inert Int64Type*
+ new();
+
+ public inert Int64Type*
+ init(Int64Type *self);
+
+ inert Int64Type*
+ init2(Int64Type *self, float boost = 1.0, bool_t indexed = true,
+ bool_t stored = true, bool_t sortable = false);
+
+ int8_t
+ Primitive_ID(Int64Type *self);
+
+ incremented CharBuf*
+ Specifier(Int64Type *self);
+
+ public bool_t
+ Equals(Int64Type *self, Obj *other);
+}
+
+
diff --git a/core/Lucy/Plan/Schema.c b/core/Lucy/Plan/Schema.c
new file mode 100644
index 0000000..7c189c9
--- /dev/null
+++ b/core/Lucy/Plan/Schema.c
@@ -0,0 +1,413 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SCHEMA
+#include <string.h>
+#include <ctype.h>
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/BlobType.h"
+#include "Lucy/Plan/NumericType.h"
+#include "Lucy/Plan/StringType.h"
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Plan/Architecture.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Util/Json.h"
+
+// Scan the array to see if an object testing as Equal is present. If not,
+// push the elem onto the end of the array.
+static void
+S_add_unique(VArray *array, Obj *elem);
+
+static void
+S_add_text_field(Schema *self, const CharBuf *field, FieldType *type);
+static void
+S_add_string_field(Schema *self, const CharBuf *field, FieldType *type);
+static void
+S_add_blob_field(Schema *self, const CharBuf *field, FieldType *type);
+static void
+S_add_numeric_field(Schema *self, const CharBuf *field, FieldType *type);
+
+Schema*
+Schema_new() {
+ Schema *self = (Schema*)VTable_Make_Obj(SCHEMA);
+ return Schema_init(self);
+}
+
+Schema*
+Schema_init(Schema *self) {
+ // Init.
+ self->analyzers = Hash_new(0);
+ self->types = Hash_new(0);
+ self->sims = Hash_new(0);
+ self->uniq_analyzers = VA_new(2);
+ VA_Resize(self->uniq_analyzers, 1);
+
+ // Assign.
+ self->arch = Schema_Architecture(self);
+ self->sim = Arch_Make_Similarity(self->arch);
+
+ return self;
+}
+
+void
+Schema_destroy(Schema *self) {
+ DECREF(self->arch);
+ DECREF(self->analyzers);
+ DECREF(self->uniq_analyzers);
+ DECREF(self->types);
+ DECREF(self->sims);
+ DECREF(self->sim);
+ SUPER_DESTROY(self, SCHEMA);
+}
+
+static void
+S_add_unique(VArray *array, Obj *elem) {
+ uint32_t i, max;
+ if (!elem) { return; }
+ for (i = 0, max = VA_Get_Size(array); i < max; i++) {
+ Obj *candidate = VA_Fetch(array, i);
+ if (!candidate) { continue; }
+ if (elem == candidate) { return; }
+ if (Obj_Get_VTable(elem) == Obj_Get_VTable(candidate)) {
+ if (Obj_Equals(elem, candidate)) { return; }
+ }
+ }
+ VA_Push(array, INCREF(elem));
+}
+
+bool_t
+Schema_equals(Schema *self, Obj *other) {
+ Schema *twin = (Schema*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, SCHEMA)) { return false; }
+ if (!Arch_Equals(self->arch, (Obj*)twin->arch)) { return false; }
+ if (!Sim_Equals(self->sim, (Obj*)twin->sim)) { return false; }
+ if (!Hash_Equals(self->types, (Obj*)twin->types)) { return false; }
+ return true;
+}
+
+Architecture*
+Schema_architecture(Schema *self) {
+ UNUSED_VAR(self);
+ return Arch_new();
+}
+
+void
+Schema_spec_field(Schema *self, const CharBuf *field, FieldType *type) {
+ FieldType *existing = Schema_Fetch_Type(self, field);
+
+ // If the field already has an association, verify pairing and return.
+ if (existing) {
+ if (FType_Equals(type, (Obj*)existing)) { return; }
+ else { THROW(ERR, "'%o' assigned conflicting FieldType", field); }
+ }
+
+ if (FType_Is_A(type, FULLTEXTTYPE)) {
+ S_add_text_field(self, field, type);
+ }
+ else if (FType_Is_A(type, STRINGTYPE)) {
+ S_add_string_field(self, field, type);
+ }
+ else if (FType_Is_A(type, BLOBTYPE)) {
+ S_add_blob_field(self, field, type);
+ }
+ else if (FType_Is_A(type, NUMERICTYPE)) {
+ S_add_numeric_field(self, field, type);
+ }
+ else {
+ THROW(ERR, "Unrecognized field type: '%o'", type);
+ }
+}
+
+static void
+S_add_text_field(Schema *self, const CharBuf *field, FieldType *type) {
+ FullTextType *fttype = (FullTextType*)CERTIFY(type, FULLTEXTTYPE);
+ Similarity *sim = FullTextType_Make_Similarity(fttype);
+ Analyzer *analyzer = FullTextType_Get_Analyzer(fttype);
+
+ // Cache helpers.
+ Hash_Store(self->sims, (Obj*)field, (Obj*)sim);
+ Hash_Store(self->analyzers, (Obj*)field, INCREF(analyzer));
+ S_add_unique(self->uniq_analyzers, (Obj*)analyzer);
+
+ // Store FieldType.
+ Hash_Store(self->types, (Obj*)field, INCREF(type));
+}
+
+static void
+S_add_string_field(Schema *self, const CharBuf *field, FieldType *type) {
+ StringType *string_type = (StringType*)CERTIFY(type, STRINGTYPE);
+ Similarity *sim = StringType_Make_Similarity(string_type);
+
+ // Cache helpers.
+ Hash_Store(self->sims, (Obj*)field, (Obj*)sim);
+
+ // Store FieldType.
+ Hash_Store(self->types, (Obj*)field, INCREF(type));
+}
+
+static void
+S_add_blob_field(Schema *self, const CharBuf *field, FieldType *type) {
+ BlobType *blob_type = (BlobType*)CERTIFY(type, BLOBTYPE);
+ Hash_Store(self->types, (Obj*)field, INCREF(blob_type));
+}
+
+static void
+S_add_numeric_field(Schema *self, const CharBuf *field, FieldType *type) {
+ NumericType *num_type = (NumericType*)CERTIFY(type, NUMERICTYPE);
+ Hash_Store(self->types, (Obj*)field, INCREF(num_type));
+}
+
+FieldType*
+Schema_fetch_type(Schema *self, const CharBuf *field) {
+ return (FieldType*)Hash_Fetch(self->types, (Obj*)field);
+}
+
+Analyzer*
+Schema_fetch_analyzer(Schema *self, const CharBuf *field) {
+ return field
+ ? (Analyzer*)Hash_Fetch(self->analyzers, (Obj*)field)
+ : NULL;
+}
+
+Similarity*
+Schema_fetch_sim(Schema *self, const CharBuf *field) {
+ Similarity *sim = NULL;
+ if (field != NULL) {
+ sim = (Similarity*)Hash_Fetch(self->sims, (Obj*)field);
+ }
+ return sim;
+}
+
+uint32_t
+Schema_num_fields(Schema *self) {
+ return Hash_Get_Size(self->types);
+}
+
+Architecture*
+Schema_get_architecture(Schema *self) {
+ return self->arch;
+}
+
+Similarity*
+Schema_get_similarity(Schema *self) {
+ return self->sim;
+}
+
+VArray*
+Schema_all_fields(Schema *self) {
+ return Hash_Keys(self->types);
+}
+
+uint32_t
+S_find_in_array(VArray *array, Obj *obj) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(array); i < max; i++) {
+ Obj *candidate = VA_Fetch(array, i);
+ if (obj == NULL && candidate == NULL) {
+ return i;
+ }
+ else if (obj != NULL && candidate != NULL) {
+ if (Obj_Get_VTable(obj) == Obj_Get_VTable(candidate)) {
+ if (Obj_Equals(obj, candidate)) {
+ return i;
+ }
+ }
+ }
+ }
+ THROW(ERR, "Couldn't find match for %o", obj);
+ UNREACHABLE_RETURN(uint32_t);
+}
+
+Hash*
+Schema_dump(Schema *self) {
+ Hash *dump = Hash_new(0);
+ Hash *type_dumps = Hash_new(Hash_Get_Size(self->types));
+ CharBuf *field;
+ FieldType *type;
+
+ // Record class name, store dumps of unique Analyzers.
+ Hash_Store_Str(dump, "_class", 6,
+ (Obj*)CB_Clone(Schema_Get_Class_Name(self)));
+ Hash_Store_Str(dump, "analyzers", 9, (Obj*)VA_Dump(self->uniq_analyzers));
+
+ // Dump FieldTypes.
+ Hash_Store_Str(dump, "fields", 6, (Obj*)type_dumps);
+ Hash_Iterate(self->types);
+ while (Hash_Next(self->types, (Obj**)&field, (Obj**)&type)) {
+ VTable *type_vtable = FType_Get_VTable(type);
+
+ // Dump known types to simplified format.
+ if (type_vtable == FULLTEXTTYPE) {
+ FullTextType *fttype = (FullTextType*)type;
+ Hash *type_dump = FullTextType_Dump_For_Schema(fttype);
+ Analyzer *analyzer = FullTextType_Get_Analyzer(fttype);
+ uint32_t tick
+ = S_find_in_array(self->uniq_analyzers, (Obj*)analyzer);
+
+ // Store the tick which references a unique analyzer.
+ Hash_Store_Str(type_dump, "analyzer", 8,
+ (Obj*)CB_newf("%u32", tick));
+
+ Hash_Store(type_dumps, (Obj*)field, (Obj*)type_dump);
+ }
+ else if (type_vtable == STRINGTYPE || type_vtable == BLOBTYPE) {
+ Hash *type_dump = FType_Dump_For_Schema(type);
+ Hash_Store(type_dumps, (Obj*)field, (Obj*)type_dump);
+ }
+ // Unknown FieldType type, so punt.
+ else {
+ Hash_Store(type_dumps, (Obj*)field, FType_Dump(type));
+ }
+ }
+
+ return dump;
+}
+
+Schema*
+Schema_load(Schema *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *class_name
+ = (CharBuf*)CERTIFY(Hash_Fetch_Str(source, "_class", 6), CHARBUF);
+ VTable *vtable = VTable_singleton(class_name, NULL);
+ Schema *loaded = (Schema*)VTable_Make_Obj(vtable);
+ Hash *type_dumps
+ = (Hash*)CERTIFY(Hash_Fetch_Str(source, "fields", 6), HASH);
+ VArray *analyzer_dumps
+ = (VArray*)CERTIFY(Hash_Fetch_Str(source, "analyzers", 9), VARRAY);
+ VArray *analyzers
+ = (VArray*)VA_Load(analyzer_dumps, (Obj*)analyzer_dumps);
+ CharBuf *field;
+ Hash *type_dump;
+ UNUSED_VAR(self);
+
+ // Start with a blank Schema.
+ Schema_init(loaded);
+ VA_Grow(loaded->uniq_analyzers, VA_Get_Size(analyzers));
+
+ Hash_Iterate(type_dumps);
+ while (Hash_Next(type_dumps, (Obj**)&field, (Obj**)&type_dump)) {
+ CharBuf *type_str;
+ CERTIFY(type_dump, HASH);
+ type_str = (CharBuf*)Hash_Fetch_Str(type_dump, "type", 4);
+ if (type_str) {
+ if (CB_Equals_Str(type_str, "fulltext", 8)) {
+ // Replace the "analyzer" tick with the real thing.
+ Obj *tick
+ = CERTIFY(Hash_Fetch_Str(type_dump, "analyzer", 8), OBJ);
+ Analyzer *analyzer
+ = (Analyzer*)VA_Fetch(analyzers,
+ (uint32_t)Obj_To_I64(tick));
+ if (!analyzer) {
+ THROW(ERR, "Can't find analyzer for '%o'", field);
+ }
+ Hash_Store_Str(type_dump, "analyzer", 8, INCREF(analyzer));
+ FullTextType *type
+ = (FullTextType*)VTable_Load_Obj(FULLTEXTTYPE,
+ (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else if (CB_Equals_Str(type_str, "string", 6)) {
+ StringType *type
+ = (StringType*)VTable_Load_Obj(STRINGTYPE,
+ (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else if (CB_Equals_Str(type_str, "blob", 4)) {
+ BlobType *type
+ = (BlobType*)VTable_Load_Obj(BLOBTYPE, (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else if (CB_Equals_Str(type_str, "i32_t", 5)) {
+ Int32Type *type
+ = (Int32Type*)VTable_Load_Obj(INT32TYPE, (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else if (CB_Equals_Str(type_str, "i64_t", 5)) {
+ Int64Type *type
+ = (Int64Type*)VTable_Load_Obj(INT64TYPE, (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else if (CB_Equals_Str(type_str, "f32_t", 5)) {
+ Float32Type *type
+ = (Float32Type*)VTable_Load_Obj(FLOAT32TYPE,
+ (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else if (CB_Equals_Str(type_str, "f64_t", 5)) {
+ Float64Type *type
+ = (Float64Type*)VTable_Load_Obj(FLOAT64TYPE,
+ (Obj*)type_dump);
+ Schema_Spec_Field(loaded, field, (FieldType*)type);
+ DECREF(type);
+ }
+ else {
+ THROW(ERR, "Unknown type '%o' for field '%o'", type_str, field);
+ }
+ }
+ else {
+ FieldType *type = (FieldType*)CERTIFY(
+ Hash_Load(type_dump, (Obj*)type_dump),
+ FIELDTYPE);
+ Schema_Spec_Field(loaded, field, type);
+ DECREF(type);
+ }
+ }
+
+ DECREF(analyzers);
+
+ return loaded;
+}
+
+void
+Schema_eat(Schema *self, Schema *other) {
+ if (!Schema_Is_A(self, Schema_Get_VTable(other))) {
+ THROW(ERR, "%o not a descendent of %o",
+ Schema_Get_Class_Name(self), Schema_Get_Class_Name(other));
+ }
+
+ CharBuf *field;
+ FieldType *type;
+ Hash_Iterate(other->types);
+ while (Hash_Next(other->types, (Obj**)&field, (Obj**)&type)) {
+ Schema_Spec_Field(self, field, type);
+ }
+}
+
+void
+Schema_write(Schema *self, Folder *folder, const CharBuf *filename) {
+ Hash *dump = Schema_Dump(self);
+ ZombieCharBuf *schema_temp = ZCB_WRAP_STR("schema.temp", 11);
+ bool_t success;
+ Folder_Delete(folder, (CharBuf*)schema_temp); // Just in case.
+ Json_spew_json((Obj*)dump, folder, (CharBuf*)schema_temp);
+ success = Folder_Rename(folder, (CharBuf*)schema_temp, filename);
+ DECREF(dump);
+ if (!success) { RETHROW(INCREF(Err_get_error())); }
+}
+
+
diff --git a/core/Lucy/Plan/Schema.cfh b/core/Lucy/Plan/Schema.cfh
new file mode 100644
index 0000000..489da49
--- /dev/null
+++ b/core/Lucy/Plan/Schema.cfh
@@ -0,0 +1,117 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** User-created specification for an inverted index.
+ *
+ * A Schema is a specification which indicates how other entities should
+ * interpret the raw data in an inverted index and interact with it.
+ *
+ * Once an actual index has been created using a particular Schema, existing
+ * field definitions may not be changed. However, it is possible to add new
+ * fields during subsequent indexing sessions.
+ */
+class Lucy::Plan::Schema inherits Lucy::Object::Obj {
+
+ Architecture *arch;
+ Similarity *sim;
+ Hash *types;
+ Hash *sims;
+ Hash *analyzers;
+ VArray *uniq_analyzers;
+
+ public inert incremented Schema*
+ new();
+
+ /** Constructor. Takes no arguments.
+ */
+ public inert Schema*
+ init(Schema *self);
+
+ /** Factory method which creates an Architecture object for this index.
+ */
+ public incremented Architecture*
+ Architecture(Schema *self);
+
+ /** Define the behavior of a field by associating it with a FieldType.
+ *
+ * If this method has already been called for the supplied
+ * <code>field</code>, it will merely test to verify that the supplied
+ * FieldType Equals() the existing one.
+ *
+ * @param name The name of the field.
+ * @param type A FieldType.
+ */
+ public void
+ Spec_Field(Schema *self, const CharBuf *name, FieldType *type);
+
+ /** Return the FieldType for the specified field. If the field can't be
+ * found, return NULL.
+ */
+ public nullable FieldType*
+ Fetch_Type(Schema *self, const CharBuf *field);
+
+ /** Return the Analyzer for the specified field.
+ */
+ nullable Analyzer*
+ Fetch_Analyzer(Schema *self, const CharBuf *field = NULL);
+
+ /** Return the Similarity for the specified field, or NULL if either the
+ * field can't be found or it isn't associated with a Similarity.
+ */
+ public nullable Similarity*
+ Fetch_Sim(Schema *self, const CharBuf *field = NULL);
+
+ /** Return the number of fields currently defined.
+ */
+ public uint32_t
+ Num_Fields(Schema *self);
+
+ /** Return all the Schema's field names as an array.
+ */
+ public incremented VArray*
+ All_Fields(Schema *self);
+
+ /** Return the Schema instance's internal Architecture object.
+ */
+ public Architecture*
+ Get_Architecture(Schema *self);
+
+ /** Return the Schema instance's internal Similarity object.
+ */
+ public Similarity*
+ Get_Similarity(Schema *self);
+
+ public incremented Hash*
+ Dump(Schema *self);
+
+ public incremented Schema*
+ Load(Schema *self, Obj *dump);
+
+ /** Absorb the field definitions of another Schema, verify compatibility.
+ */
+ void
+ Eat(Schema *self, Schema *other);
+
+ void
+ Write(Schema *self, Folder *folder, const CharBuf *filename = NULL);
+
+ public void
+ Destroy(Schema *self);
+}
+
+
diff --git a/core/Lucy/Plan/StringType.c b/core/Lucy/Plan/StringType.c
new file mode 100644
index 0000000..3c5e9cd
--- /dev/null
+++ b/core/Lucy/Plan/StringType.c
@@ -0,0 +1,136 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_STRINGTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/StringType.h"
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/Similarity.h"
+
+StringType*
+StringType_new() {
+ StringType *self = (StringType*)VTable_Make_Obj(STRINGTYPE);
+ return StringType_init(self);
+}
+
+StringType*
+StringType_init(StringType *self) {
+ return StringType_init2(self, 1.0, true, true, false);
+}
+
+StringType*
+StringType_init2(StringType *self, float boost, bool_t indexed,
+ bool_t stored, bool_t sortable) {
+ FType_init((FieldType*)self);
+ self->boost = boost;
+ self->indexed = indexed;
+ self->stored = stored;
+ self->sortable = sortable;
+ return self;
+}
+
+bool_t
+StringType_equals(StringType *self, Obj *other) {
+ StringType *twin = (StringType*)other;
+ if (twin == self) { return true; }
+ if (!FType_equals((FieldType*)self, other)) { return false; }
+ return true;
+}
+
+Hash*
+StringType_dump_for_schema(StringType *self) {
+ Hash *dump = Hash_new(0);
+ Hash_Store_Str(dump, "type", 4, (Obj*)CB_newf("string"));
+
+ // Store attributes that override the defaults.
+ if (self->boost != 1.0) {
+ Hash_Store_Str(dump, "boost", 5, (Obj*)CB_newf("%f64", self->boost));
+ }
+ if (!self->indexed) {
+ Hash_Store_Str(dump, "indexed", 7, (Obj*)CB_newf("0"));
+ }
+ if (!self->stored) {
+ Hash_Store_Str(dump, "stored", 6, (Obj*)CB_newf("0"));
+ }
+ if (self->sortable) {
+ Hash_Store_Str(dump, "sortable", 8, (Obj*)CB_newf("1"));
+ }
+
+ return dump;
+}
+
+Hash*
+StringType_dump(StringType *self) {
+ Hash *dump = StringType_Dump_For_Schema(self);
+ Hash_Store_Str(dump, "_class", 6,
+ (Obj*)CB_Clone(StringType_Get_Class_Name(self)));
+ DECREF(Hash_Delete_Str(dump, "type", 4));
+ return dump;
+}
+
+StringType*
+StringType_load(StringType *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ CharBuf *class_name = (CharBuf*)Hash_Fetch_Str(source, "_class", 6);
+ VTable *vtable
+ = (class_name != NULL && Obj_Is_A((Obj*)class_name, CHARBUF))
+ ? VTable_singleton(class_name, NULL)
+ : STRINGTYPE;
+ StringType *loaded = (StringType*)VTable_Make_Obj(vtable);
+ Obj *boost_dump = Hash_Fetch_Str(source, "boost", 5);
+ Obj *indexed_dump = Hash_Fetch_Str(source, "indexed", 7);
+ Obj *stored_dump = Hash_Fetch_Str(source, "stored", 6);
+ Obj *sortable_dump = Hash_Fetch_Str(source, "sortable", 8);
+ UNUSED_VAR(self);
+
+ StringType_init(loaded);
+ if (boost_dump) {
+ loaded->boost = (float)Obj_To_F64(boost_dump);
+ }
+ if (indexed_dump) {
+ loaded->indexed = (bool_t)Obj_To_I64(indexed_dump);
+ }
+ if (stored_dump) {
+ loaded->stored = (bool_t)Obj_To_I64(stored_dump);
+ }
+ if (sortable_dump) {
+ loaded->sortable = (bool_t)Obj_To_I64(sortable_dump);
+ }
+
+ return loaded;
+}
+
+Similarity*
+StringType_make_similarity(StringType *self) {
+ UNUSED_VAR(self);
+ return Sim_new();
+}
+
+Posting*
+StringType_make_posting(StringType *self, Similarity *similarity) {
+ if (similarity) {
+ return (Posting*)ScorePost_new(similarity);
+ }
+ else {
+ Similarity *sim = StringType_Make_Similarity(self);
+ Posting *posting = (Posting*)ScorePost_new(sim);
+ DECREF(sim);
+ return posting;
+ }
+}
+
+
diff --git a/core/Lucy/Plan/StringType.cfh b/core/Lucy/Plan/StringType.cfh
new file mode 100644
index 0000000..b85ee01
--- /dev/null
+++ b/core/Lucy/Plan/StringType.cfh
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Non-tokenized text type.
+ *
+ * Lucy::Plan::StringType is used for "exact-match" strings.
+ */
+class Lucy::Plan::StringType
+ inherits Lucy::Plan::TextType : dumpable {
+
+ /**
+ * @param boost floating point per-field boost.
+ * @param indexed boolean indicating whether the field should be indexed.
+ * @param stored boolean indicating whether the field should be stored.
+ * @param sortable boolean indicating whether the field should be
+ * sortable.
+ */
+ public inert StringType*
+ init(StringType *self);
+
+ inert StringType*
+ init2(StringType *self, float boost = 1.0, bool_t indexed = true,
+ bool_t stored = true, bool_t sortable = false);
+
+ public inert incremented StringType*
+ new();
+
+ public incremented Similarity*
+ Make_Similarity(StringType *self);
+
+ incremented Hash*
+ Dump_For_Schema(StringType *self);
+
+ public incremented Hash*
+ Dump(StringType *self);
+
+ public incremented StringType*
+ Load(StringType *self, Obj *dump);
+
+ public bool_t
+ Equals(StringType *self, Obj *other);
+}
+
+
diff --git a/core/Lucy/Plan/TextType.c b/core/Lucy/Plan/TextType.c
new file mode 100644
index 0000000..0a84e9b
--- /dev/null
+++ b/core/Lucy/Plan/TextType.c
@@ -0,0 +1,157 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TEXTTYPE
+#define C_LUCY_TEXTTERMSTEPPER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Plan/TextType.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/StringHelper.h"
+
+CharBuf*
+TextType_make_blank(TextType *self) {
+ UNUSED_VAR(self);
+ return CB_new(0);
+}
+
+TermStepper*
+TextType_make_term_stepper(TextType *self) {
+ UNUSED_VAR(self);
+ return (TermStepper*)TextTermStepper_new();
+}
+
+int8_t
+TextType_primitive_id(TextType *self) {
+ UNUSED_VAR(self);
+ return FType_TEXT;
+}
+
+/***************************************************************************/
+
+TextTermStepper*
+TextTermStepper_new() {
+ TextTermStepper *self
+ = (TextTermStepper*)VTable_Make_Obj(TEXTTERMSTEPPER);
+ return TextTermStepper_init(self);
+}
+
+TextTermStepper*
+TextTermStepper_init(TextTermStepper *self) {
+ TermStepper_init((TermStepper*)self);
+ self->value = (Obj*)CB_new(0);
+ return self;
+}
+
+void
+TextTermStepper_set_value(TextTermStepper *self, Obj *value) {
+ CERTIFY(value, CHARBUF);
+ DECREF(self->value);
+ self->value = INCREF(value);
+}
+
+void
+TextTermStepper_reset(TextTermStepper *self) {
+ CB_Set_Size((CharBuf*)self->value, 0);
+}
+
+void
+TextTermStepper_write_key_frame(TextTermStepper *self, OutStream *outstream,
+ Obj *value) {
+ Obj_Serialize(value, outstream);
+ Obj_Mimic(self->value, value);
+}
+
+void
+TextTermStepper_write_delta(TextTermStepper *self, OutStream *outstream,
+ Obj *value) {
+ CharBuf *new_value = (CharBuf*)CERTIFY(value, CHARBUF);
+ CharBuf *last_value = (CharBuf*)self->value;
+ char *new_text = (char*)CB_Get_Ptr8(new_value);
+ size_t new_size = CB_Get_Size(new_value);
+ char *last_text = (char*)CB_Get_Ptr8(last_value);
+ size_t last_size = CB_Get_Size(last_value);
+
+ // Count how many bytes the strings share at the top.
+ const int32_t overlap = StrHelp_overlap(last_text, new_text,
+ last_size, new_size);
+ const char *const diff_start_str = new_text + overlap;
+ const size_t diff_len = new_size - overlap;
+
+ // Write number of common bytes and common bytes.
+ OutStream_Write_C32(outstream, overlap);
+ OutStream_Write_String(outstream, diff_start_str, diff_len);
+
+ // Update value.
+ CB_Mimic((CharBuf*)self->value, value);
+}
+
+void
+TextTermStepper_read_key_frame(TextTermStepper *self, InStream *instream) {
+ const uint32_t text_len = InStream_Read_C32(instream);
+ CharBuf *value;
+ char *ptr;
+
+ // Allocate space.
+ if (self->value == NULL) {
+ self->value = (Obj*)CB_new(text_len);
+ }
+ value = (CharBuf*)self->value;
+ ptr = CB_Grow(value, text_len);
+
+ // Set the value text.
+ InStream_Read_Bytes(instream, ptr, text_len);
+ CB_Set_Size(value, text_len);
+ if (!StrHelp_utf8_valid(ptr, text_len)) {
+ THROW(ERR, "Invalid UTF-8 sequence in '%o' at byte %i64",
+ InStream_Get_Filename(instream),
+ InStream_Tell(instream) - text_len);
+ }
+
+ // Null-terminate.
+ ptr[text_len] = '\0';
+}
+
+void
+TextTermStepper_read_delta(TextTermStepper *self, InStream *instream) {
+ const uint32_t text_overlap = InStream_Read_C32(instream);
+ const uint32_t finish_chars_len = InStream_Read_C32(instream);
+ const uint32_t total_text_len = text_overlap + finish_chars_len;
+ CharBuf *value;
+ char *ptr;
+
+ // Allocate space.
+ if (self->value == NULL) {
+ self->value = (Obj*)CB_new(total_text_len);
+ }
+ value = (CharBuf*)self->value;
+ ptr = CB_Grow(value, total_text_len);
+
+ // Set the value text.
+ InStream_Read_Bytes(instream, ptr + text_overlap, finish_chars_len);
+ CB_Set_Size(value, total_text_len);
+ if (!StrHelp_utf8_valid(ptr, total_text_len)) {
+ THROW(ERR, "Invalid UTF-8 sequence in '%o' at byte %i64",
+ InStream_Get_Filename(instream),
+ InStream_Tell(instream) - finish_chars_len);
+ }
+
+ // Null-terminate.
+ ptr[total_text_len] = '\0';
+}
+
+
diff --git a/core/Lucy/Plan/TextType.cfh b/core/Lucy/Plan/TextType.cfh
new file mode 100644
index 0000000..33cdc17
--- /dev/null
+++ b/core/Lucy/Plan/TextType.cfh
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Plan::TextType inherits Lucy::Plan::FieldType {
+ incremented CharBuf*
+ Make_Blank(TextType *self);
+
+ incremented TermStepper*
+ Make_Term_Stepper(TextType *self);
+
+ int8_t
+ Primitive_ID(TextType *self);
+}
+
+class Lucy::Index::TermStepper::TextTermStepper
+ inherits Lucy::Index::TermStepper {
+
+ inert incremented TextTermStepper*
+ new();
+
+ inert TextTermStepper*
+ init(TextTermStepper *self);
+
+ public void
+ Reset(TextTermStepper *self);
+
+ /**
+ * @param value A CharBuf.
+ */
+ public void
+ Set_Value(TextTermStepper *self, Obj *value = NULL);
+
+ public void
+ Write_Key_Frame(TextTermStepper *self, OutStream *outstream, Obj *value);
+
+ public void
+ Write_Delta(TextTermStepper *self, OutStream *outstream, Obj *value);
+
+ public void
+ Read_Key_Frame(TextTermStepper *self, InStream *instream);
+
+ public void
+ Read_Delta(TextTermStepper *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/ANDMatcher.c b/core/Lucy/Search/ANDMatcher.c
new file mode 100644
index 0000000..2277300
--- /dev/null
+++ b/core/Lucy/Search/ANDMatcher.c
@@ -0,0 +1,163 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ANDMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/ANDMatcher.h"
+#include "Lucy/Index/Similarity.h"
+
+ANDMatcher*
+ANDMatcher_new(VArray *children, Similarity *sim) {
+ ANDMatcher *self = (ANDMatcher*)VTable_Make_Obj(ANDMATCHER);
+ return ANDMatcher_init(self, children, sim);
+}
+
+ANDMatcher*
+ANDMatcher_init(ANDMatcher *self, VArray *children, Similarity *sim) {
+ uint32_t i;
+
+ // Init.
+ PolyMatcher_init((PolyMatcher*)self, children, sim);
+ self->first_time = true;
+
+ // Assign.
+ self->more = self->num_kids ? true : false;
+ self->kids = (Matcher**)MALLOCATE(self->num_kids * sizeof(Matcher*));
+ for (i = 0; i < self->num_kids; i++) {
+ Matcher *child = (Matcher*)VA_Fetch(children, i);
+ self->kids[i] = child;
+ if (!Matcher_Next(child)) { self->more = false; }
+ }
+
+ // Derive.
+ self->matching_kids = self->num_kids;
+
+ return self;
+}
+
+void
+ANDMatcher_destroy(ANDMatcher *self) {
+ FREEMEM(self->kids);
+ SUPER_DESTROY(self, ANDMATCHER);
+}
+
+int32_t
+ANDMatcher_next(ANDMatcher *self) {
+ if (self->first_time) {
+ return ANDMatcher_Advance(self, 1);
+ }
+ if (self->more) {
+ const int32_t target = Matcher_Get_Doc_ID(self->kids[0]) + 1;
+ return ANDMatcher_Advance(self, target);
+ }
+ else {
+ return 0;
+ }
+}
+
+int32_t
+ANDMatcher_advance(ANDMatcher *self, int32_t target) {
+ Matcher **const kids = self->kids;
+ const uint32_t num_kids = self->num_kids;
+ int32_t highest = 0;
+
+ if (!self->more) { return 0; }
+
+ // First step: Advance first child and use its doc as a starting point.
+ if (self->first_time) {
+ self->first_time = false;
+ }
+ else {
+ highest = Matcher_Advance(kids[0], target);
+ if (!highest) {
+ self->more = false;
+ return 0;
+ }
+ }
+
+ // Second step: reconcile.
+ while (1) {
+ uint32_t i;
+ bool_t agreement = true;
+
+ // Scoot all Matchers up.
+ for (i = 0; i < num_kids; i++) {
+ Matcher *const child = kids[i];
+ int32_t candidate = Matcher_Get_Doc_ID(child);
+
+ // If this child is highest, others will need to catch up.
+ if (highest < candidate) {
+ highest = candidate;
+ }
+
+ // If least doc Matchers can agree on exceeds target, raise bar.
+ if (target < highest) {
+ target = highest;
+ }
+
+ // Scoot this Matcher up if not already at highest.
+ if (candidate < target) {
+ // This Matcher is definitely the highest right now.
+ highest = Matcher_Advance(child, target);
+ if (!highest) {
+ self->more = false;
+ return 0;
+ }
+ }
+ }
+
+ // If Matchers don't agree, send back through the loop.
+ for (i = 0; i < num_kids; i++) {
+ Matcher *const child = kids[i];
+ const int32_t candidate = Matcher_Get_Doc_ID(child);
+ if (candidate != highest) {
+ agreement = false;
+ break;
+ }
+ }
+
+ if (!agreement) {
+ continue;
+ }
+ if (highest >= target) {
+ break;
+ }
+ }
+
+ return highest;
+}
+
+int32_t
+ANDMatcher_get_doc_id(ANDMatcher *self) {
+ return Matcher_Get_Doc_ID(self->kids[0]);
+}
+
+float
+ANDMatcher_score(ANDMatcher *self) {
+ uint32_t i;
+ Matcher **const kids = self->kids;
+ float score = 0.0f;
+
+ for (i = 0; i < self->num_kids; i++) {
+ score += Matcher_Score(kids[i]);
+ }
+
+ score *= self->coord_factors[self->matching_kids];
+
+ return score;
+}
+
diff --git a/core/Lucy/Search/ANDMatcher.cfh b/core/Lucy/Search/ANDMatcher.cfh
new file mode 100644
index 0000000..a0646b5
--- /dev/null
+++ b/core/Lucy/Search/ANDMatcher.cfh
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Intersect multiple required Matchers.
+ */
+
+class Lucy::Search::ANDMatcher inherits Lucy::Search::PolyMatcher {
+
+ Matcher **kids;
+ bool_t more;
+ bool_t first_time;
+
+ inert incremented ANDMatcher*
+ new(VArray *children, Similarity *sim);
+
+ inert ANDMatcher*
+ init(ANDMatcher *self, VArray *children, Similarity *similarity);
+
+ public void
+ Destroy(ANDMatcher *self);
+
+ public int32_t
+ Next(ANDMatcher *self);
+
+ public int32_t
+ Advance(ANDMatcher *self, int32_t target);
+
+ public float
+ Score(ANDMatcher *self);
+
+ public int32_t
+ Get_Doc_ID(ANDMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/ANDQuery.c b/core/Lucy/Search/ANDQuery.c
new file mode 100644
index 0000000..41634a9
--- /dev/null
+++ b/core/Lucy/Search/ANDQuery.c
@@ -0,0 +1,135 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ANDQUERY
+#define C_LUCY_ANDCOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/ANDMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+ANDQuery*
+ANDQuery_new(VArray *children) {
+ ANDQuery *self = (ANDQuery*)VTable_Make_Obj(ANDQUERY);
+ return ANDQuery_init(self, children);
+}
+
+ANDQuery*
+ANDQuery_init(ANDQuery *self, VArray *children) {
+ return (ANDQuery*)PolyQuery_init((PolyQuery*)self, children);
+}
+
+CharBuf*
+ANDQuery_to_string(ANDQuery *self) {
+ uint32_t num_kids = VA_Get_Size(self->children);
+ if (!num_kids) { return CB_new_from_trusted_utf8("()", 2); }
+ else {
+ CharBuf *retval = CB_new_from_trusted_utf8("(", 1);
+ uint32_t i;
+ for (i = 0; i < num_kids; i++) {
+ CharBuf *kid_string = Obj_To_String(VA_Fetch(self->children, i));
+ CB_Cat(retval, kid_string);
+ DECREF(kid_string);
+ if (i == num_kids - 1) {
+ CB_Cat_Trusted_Str(retval, ")", 1);
+ }
+ else {
+ CB_Cat_Trusted_Str(retval, " AND ", 5);
+ }
+ }
+ return retval;
+ }
+}
+
+
+bool_t
+ANDQuery_equals(ANDQuery *self, Obj *other) {
+ if ((ANDQuery*)other == self) { return true; }
+ if (!Obj_Is_A(other, ANDQUERY)) { return false; }
+ return PolyQuery_equals((PolyQuery*)self, other);
+}
+
+Compiler*
+ANDQuery_make_compiler(ANDQuery *self, Searcher *searcher, float boost) {
+ return (Compiler*)ANDCompiler_new(self, searcher, boost);
+}
+
+/**********************************************************************/
+
+ANDCompiler*
+ANDCompiler_new(ANDQuery *parent, Searcher *searcher, float boost) {
+ ANDCompiler *self = (ANDCompiler*)VTable_Make_Obj(ANDCOMPILER);
+ return ANDCompiler_init(self, parent, searcher, boost);
+}
+
+ANDCompiler*
+ANDCompiler_init(ANDCompiler *self, ANDQuery *parent, Searcher *searcher,
+ float boost) {
+ PolyCompiler_init((PolyCompiler*)self, (PolyQuery*)parent, searcher,
+ boost);
+ ANDCompiler_Normalize(self);
+ return self;
+}
+
+Matcher*
+ANDCompiler_make_matcher(ANDCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ uint32_t num_kids = VA_Get_Size(self->children);
+
+ if (num_kids == 1) {
+ Compiler *only_child = (Compiler*)VA_Fetch(self->children, 0);
+ return Compiler_Make_Matcher(only_child, reader, need_score);
+ }
+ else {
+ uint32_t i;
+ VArray *child_matchers = VA_new(num_kids);
+
+ // Add child matchers one by one.
+ for (i = 0; i < num_kids; i++) {
+ Compiler *child = (Compiler*)VA_Fetch(self->children, i);
+ Matcher *child_matcher
+ = Compiler_Make_Matcher(child, reader, need_score);
+
+ // If any required clause fails, the whole thing fails.
+ if (child_matcher == NULL) {
+ DECREF(child_matchers);
+ return NULL;
+ }
+ else {
+ VA_Push(child_matchers, (Obj*)child_matcher);
+ }
+ }
+
+ {
+ Matcher *retval
+ = (Matcher*)ANDMatcher_new(child_matchers,
+ ANDCompiler_Get_Similarity(self));
+ DECREF(child_matchers);
+ return retval;
+ }
+ }
+}
+
+
diff --git a/core/Lucy/Search/ANDQuery.cfh b/core/Lucy/Search/ANDQuery.cfh
new file mode 100644
index 0000000..87cf322
--- /dev/null
+++ b/core/Lucy/Search/ANDQuery.cfh
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Intersect multiple result sets.
+ *
+ * ANDQuery is a composite L<Query|Lucy::Search::Query> which matches
+ * only when all of its children match, so its result set is the intersection
+ * of their result sets. Documents which match receive a summed score.
+ */
+class Lucy::Search::ANDQuery inherits Lucy::Search::PolyQuery
+ : dumpable {
+
+ inert incremented ANDQuery*
+ new(VArray *children = NULL);
+
+ /**
+ * @param children An array of child Queries.
+ */
+ public inert ANDQuery*
+ init(ANDQuery *self, VArray *children = NULL);
+
+ public incremented Compiler*
+ Make_Compiler(ANDQuery *self, Searcher *searcher, float boost);
+
+ public incremented CharBuf*
+ To_String(ANDQuery *self);
+
+ public bool_t
+ Equals(ANDQuery *self, Obj *other);
+}
+
+class Lucy::Search::ANDCompiler
+ inherits Lucy::Search::PolyCompiler {
+
+ inert incremented ANDCompiler*
+ new(ANDQuery *parent, Searcher *searcher, float boost);
+
+ inert ANDCompiler*
+ init(ANDCompiler *self, ANDQuery *parent, Searcher *searcher,
+ float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(ANDCompiler *self, SegReader *reader, bool_t need_score);
+}
+
+
diff --git a/core/Lucy/Search/BitVecMatcher.c b/core/Lucy/Search/BitVecMatcher.c
new file mode 100644
index 0000000..7c9f25f
--- /dev/null
+++ b/core/Lucy/Search/BitVecMatcher.c
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BITVECMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/BitVecMatcher.h"
+
+BitVecMatcher*
+BitVecMatcher_new(BitVector *bit_vector) {
+ BitVecMatcher *self = (BitVecMatcher*)VTable_Make_Obj(BITVECMATCHER);
+ return BitVecMatcher_init(self, bit_vector);
+}
+
+BitVecMatcher*
+BitVecMatcher_init(BitVecMatcher *self, BitVector *bit_vector) {
+ Matcher_init((Matcher*)self);
+ self->bit_vec = (BitVector*)INCREF(bit_vector);
+ self->doc_id = 0;
+ return self;
+}
+
+void
+BitVecMatcher_destroy(BitVecMatcher *self) {
+ DECREF(self->bit_vec);
+ SUPER_DESTROY(self, BITVECMATCHER);
+}
+
+int32_t
+BitVecMatcher_next(BitVecMatcher *self) {
+ self->doc_id = BitVec_Next_Hit(self->bit_vec, self->doc_id + 1);
+ return self->doc_id == -1 ? 0 : self->doc_id;
+}
+
+int32_t
+BitVecMatcher_advance(BitVecMatcher *self, int32_t target) {
+ self->doc_id = BitVec_Next_Hit(self->bit_vec, target);
+ return self->doc_id == -1 ? 0 : self->doc_id;
+}
+
+int32_t
+BitVecMatcher_get_doc_id(BitVecMatcher *self) {
+ return self->doc_id;
+}
+
+
diff --git a/core/Lucy/Search/BitVecMatcher.cfh b/core/Lucy/Search/BitVecMatcher.cfh
new file mode 100644
index 0000000..53225f2
--- /dev/null
+++ b/core/Lucy/Search/BitVecMatcher.cfh
@@ -0,0 +1,45 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Iterator for deleted document ids.
+ */
+class Lucy::Search::BitVecMatcher inherits Lucy::Search::Matcher {
+
+ BitVector *bit_vec;
+ int32_t doc_id;
+
+ public inert incremented BitVecMatcher*
+ new(BitVector *bit_vector);
+
+ public inert BitVecMatcher*
+ init(BitVecMatcher *self, BitVector *bit_vector);
+
+ public int32_t
+ Next(BitVecMatcher *self);
+
+ public int32_t
+ Advance(BitVecMatcher *self, int32_t target);
+
+ public int32_t
+ Get_Doc_ID(BitVecMatcher *self);
+
+ public void
+ Destroy(BitVecMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/Collector.c b/core/Lucy/Search/Collector.c
new file mode 100644
index 0000000..f0da8ee
--- /dev/null
+++ b/core/Lucy/Search/Collector.c
@@ -0,0 +1,136 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_COLLECTOR
+#define C_LUCY_BITCOLLECTOR
+#define C_LUCY_OFFSETCOLLECTOR
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Collector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Search/Matcher.h"
+
+Collector*
+Coll_init(Collector *self) {
+ ABSTRACT_CLASS_CHECK(self, COLLECTOR);
+ self->reader = NULL;
+ self->matcher = NULL;
+ self->base = 0;
+ return self;
+}
+
+void
+Coll_destroy(Collector *self) {
+ DECREF(self->reader);
+ DECREF(self->matcher);
+ SUPER_DESTROY(self, COLLECTOR);
+}
+
+void
+Coll_set_reader(Collector *self, SegReader *reader) {
+ DECREF(self->reader);
+ self->reader = (SegReader*)INCREF(reader);
+}
+
+void
+Coll_set_matcher(Collector *self, Matcher *matcher) {
+ DECREF(self->matcher);
+ self->matcher = (Matcher*)INCREF(matcher);
+}
+
+void
+Coll_set_base(Collector *self, int32_t base) {
+ self->base = base;
+}
+
+BitCollector*
+BitColl_new(BitVector *bit_vec) {
+ BitCollector *self = (BitCollector*)VTable_Make_Obj(BITCOLLECTOR);
+ return BitColl_init(self, bit_vec);
+}
+
+BitCollector*
+BitColl_init(BitCollector *self, BitVector *bit_vec) {
+ Coll_init((Collector*)self);
+ self->bit_vec = (BitVector*)INCREF(bit_vec);
+ return self;
+}
+
+void
+BitColl_destroy(BitCollector *self) {
+ DECREF(self->bit_vec);
+ SUPER_DESTROY(self, BITCOLLECTOR);
+}
+
+void
+BitColl_collect(BitCollector *self, int32_t doc_id) {
+ // Add the doc_id to the BitVector.
+ BitVec_Set(self->bit_vec, (self->base + doc_id));
+}
+
+bool_t
+BitColl_need_score(BitCollector *self) {
+ UNUSED_VAR(self);
+ return false;
+}
+
+OffsetCollector*
+OffsetColl_new(Collector *inner_coll, int32_t offset) {
+ OffsetCollector *self
+ = (OffsetCollector*)VTable_Make_Obj(OFFSETCOLLECTOR);
+ return OffsetColl_init(self, inner_coll, offset);
+}
+
+OffsetCollector*
+OffsetColl_init(OffsetCollector *self, Collector *inner_coll, int32_t offset) {
+ Coll_init((Collector*)self);
+ self->offset = offset;
+ self->inner_coll = (Collector*)INCREF(inner_coll);
+ return self;
+}
+
+void
+OffsetColl_destroy(OffsetCollector *self) {
+ DECREF(self->inner_coll);
+ SUPER_DESTROY(self, OFFSETCOLLECTOR);
+}
+
+void
+OffsetColl_set_reader(OffsetCollector *self, SegReader *reader) {
+ Coll_Set_Reader(self->inner_coll, reader);
+}
+
+void
+OffsetColl_set_base(OffsetCollector *self, int32_t base) {
+ Coll_Set_Base(self->inner_coll, base);
+}
+
+void
+OffsetColl_set_matcher(OffsetCollector *self, Matcher *matcher) {
+ Coll_Set_Matcher(self->inner_coll, matcher);
+}
+
+void
+OffsetColl_collect(OffsetCollector *self, int32_t doc_id) {
+ Coll_Collect(self->inner_coll, (doc_id + self->offset));
+}
+
+bool_t
+OffsetColl_need_score(OffsetCollector *self) {
+ return Coll_Need_Score(self->inner_coll);
+}
+
+
diff --git a/core/Lucy/Search/Collector.cfh b/core/Lucy/Search/Collector.cfh
new file mode 100644
index 0000000..b24e8ed
--- /dev/null
+++ b/core/Lucy/Search/Collector.cfh
@@ -0,0 +1,143 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Process hits.
+ *
+ * A Collector decides what to do with the hits that a
+ * L<Matcher|Lucy::Search::Matcher> iterates through, based on how the
+ * abstract Collect() method is implemented.
+ *
+ * Collectors operate on individual segments, but must operate within the
+ * context of a larger collection. Each time the collector moves to a new
+ * segment, Set_Reader(), Set_Base() and Set_Matcher() will be called, and the
+ * collector must take the updated information into account.
+ */
+
+abstract class Lucy::Search::Collector cnick Coll
+ inherits Lucy::Object::Obj {
+
+ SegReader *reader;
+ Matcher *matcher;
+ int32_t base;
+
+ /** Abstract constructor. Takes no arguments.
+ */
+ public inert Collector*
+ init(Collector *self);
+
+ public void
+ Destroy(Collector *self);
+
+ /** Do something with a doc id. (For instance, keep track of the docs
+ * with the ten highest scores.)
+ *
+ * @param doc_id A segment document id.
+ */
+ public abstract void
+ Collect(Collector *self, int32_t doc_id);
+
+ /** Setter for "reader".
+ */
+ public void
+ Set_Reader(Collector *self, SegReader *reader);
+
+ /** Set the "base" document id, an offset which must be added to the
+ * <code>doc_id</code> supplied via Collect() to get the doc id for the
+ * larger index.
+ */
+ public void
+ Set_Base(Collector *self, int32_t base);
+
+ /** Indicate whether the Collector will call Score() on its Matcher.
+ */
+ public abstract bool_t
+ Need_Score(Collector *self);
+
+ /** Setter for "matcher".
+ */
+ public void
+ Set_Matcher(Collector *self, Matcher *matcher);
+}
+
+/** Collector which records doc nums in a BitVector.
+ *
+ * BitCollector is a Collector which saves matching document ids in a
+ * L<BitVector|Lucy::Object::BitVector>. It is useful for recording the
+ * entire set of documents which matches a query.
+ */
+class Lucy::Search::Collector::BitCollector cnick BitColl
+ inherits Lucy::Search::Collector {
+
+ BitVector *bit_vec;
+
+ /**
+ * @param bit_vector A Lucy::Object::BitVector.
+ */
+ public inert BitCollector*
+ init(BitCollector *self, BitVector *bit_vector);
+
+ public void
+ Destroy(BitCollector *self);
+
+ /** Set bit in the object's BitVector for the supplied doc id.
+ */
+ public void
+ Collect(BitCollector *self, int32_t doc_id);
+
+ /** Returns false, since BitCollector requires only doc ids.
+ */
+ public bool_t
+ Need_Score(BitCollector *self);
+}
+
+class Lucy::Search::Collector::OffsetCollector cnick OffsetColl
+ inherits Lucy::Search::Collector {
+
+ int32_t offset;
+ Collector *inner_coll;
+
+ inert incremented OffsetCollector*
+ new(Collector *collector, int32_t offset);
+
+ /** Wrap another Collector, adding a constant offset to each document
+ * number. Useful when combining results from multiple independent
+ * indexes.
+ */
+ inert OffsetCollector*
+ init(OffsetCollector *self, Collector *collector, int32_t offset);
+
+ public void
+ Destroy(OffsetCollector *self);
+
+ public void
+ Collect(OffsetCollector *self, int32_t doc_id);
+
+ public bool_t
+ Need_Score(OffsetCollector *self);
+
+ public void
+ Set_Reader(OffsetCollector *self, SegReader *reader);
+
+ public void
+ Set_Base(OffsetCollector *self, int32_t base);
+
+ public void
+ Set_Matcher(OffsetCollector *self, Matcher *matcher);
+}
+
+
diff --git a/core/Lucy/Search/Collector/SortCollector.c b/core/Lucy/Search/Collector/SortCollector.c
new file mode 100644
index 0000000..bf14149
--- /dev/null
+++ b/core/Lucy/Search/Collector/SortCollector.c
@@ -0,0 +1,631 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTCOLLECTOR
+#define C_LUCY_MATCHDOC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Collector/SortCollector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Index/SortCache/NumericSortCache.h"
+#include "Lucy/Index/SortCache/TextSortCache.h"
+#include "Lucy/Index/SortReader.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/HitQueue.h"
+#include "Lucy/Search/MatchDoc.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Search/SortRule.h"
+#include "Lucy/Search/SortSpec.h"
+
+#define COMPARE_BY_SCORE 0x1
+#define COMPARE_BY_SCORE_REV 0x2
+#define COMPARE_BY_DOC_ID 0x3
+#define COMPARE_BY_DOC_ID_REV 0x4
+#define COMPARE_BY_ORD1 0x5
+#define COMPARE_BY_ORD1_REV 0x6
+#define COMPARE_BY_ORD2 0x7
+#define COMPARE_BY_ORD2_REV 0x8
+#define COMPARE_BY_ORD4 0x9
+#define COMPARE_BY_ORD4_REV 0xA
+#define COMPARE_BY_ORD8 0xB
+#define COMPARE_BY_ORD8_REV 0xC
+#define COMPARE_BY_ORD16 0xD
+#define COMPARE_BY_ORD16_REV 0xE
+#define COMPARE_BY_ORD32 0xF
+#define COMPARE_BY_ORD32_REV 0x10
+#define COMPARE_BY_NATIVE_ORD16 0x11
+#define COMPARE_BY_NATIVE_ORD16_REV 0x12
+#define COMPARE_BY_NATIVE_ORD32 0x13
+#define COMPARE_BY_NATIVE_ORD32_REV 0x14
+#define AUTO_ACCEPT 0x15
+#define AUTO_REJECT 0x16
+#define AUTO_TIE 0x17
+#define ACTIONS_MASK 0x1F
+
+// Pick an action based on a SortRule and if needed, a SortCache.
+static int8_t
+S_derive_action(SortRule *rule, SortCache *sort_cache);
+
+// Decide whether a doc should be inserted into the HitQueue.
+static INLINE bool_t
+SI_competitive(SortCollector *self, int32_t doc_id);
+
+SortCollector*
+SortColl_new(Schema *schema, SortSpec *sort_spec, uint32_t wanted) {
+ SortCollector *self = (SortCollector*)VTable_Make_Obj(SORTCOLLECTOR);
+ return SortColl_init(self, schema, sort_spec, wanted);
+}
+
+// Default to sort-by-score-then-doc-id.
+static VArray*
+S_default_sort_rules() {
+ VArray *rules = VA_new(1);
+ VA_Push(rules, (Obj*)SortRule_new(SortRule_SCORE, NULL, false));
+ VA_Push(rules, (Obj*)SortRule_new(SortRule_DOC_ID, NULL, false));
+ return rules;
+}
+
+SortCollector*
+SortColl_init(SortCollector *self, Schema *schema, SortSpec *sort_spec,
+ uint32_t wanted) {
+ VArray *rules = sort_spec
+ ? (VArray*)INCREF(SortSpec_Get_Rules(sort_spec))
+ : S_default_sort_rules();
+ uint32_t num_rules = VA_Get_Size(rules);
+ uint32_t i;
+
+ // Validate.
+ if (sort_spec && !schema) {
+ THROW(ERR, "Can't supply a SortSpec without a Schema.");
+ }
+ if (!num_rules) {
+ THROW(ERR, "Can't supply a SortSpec with no SortRules.");
+ }
+
+ // Init.
+ Coll_init((Collector*)self);
+ self->total_hits = 0;
+ self->bubble_doc = I32_MAX;
+ self->bubble_score = F32_NEGINF;
+ self->seg_doc_max = 0;
+
+ // Assign.
+ self->wanted = wanted;
+
+ // Derive.
+ self->hit_q = HitQ_new(schema, sort_spec, wanted);
+ self->rules = rules; // absorb refcount.
+ self->num_rules = num_rules;
+ self->sort_caches = (SortCache**)CALLOCATE(num_rules, sizeof(SortCache*));
+ self->ord_arrays = (void**)CALLOCATE(num_rules, sizeof(void*));
+ self->actions = (uint8_t*)CALLOCATE(num_rules, sizeof(uint8_t));
+
+ // Build up an array of "actions" which we will execute during each call
+ // to Collect(). Determine whether we need to track scores and field
+ // values.
+ self->need_score = false;
+ self->need_values = false;
+ for (i = 0; i < num_rules; i++) {
+ SortRule *rule = (SortRule*)VA_Fetch(rules, i);
+ int32_t rule_type = SortRule_Get_Type(rule);
+ self->actions[i] = S_derive_action(rule, NULL);
+ if (rule_type == SortRule_SCORE) {
+ self->need_score = true;
+ }
+ else if (rule_type == SortRule_FIELD) {
+ CharBuf *field = SortRule_Get_Field(rule);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ if (!type || !FType_Sortable(type)) {
+ THROW(ERR, "'%o' isn't a sortable field", field);
+ }
+ self->need_values = true;
+ }
+ }
+
+ // Perform an optimization. So long as we always collect docs in
+ // ascending order, Collect() will favor lower doc numbers -- so we may
+ // not need to execute a final COMPARE_BY_DOC_ID action.
+ self->num_actions = num_rules;
+ if (self->actions[num_rules - 1] == COMPARE_BY_DOC_ID) {
+ self->num_actions--;
+ }
+
+ // Override our derived actions with an action which will be excecuted
+ // autmatically until the queue fills up.
+ self->auto_actions = (uint8_t*)MALLOCATE(1);
+ self->auto_actions[0] = wanted ? AUTO_ACCEPT : AUTO_REJECT;
+ self->derived_actions = self->actions;
+ self->actions = self->auto_actions;
+
+
+ // Prepare a MatchDoc-in-waiting.
+ {
+ VArray *values = self->need_values ? VA_new(num_rules) : NULL;
+ float score = self->need_score ? F32_NEGINF : F32_NAN;
+ self->bumped = MatchDoc_new(I32_MAX, score, values);
+ DECREF(values);
+ }
+
+ return self;
+}
+
+void
+SortColl_destroy(SortCollector *self) {
+ DECREF(self->hit_q);
+ DECREF(self->rules);
+ DECREF(self->bumped);
+ FREEMEM(self->sort_caches);
+ FREEMEM(self->ord_arrays);
+ FREEMEM(self->auto_actions);
+ FREEMEM(self->derived_actions);
+ SUPER_DESTROY(self, SORTCOLLECTOR);
+}
+
+static int8_t
+S_derive_action(SortRule *rule, SortCache *cache) {
+ int32_t rule_type = SortRule_Get_Type(rule);
+ bool_t reverse = !!SortRule_Get_Reverse(rule);
+
+ if (rule_type == SortRule_SCORE) {
+ return COMPARE_BY_SCORE + reverse;
+ }
+ else if (rule_type == SortRule_DOC_ID) {
+ return COMPARE_BY_DOC_ID + reverse;
+ }
+ else if (rule_type == SortRule_FIELD) {
+ if (cache) {
+ int8_t width = SortCache_Get_Ord_Width(cache);
+ switch (width) {
+ case 1: return COMPARE_BY_ORD1 + reverse;
+ case 2: return COMPARE_BY_ORD2 + reverse;
+ case 4: return COMPARE_BY_ORD4 + reverse;
+ case 8: return COMPARE_BY_ORD8 + reverse;
+ case 16:
+ if (SortCache_Get_Native_Ords(cache)) {
+ return COMPARE_BY_NATIVE_ORD16 + reverse;
+ }
+ else {
+ return COMPARE_BY_ORD16 + reverse;
+ }
+ case 32:
+ if (SortCache_Get_Native_Ords(cache)) {
+ return COMPARE_BY_NATIVE_ORD32 + reverse;
+ }
+ else {
+ return COMPARE_BY_ORD32 + reverse;
+ }
+ default: THROW(ERR, "Unknown width: %i8", width);
+ }
+ }
+ else {
+ return AUTO_TIE;
+ }
+ }
+ else {
+ THROW(ERR, "Unrecognized SortRule type %i32", rule_type);
+ }
+ UNREACHABLE_RETURN(int8_t);
+}
+
+void
+SortColl_set_reader(SortCollector *self, SegReader *reader) {
+ SortReader *sort_reader
+ = (SortReader*)SegReader_Fetch(reader, VTable_Get_Name(SORTREADER));
+
+ // Reset threshold variables and trigger auto-action behavior.
+ self->bumped->doc_id = I32_MAX;
+ self->bubble_doc = I32_MAX;
+ self->bumped->score = self->need_score ? F32_NEGINF : F32_NAN;
+ self->bubble_score = self->need_score ? F32_NEGINF : F32_NAN;
+ self->actions = self->auto_actions;
+
+ // Obtain sort caches. Derive actions array for this segment.
+ if (self->need_values && sort_reader) {
+ uint32_t i, max;
+ for (i = 0, max = self->num_rules; i < max; i++) {
+ SortRule *rule = (SortRule*)VA_Fetch(self->rules, i);
+ CharBuf *field = SortRule_Get_Field(rule);
+ SortCache *cache = field
+ ? SortReader_Fetch_Sort_Cache(sort_reader, field)
+ : NULL;
+ self->sort_caches[i] = cache;
+ self->derived_actions[i] = S_derive_action(rule, cache);
+ if (cache) { self->ord_arrays[i] = SortCache_Get_Ords(cache); }
+ else { self->ord_arrays[i] = NULL; }
+ }
+ }
+ self->seg_doc_max = reader ? SegReader_Doc_Max(reader) : 0;
+ Coll_set_reader((Collector*)self, reader);
+}
+
+VArray*
+SortColl_pop_match_docs(SortCollector *self) {
+ return HitQ_Pop_All(self->hit_q);
+}
+
+uint32_t
+SortColl_get_total_hits(SortCollector *self) {
+ return self->total_hits;
+}
+
+bool_t
+SortColl_need_score(SortCollector *self) {
+ return self->need_score;
+}
+
+void
+SortColl_collect(SortCollector *self, int32_t doc_id) {
+ // Add to the total number of hits.
+ self->total_hits++;
+
+ // Collect this hit if it's competitive.
+ if (SI_competitive(self, doc_id)) {
+ MatchDoc *const match_doc = self->bumped;
+ match_doc->doc_id = doc_id + self->base;
+
+ if (self->need_score && match_doc->score == F32_NEGINF) {
+ match_doc->score = Matcher_Score(self->matcher);
+ }
+
+ // Fetch values so that cross-segment sorting can work.
+ if (self->need_values) {
+ VArray *values = match_doc->values;
+ uint32_t i, max;
+
+ for (i = 0, max = self->num_rules; i < max; i++) {
+ SortCache *cache = self->sort_caches[i];
+ Obj *old_val = (Obj*)VA_Delete(values, i);
+ if (cache) {
+ int32_t ord = SortCache_Ordinal(cache, doc_id);
+ Obj *blank = old_val
+ ? old_val
+ : SortCache_Make_Blank(cache);
+ Obj *val = SortCache_Value(cache, ord, blank);
+ if (val) { VA_Store(values, i, (Obj*)val); }
+ else { DECREF(blank); }
+ }
+ }
+ }
+
+ // Insert the new MatchDoc.
+ self->bumped = (MatchDoc*)HitQ_Jostle(self->hit_q, (Obj*)match_doc);
+
+ if (self->bumped) {
+ if (self->bumped == match_doc) {
+ /* The queue is full, and we have established a threshold for
+ * this segment as to what sort of document is definitely not
+ * acceptable. Turn off AUTO_ACCEPT and start actually
+ * testing whether hits are competitive. */
+ self->bubble_score = match_doc->score;
+ self->bubble_doc = doc_id;
+ self->actions = self->derived_actions;
+ }
+
+ // Recycle.
+ self->bumped->score = self->need_score ? F32_NEGINF : F32_NAN;
+ }
+ else {
+ // The queue isn't full yet, so create a fresh MatchDoc.
+ VArray *values = self->need_values
+ ? VA_new(self->num_rules)
+ : NULL;
+ float fake_score = self->need_score ? F32_NEGINF : F32_NAN;
+ self->bumped = MatchDoc_new(I32_MAX, fake_score, values);
+ DECREF(values);
+ }
+
+ }
+}
+
+static INLINE int32_t
+SI_compare_by_ord1(SortCollector *self, uint32_t tick, int32_t a, int32_t b) {
+ void *const ords = self->ord_arrays[tick];
+ int32_t a_ord = NumUtil_u1get(ords, a);
+ int32_t b_ord = NumUtil_u1get(ords, b);
+ return a_ord - b_ord;
+}
+static INLINE int32_t
+SI_compare_by_ord2(SortCollector *self, uint32_t tick, int32_t a, int32_t b) {
+ void *const ords = self->ord_arrays[tick];
+ int32_t a_ord = NumUtil_u2get(ords, a);
+ int32_t b_ord = NumUtil_u2get(ords, b);
+ return a_ord - b_ord;
+}
+static INLINE int32_t
+SI_compare_by_ord4(SortCollector *self, uint32_t tick, int32_t a, int32_t b) {
+ void *const ords = self->ord_arrays[tick];
+ int32_t a_ord = NumUtil_u4get(ords, a);
+ int32_t b_ord = NumUtil_u4get(ords, b);
+ return a_ord - b_ord;
+}
+static INLINE int32_t
+SI_compare_by_ord8(SortCollector *self, uint32_t tick, int32_t a, int32_t b) {
+ uint8_t *ords = (uint8_t*)self->ord_arrays[tick];
+ int32_t a_ord = ords[a];
+ int32_t b_ord = ords[b];
+ return a_ord - b_ord;
+}
+static INLINE int32_t
+SI_compare_by_ord16(SortCollector *self, uint32_t tick, int32_t a, int32_t b) {
+ uint8_t *ord_bytes = (uint8_t*)self->ord_arrays[tick];
+ uint8_t *address_a = ord_bytes + a * sizeof(uint16_t);
+ uint8_t *address_b = ord_bytes + b * sizeof(uint16_t);
+ int32_t ord_a = NumUtil_decode_bigend_u16(address_a);
+ int32_t ord_b = NumUtil_decode_bigend_u16(address_b);
+ return ord_a - ord_b;
+}
+static INLINE int32_t
+SI_compare_by_ord32(SortCollector *self, uint32_t tick, int32_t a, int32_t b) {
+ uint8_t *ord_bytes = (uint8_t*)self->ord_arrays[tick];
+ uint8_t *address_a = ord_bytes + a * sizeof(uint32_t);
+ uint8_t *address_b = ord_bytes + b * sizeof(uint32_t);
+ int32_t ord_a = NumUtil_decode_bigend_u32(address_a);
+ int32_t ord_b = NumUtil_decode_bigend_u32(address_b);
+ return ord_a - ord_b;
+}
+static INLINE int32_t
+SI_compare_by_native_ord16(SortCollector *self, uint32_t tick,
+ int32_t a, int32_t b) {
+ uint16_t *ords = (uint16_t*)self->ord_arrays[tick];
+ int32_t a_ord = ords[a];
+ int32_t b_ord = ords[b];
+ return a_ord - b_ord;
+}
+static INLINE int32_t
+SI_compare_by_native_ord32(SortCollector *self, uint32_t tick,
+ int32_t a, int32_t b) {
+ int32_t *ords = (int32_t*)self->ord_arrays[tick];
+ return ords[a] - ords[b];
+}
+
+// Bounds checking for doc id against the segment doc_max. We assume that any
+// sort cache ord arrays can accomodate lookups up to this number.
+static INLINE int32_t
+SI_validate_doc_id(SortCollector *self, int32_t doc_id) {
+ // Check as uint32_t since we're using these doc ids as array indexes.
+ if ((uint32_t)doc_id > (uint32_t)self->seg_doc_max) {
+ THROW(ERR, "Doc ID %i32 greater than doc max %i32", doc_id,
+ self->seg_doc_max);
+ }
+ return doc_id;
+}
+
+static INLINE bool_t
+SI_competitive(SortCollector *self, int32_t doc_id) {
+ /* Ordinarily, we would cache local copies of more member variables in
+ * const automatic variables in order to improve code clarity and provide
+ * more hints to the compiler about what variables are actually invariant
+ * for the duration of this routine:
+ *
+ * uint8_t *const actions = self->actions;
+ * const uint32_t num_rules = self->num_rules;
+ * const int32_t bubble_doc = self->bubble_doc;
+ *
+ * However, our major goal is to return as quickly as possible, and the
+ * common case is that we'll have our answer before the first loop iter
+ * finishes -- so we don't worry about the cost of performing extra
+ * dereferencing on subsequent loop iters.
+ *
+ * The goal of returning quickly also drives the choice of a "do-while"
+ * loop instead of a "for" loop, and the switch statement optimized for
+ * compilation to a jump table.
+ */
+ uint8_t *const actions = self->actions;
+ uint32_t i = 0;
+
+ // Iterate through our array of actions, returning as quickly as possible.
+ do {
+ switch (actions[i] & ACTIONS_MASK) {
+ case AUTO_ACCEPT:
+ return true;
+ case AUTO_REJECT:
+ return false;
+ case AUTO_TIE:
+ break;
+ case COMPARE_BY_SCORE: {
+ float score = Matcher_Score(self->matcher);
+ if (*(int32_t*)&score == *(int32_t*)&self->bubble_score) {
+ break;
+ }
+ if (score > self->bubble_score) {
+ self->bumped->score = score;
+ return true;
+ }
+ else if (score < self->bubble_score) {
+ return false;
+ }
+ }
+ break;
+ case COMPARE_BY_SCORE_REV: {
+ float score = Matcher_Score(self->matcher);
+ if (*(int32_t*)&score == *(int32_t*)&self->bubble_score) {
+ break;
+ }
+ if (score < self->bubble_score) {
+ self->bumped->score = score;
+ return true;
+ }
+ else if (score > self->bubble_score) {
+ return false;
+ }
+ }
+ break;
+ case COMPARE_BY_DOC_ID:
+ if (doc_id > self->bubble_doc) { return false; }
+ else if (doc_id < self->bubble_doc) { return true; }
+ break;
+ case COMPARE_BY_DOC_ID_REV:
+ if (doc_id > self->bubble_doc) { return true; }
+ else if (doc_id < self->bubble_doc) { return false; }
+ break;
+ case COMPARE_BY_ORD1: {
+ int32_t comparison
+ = SI_compare_by_ord1(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD1_REV: {
+ int32_t comparison
+ = SI_compare_by_ord1(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD2: {
+ int32_t comparison
+ = SI_compare_by_ord2(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD2_REV: {
+ int32_t comparison
+ = SI_compare_by_ord2(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD4: {
+ int32_t comparison
+ = SI_compare_by_ord4(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD4_REV: {
+ int32_t comparison
+ = SI_compare_by_ord4(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD8: {
+ int32_t comparison
+ = SI_compare_by_ord8(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD8_REV: {
+ int32_t comparison
+ = SI_compare_by_ord8(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD16: {
+ int32_t comparison
+ = SI_compare_by_ord16(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD16_REV: {
+ int32_t comparison
+ = SI_compare_by_ord16(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD32: {
+ int32_t comparison
+ = SI_compare_by_ord32(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_ORD32_REV: {
+ int32_t comparison
+ = SI_compare_by_ord32(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_NATIVE_ORD16: {
+ int32_t comparison
+ = SI_compare_by_native_ord16(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_NATIVE_ORD16_REV: {
+ int32_t comparison
+ = SI_compare_by_native_ord16(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_NATIVE_ORD32: {
+ int32_t comparison
+ = SI_compare_by_native_ord32(
+ self, i, SI_validate_doc_id(self, doc_id),
+ self->bubble_doc);
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_NATIVE_ORD32_REV: {
+ int32_t comparison
+ = SI_compare_by_native_ord32(
+ self, i, self->bubble_doc,
+ SI_validate_doc_id(self, doc_id));
+ if (comparison < 0) { return true; }
+ else if (comparison > 0) { return false; }
+ }
+ break;
+ default:
+ THROW(ERR, "UNEXPECTED action %u8", actions[i]);
+ }
+ } while (++i < self->num_actions);
+
+ // If we've made it this far and we're still tied, reject the doc so that
+ // we prefer items already in the queue. This has the effect of
+ // implicitly breaking ties by doc num, since docs are collected in order.
+ return false;
+}
+
+
diff --git a/core/Lucy/Search/Collector/SortCollector.cfh b/core/Lucy/Search/Collector/SortCollector.cfh
new file mode 100644
index 0000000..99a8900
--- /dev/null
+++ b/core/Lucy/Search/Collector/SortCollector.cfh
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Collect top-sorting documents.
+ *
+ * A SortCollector sorts hits according to a SortSpec, keeping the highest
+ * ranking N documents in a priority queue.
+ */
+class Lucy::Search::Collector::SortCollector cnick SortColl
+ inherits Lucy::Search::Collector {
+
+ uint32_t wanted;
+ uint32_t total_hits;
+ HitQueue *hit_q;
+ MatchDoc *bumped;
+ VArray *rules;
+ SortCache **sort_caches;
+ void **ord_arrays;
+ uint8_t *actions;
+ uint8_t *auto_actions;
+ uint8_t *derived_actions;
+ uint32_t num_rules;
+ uint32_t num_actions;
+ float bubble_score;
+ int32_t bubble_doc;
+ int32_t seg_doc_max;
+ bool_t need_score;
+ bool_t need_values;
+
+ inert incremented SortCollector*
+ new(Schema *schema = NULL, SortSpec *sort_spec = NULL, uint32_t wanted);
+
+ /**
+ * @param schema A Schema. Required if <code>sort_spec</code> provided.
+ * @param sort_spec A SortSpec. If NULL, sort by descending score first
+ * and ascending doc id second.
+ * @param wanted Maximum number of hits to collect.
+ */
+ inert SortCollector*
+ init(SortCollector *self, Schema *schema = NULL,
+ SortSpec *sort_spec = NULL, uint32_t wanted);
+
+ /** Keep highest ranking docs.
+ */
+ public void
+ Collect(SortCollector *self, int32_t doc_id);
+
+ /** Empty out the HitQueue and return an array of sorted MatchDocs.
+ */
+ incremented VArray*
+ Pop_Match_Docs(SortCollector *self);
+
+ /** Accessor for "total_hits" member, which tracks the number of times
+ * that Collect() was called.
+ */
+ uint32_t
+ Get_Total_Hits(SortCollector *self);
+
+ public void
+ Set_Reader(SortCollector *self, SegReader *reader);
+
+ public bool_t
+ Need_Score(SortCollector *self);
+
+ public void
+ Destroy(SortCollector *self);
+}
+
+
diff --git a/core/Lucy/Search/Compiler.c b/core/Lucy/Search/Compiler.c
new file mode 100644
index 0000000..3490b02
--- /dev/null
+++ b/core/Lucy/Search/Compiler.c
@@ -0,0 +1,140 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_COMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+Compiler*
+Compiler_init(Compiler *self, Query *parent, Searcher *searcher,
+ Similarity *sim, float boost) {
+ Query_init((Query*)self, boost);
+ if (!sim) {
+ Schema *schema = Searcher_Get_Schema(searcher);
+ sim = Schema_Get_Similarity(schema);
+ }
+ self->parent = (Query*)INCREF(parent);
+ self->sim = (Similarity*)INCREF(sim);
+ ABSTRACT_CLASS_CHECK(self, COMPILER);
+ return self;
+}
+
+void
+Compiler_destroy(Compiler *self) {
+ DECREF(self->parent);
+ DECREF(self->sim);
+ SUPER_DESTROY(self, COMPILER);
+}
+
+float
+Compiler_get_weight(Compiler *self) {
+ return Compiler_Get_Boost(self);
+}
+
+Similarity*
+Compiler_get_similarity(Compiler *self) {
+ return self->sim;
+}
+
+Query*
+Compiler_get_parent(Compiler *self) {
+ return self->parent;
+}
+
+float
+Compiler_sum_of_squared_weights(Compiler *self) {
+ UNUSED_VAR(self);
+ return 1.0f;
+}
+
+void
+Compiler_apply_norm_factor(Compiler *self, float factor) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(factor);
+}
+
+void
+Compiler_normalize(Compiler *self) {
+ // factor = (tf_q * idf_t)
+ float factor = Compiler_Sum_Of_Squared_Weights(self);
+
+ // factor /= norm_q
+ factor = Sim_Query_Norm(self->sim, factor);
+
+ // weight *= factor
+ Compiler_Apply_Norm_Factor(self, factor);
+}
+
+VArray*
+Compiler_highlight_spans(Compiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(searcher);
+ UNUSED_VAR(doc_vec);
+ UNUSED_VAR(field);
+ return VA_new(0);
+}
+
+CharBuf*
+Compiler_to_string(Compiler *self) {
+ CharBuf *stringified_query = Query_To_String(self->parent);
+ CharBuf *string = CB_new_from_trusted_utf8("compiler(", 9);
+ CB_Cat(string, stringified_query);
+ CB_Cat_Trusted_Str(string, ")", 1);
+ DECREF(stringified_query);
+ return string;
+}
+
+bool_t
+Compiler_equals(Compiler *self, Obj *other) {
+ Compiler *twin = (Compiler*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, COMPILER)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!Query_Equals(self->parent, (Obj*)twin->parent)) { return false; }
+ if (!Sim_Equals(self->sim, (Obj*)twin->sim)) { return false; }
+ return true;
+}
+
+void
+Compiler_serialize(Compiler *self, OutStream *outstream) {
+ ABSTRACT_CLASS_CHECK(self, COMPILER);
+ OutStream_Write_F32(outstream, self->boost);
+ FREEZE(self->parent, outstream);
+ FREEZE(self->sim, outstream);
+}
+
+Compiler*
+Compiler_deserialize(Compiler *self, InStream *instream) {
+ if (!self) { THROW(ERR, "Compiler_Deserialize is abstract"); }
+ self->boost = InStream_Read_F32(instream);
+ self->parent = (Query*)THAW(instream);
+ self->sim = (Similarity*)THAW(instream);
+ return self;
+}
+
+
diff --git a/core/Lucy/Search/Compiler.cfh b/core/Lucy/Search/Compiler.cfh
new file mode 100644
index 0000000..9c2963c
--- /dev/null
+++ b/core/Lucy/Search/Compiler.cfh
@@ -0,0 +1,170 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Query-to-Matcher compiler.
+ *
+ * The purpose of the Compiler class is to take a specification in the form of
+ * a L<Query|Lucy::Search::Query> object and compile a
+ * L<Matcher|Lucy::Search::Matcher> object that can do real work.
+ *
+ * The simplest Compiler subclasses -- such as those associated with
+ * constant-scoring Query types -- might simply implement a Make_Matcher()
+ * method which passes along information verbatim from the Query to the
+ * Matcher's constructor.
+ *
+ * However it is common for the Compiler to perform some calculations which
+ * affect it's "weight" -- a floating point multiplier that the Matcher will
+ * factor into each document's score. If that is the case, then the Compiler
+ * subclass may wish to override Get_Weight(), Sum_Of_Squared_Weights(), and
+ * Apply_Norm_Factor().
+ *
+ * Compiling a Matcher is a two stage process.
+ *
+ * The first stage takes place during the Compiler's constructor, which is
+ * where the Query object meets a L<Searcher|Lucy::Search::Searcher>
+ * object for the first time. Searchers operate on a specific document
+ * collection and they can tell you certain statistical information about the
+ * collection -- such as how many total documents are in the collection, or
+ * how many documents in the collection a particular term is present in.
+ * Lucy's core Compiler classes plug this information into the classic
+ * TF/IDF weighting algorithm to adjust the Compiler's weight; custom
+ * subclasses might do something similar.
+ *
+ * The second stage of compilation is Make_Matcher(), method, which is where
+ * the Compiler meets a L<SegReader|Lucy::Index::SegReader> object.
+ * SegReaders are associated with a single segment within a single index on a
+ * single machine, and are thus lower-level than Searchers, which may
+ * represent a document collection spread out over a search cluster
+ * (comprising several indexes and many segments). The Compiler object can
+ * use new information supplied by the SegReader -- such as whether a term is
+ * missing from the local index even though it is present within the larger
+ * collection represented by the Searcher -- when figuring out what to feed to
+ * the Matchers's constructor, or whether Make_Matcher() should return a
+ * Matcher at all.
+ */
+class Lucy::Search::Compiler inherits Lucy::Search::Query {
+
+ Query *parent;
+ Similarity *sim;
+
+ /** Abstract constructor.
+ *
+ * @param parent The parent Query.
+ * @param searcher A Lucy::Search::Searcher, such as an
+ * IndexSearcher.
+ * @param similarity A Similarity.
+ * @param boost An arbitrary scoring multiplier. Defaults to the boost of
+ * the parent Query.
+ */
+ public inert Compiler*
+ init(Compiler *self, Query *parent, Searcher *searcher,
+ Similarity *similarity = NULL, float boost);
+
+ /** Factory method returning a Matcher.
+ *
+ * @param reader A SegReader.
+ * @param need_score Indicate whether the Matcher must implement Score().
+ * @return a Matcher, or NULL if the Matcher would have matched no
+ * documents.
+ */
+ public abstract incremented nullable Matcher*
+ Make_Matcher(Compiler *self, SegReader *reader, bool_t need_score);
+
+ /** Return the Compiler's numerical weight, a scoring multiplier. By
+ * default, returns the object's boost.
+ */
+ public float
+ Get_Weight(Compiler *self);
+
+ /** Accessor for the Compiler's Similarity object.
+ */
+ public nullable Similarity*
+ Get_Similarity(Compiler *self);
+
+ /** Accessor for the Compiler's parent Query object.
+ */
+ public Query*
+ Get_Parent(Compiler *self);
+
+ /** Compute and return a raw weighting factor. (This quantity is used by
+ * Normalize()). By default, simply returns 1.0.
+ */
+ public float
+ Sum_Of_Squared_Weights(Compiler *self);
+
+ /** Apply a floating point normalization multiplier. For a TermCompiler,
+ * this involves multiplying its own weight by the supplied factor;
+ * combining classes such as ORCompiler would apply the factor recursively
+ * to their children.
+ *
+ * The default implementation is a no-op; subclasses may wish to multiply
+ * their internal weight by the supplied factor.
+ *
+ * @param factor The multiplier.
+ */
+ public void
+ Apply_Norm_Factor(Compiler *self, float factor);
+
+ /** Take a newly minted Compiler object and apply query-specific
+ * normalization factors. Should be called at or near the end of
+ * construction.
+ *
+ * For a TermQuery, the scoring formula is approximately:
+ *
+ * (tf_d * idf_t / norm_d) * (tf_q * idf_t / norm_q)
+ *
+ * Normalize() is theoretically concerned with applying the second half of
+ * that formula to a the Compiler's weight. What actually happens depends
+ * on how the Compiler and Similarity methods called internally are
+ * implemented.
+ */
+ public void
+ Normalize(Compiler *self);
+
+ /** Return an array of Span objects, indicating where in the given field
+ * the text that matches the parent Query occurs and how well each snippet
+ * matches. The Span's offset and length are measured in Unicode code
+ * points.
+ *
+ * The default implementation returns an empty array.
+ *
+ * @param searcher A Searcher.
+ * @param doc_vec A DocVector.
+ * @param field The name of the field.
+ */
+ public incremented VArray*
+ Highlight_Spans(Compiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field);
+
+ public void
+ Serialize(Compiler *self, OutStream *outstream);
+
+ public incremented Compiler*
+ Deserialize(Compiler *self, InStream *instream);
+
+ public bool_t
+ Equals(Compiler *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(Compiler *self);
+
+ public void
+ Destroy(Compiler *self);
+}
+
+
diff --git a/core/Lucy/Search/HitQueue.c b/core/Lucy/Search/HitQueue.c
new file mode 100644
index 0000000..a355795
--- /dev/null
+++ b/core/Lucy/Search/HitQueue.c
@@ -0,0 +1,187 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HITQUEUE
+#define C_LUCY_MATCHDOC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/HitQueue.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/MatchDoc.h"
+#include "Lucy/Search/SortRule.h"
+#include "Lucy/Search/SortSpec.h"
+
+
+#define COMPARE_BY_SCORE 1
+#define COMPARE_BY_SCORE_REV 2
+#define COMPARE_BY_DOC_ID 3
+#define COMPARE_BY_DOC_ID_REV 4
+#define COMPARE_BY_VALUE 5
+#define COMPARE_BY_VALUE_REV 6
+#define ACTIONS_MASK 0xF
+
+HitQueue*
+HitQ_new(Schema *schema, SortSpec *sort_spec, uint32_t wanted) {
+ HitQueue *self = (HitQueue*)VTable_Make_Obj(HITQUEUE);
+ return HitQ_init(self, schema, sort_spec, wanted);
+}
+
+HitQueue*
+HitQ_init(HitQueue *self, Schema *schema, SortSpec *sort_spec,
+ uint32_t wanted) {
+ if (sort_spec) {
+ uint32_t i;
+ VArray *rules = SortSpec_Get_Rules(sort_spec);
+ uint32_t num_rules = VA_Get_Size(rules);
+ uint32_t action_num = 0;
+
+ if (!schema) {
+ THROW(ERR, "Can't supply sort_spec without schema");
+ }
+
+ self->need_values = false;
+ self->num_actions = num_rules;
+ self->actions = (uint8_t*)MALLOCATE(num_rules * sizeof(uint8_t));
+ self->field_types = (FieldType**)CALLOCATE(num_rules, sizeof(FieldType*));
+
+ for (i = 0; i < num_rules; i++) {
+ SortRule *rule = (SortRule*)VA_Fetch(rules, i);
+ int32_t rule_type = SortRule_Get_Type(rule);
+ bool_t reverse = SortRule_Get_Reverse(rule);
+
+ if (rule_type == SortRule_SCORE) {
+ self->actions[action_num++] = reverse
+ ? COMPARE_BY_SCORE_REV
+ : COMPARE_BY_SCORE;
+ }
+ else if (rule_type == SortRule_DOC_ID) {
+ self->actions[action_num++] = reverse
+ ? COMPARE_BY_DOC_ID_REV
+ : COMPARE_BY_DOC_ID;
+ }
+ else if (rule_type == SortRule_FIELD) {
+ CharBuf *field = SortRule_Get_Field(rule);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ if (type) {
+ self->field_types[action_num] = (FieldType*)INCREF(type);
+ self->actions[action_num++] = reverse
+ ? COMPARE_BY_VALUE_REV
+ : COMPARE_BY_VALUE;
+ self->need_values = true;
+ }
+ else {
+ // Skip over fields we don't know how to sort on.
+ continue;
+ }
+ }
+ else {
+ THROW(ERR, "Unknown SortRule type: %i32", rule_type);
+ }
+ }
+ }
+ else {
+ self->num_actions = 2;
+ self->actions = (uint8_t*)MALLOCATE(self->num_actions * sizeof(uint8_t));
+ self->actions[0] = COMPARE_BY_SCORE;
+ self->actions[1] = COMPARE_BY_DOC_ID;
+ }
+
+ return (HitQueue*)PriQ_init((PriorityQueue*)self, wanted);
+}
+
+void
+HitQ_destroy(HitQueue *self) {
+ FieldType **types = self->field_types;
+ FieldType **const limit = types + self->num_actions - 1;
+ for (; types < limit; types++) {
+ if (types) { DECREF(*types); }
+ }
+ FREEMEM(self->actions);
+ FREEMEM(self->field_types);
+ SUPER_DESTROY(self, HITQUEUE);
+}
+
+Obj*
+HitQ_jostle(HitQueue *self, Obj *element) {
+ MatchDoc *match_doc = (MatchDoc*)CERTIFY(element, MATCHDOC);
+ HitQ_jostle_t super_jostle
+ = (HitQ_jostle_t)SUPER_METHOD(HITQUEUE, HitQ, Jostle);
+ if (self->need_values) {
+ CERTIFY(match_doc->values, VARRAY);
+ }
+ return super_jostle(self, element);
+}
+
+static INLINE int32_t
+SI_compare_by_value(HitQueue *self, uint32_t tick, MatchDoc *a, MatchDoc *b) {
+ Obj *a_val = VA_Fetch(a->values, tick);
+ Obj *b_val = VA_Fetch(b->values, tick);
+ FieldType *field_type = self->field_types[tick];
+ return FType_null_back_compare_values(field_type, a_val, b_val);
+}
+
+bool_t
+HitQ_less_than(HitQueue *self, Obj *obj_a, Obj *obj_b) {
+ MatchDoc *const a = (MatchDoc*)obj_a;
+ MatchDoc *const b = (MatchDoc*)obj_b;
+ uint32_t i = 0;
+ uint8_t *const actions = self->actions;
+
+ do {
+ switch (actions[i] & ACTIONS_MASK) {
+ case COMPARE_BY_SCORE:
+ // Prefer high scores.
+ if (a->score > b->score) { return false; }
+ else if (a->score < b->score) { return true; }
+ break;
+ case COMPARE_BY_SCORE_REV:
+ if (a->score > b->score) { return true; }
+ else if (a->score < b->score) { return false; }
+ break;
+ case COMPARE_BY_DOC_ID:
+ // Prefer low doc ids.
+ if (a->doc_id > b->doc_id) { return true; }
+ else if (a->doc_id < b->doc_id) { return false; }
+ break;
+ case COMPARE_BY_DOC_ID_REV:
+ if (a->doc_id > b->doc_id) { return false; }
+ else if (a->doc_id < b->doc_id) { return true; }
+ break;
+ case COMPARE_BY_VALUE: {
+ int32_t comparison = SI_compare_by_value(self, i, a, b);
+ if (comparison > 0) { return true; }
+ else if (comparison < 0) { return false; }
+ }
+ break;
+ case COMPARE_BY_VALUE_REV: {
+ int32_t comparison = SI_compare_by_value(self, i, b, a);
+ if (comparison > 0) { return true; }
+ else if (comparison < 0) { return false; }
+ }
+ break;
+ default:
+ THROW(ERR, "Unexpected action %u8", actions[i]);
+ }
+
+ } while (++i < self->num_actions);
+
+ return false;
+}
+
+
diff --git a/core/Lucy/Search/HitQueue.cfh b/core/Lucy/Search/HitQueue.cfh
new file mode 100644
index 0000000..d7defc4
--- /dev/null
+++ b/core/Lucy/Search/HitQueue.cfh
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Track highest sorting hits.
+ *
+ * HitQueue sorts MatchDoc objects according to a SortSpec. Good matches
+ * float to the top of the queue and poor matches fall out the bottom.
+ */
+
+class Lucy::Search::HitQueue cnick HitQ
+ inherits Lucy::Util::PriorityQueue {
+
+ FieldType **field_types;
+ uint8_t *actions;
+ uint32_t num_actions;
+ bool_t need_values;
+
+ inert incremented HitQueue*
+ new(Schema *schema = NULL, SortSpec *sort_spec = NULL, uint32_t wanted);
+
+ /**
+ * @param schema A Schema. Required if <code>sort_spec</code> supplied.
+ * @param sort_spec A SortSpec. If not supplied, the HitQueue will sort
+ * by descending score first and ascending doc id second.
+ * @param wanted Max elements the queue can hold.
+ */
+ inert HitQueue*
+ init(HitQueue *self, Schema *schema = NULL, SortSpec *sort_spec = NULL,
+ uint32_t wanted);
+
+ public void
+ Destroy(HitQueue *self);
+
+ /** If sorting on fields, first verifies that the MatchDoc has a valid
+ * values array, then invokes parent method.
+ */
+ incremented nullable Obj*
+ Jostle(HitQueue *self, decremented Obj *element);
+
+ bool_t
+ Less_Than(HitQueue *self, Obj *a, Obj *b);
+}
+
+
diff --git a/core/Lucy/Search/Hits.c b/core/Lucy/Search/Hits.c
new file mode 100644
index 0000000..89008c9
--- /dev/null
+++ b/core/Lucy/Search/Hits.c
@@ -0,0 +1,75 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_HITS
+#define C_LUCY_MATCHDOC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Hits.h"
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/MatchDoc.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/TopDocs.h"
+
+Hits*
+Hits_new(Searcher *searcher, TopDocs *top_docs, uint32_t offset) {
+ Hits *self = (Hits*)VTable_Make_Obj(HITS);
+ return Hits_init(self, searcher, top_docs, offset);
+}
+
+Hits*
+Hits_init(Hits *self, Searcher *searcher, TopDocs *top_docs, uint32_t offset) {
+ self->searcher = (Searcher*)INCREF(searcher);
+ self->top_docs = (TopDocs*)INCREF(top_docs);
+ self->match_docs = (VArray*)INCREF(TopDocs_Get_Match_Docs(top_docs));
+ self->offset = offset;
+ return self;
+}
+
+void
+Hits_destroy(Hits *self) {
+ DECREF(self->searcher);
+ DECREF(self->top_docs);
+ DECREF(self->match_docs);
+ SUPER_DESTROY(self, HITS);
+}
+
+HitDoc*
+Hits_next(Hits *self) {
+ MatchDoc *match_doc = (MatchDoc*)VA_Fetch(self->match_docs, self->offset);
+ self->offset++;
+
+ if (!match_doc) {
+ /** Bail if there aren't any more *captured* hits. (There may be more
+ * total hits.) */
+ return NULL;
+ }
+ else {
+ // Lazily fetch HitDoc, set score.
+ HitDoc *hit_doc = Searcher_Fetch_Doc(self->searcher,
+ match_doc->doc_id);
+ HitDoc_Set_Score(hit_doc, match_doc->score);
+ return hit_doc;
+ }
+}
+
+uint32_t
+Hits_total_hits(Hits *self) {
+ return TopDocs_Get_Total_Hits(self->top_docs);
+}
+
+
diff --git a/core/Lucy/Search/Hits.cfh b/core/Lucy/Search/Hits.cfh
new file mode 100644
index 0000000..5ad49b2
--- /dev/null
+++ b/core/Lucy/Search/Hits.cfh
@@ -0,0 +1,54 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Access search results.
+ *
+ * Hits objects are iterators used to access the results of a search.
+ */
+class Lucy::Search::Hits inherits Lucy::Object::Obj {
+
+ Searcher *searcher;
+ TopDocs *top_docs;
+ VArray *match_docs;
+ uint32_t offset;
+
+ inert incremented Hits*
+ new(Searcher *searcher, TopDocs *top_docs, uint32_t offset = 0);
+
+ inert Hits*
+ init(Hits *self, Searcher *searcher, TopDocs *top_docs,
+ uint32_t offset = 0);
+
+ /** Return the next hit, or NULL when the iterator is exhausted.
+ */
+ public incremented nullable HitDoc*
+ Next(Hits *self);
+
+ /** Return the total number of documents which matched the Query used to
+ * produce the Hits object. Note that this is the total number of
+ * matches, not just the number of matches represented by the Hits
+ * iterator.
+ */
+ public uint32_t
+ Total_Hits(Hits *self);
+
+ public void
+ Destroy(Hits *self);
+}
+
+
diff --git a/core/Lucy/Search/IndexSearcher.c b/core/Lucy/Search/IndexSearcher.c
new file mode 100644
index 0000000..8b7be5a
--- /dev/null
+++ b/core/Lucy/Search/IndexSearcher.c
@@ -0,0 +1,170 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INDEXSEARCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/IndexSearcher.h"
+
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Index/DeletionsReader.h"
+#include "Lucy/Index/DocReader.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/IndexReader.h"
+#include "Lucy/Index/LexiconReader.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Index/HighlightReader.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Collector.h"
+#include "Lucy/Search/Collector/SortCollector.h"
+#include "Lucy/Search/HitQueue.h"
+#include "Lucy/Search/MatchDoc.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/SortRule.h"
+#include "Lucy/Search/SortSpec.h"
+#include "Lucy/Search/TopDocs.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/FSFolder.h"
+
+IndexSearcher*
+IxSearcher_new(Obj *index) {
+ IndexSearcher *self = (IndexSearcher*)VTable_Make_Obj(INDEXSEARCHER);
+ return IxSearcher_init(self, index);
+}
+
+IndexSearcher*
+IxSearcher_init(IndexSearcher *self, Obj *index) {
+ if (Obj_Is_A(index, INDEXREADER)) {
+ self->reader = (IndexReader*)INCREF(index);
+ }
+ else {
+ self->reader = IxReader_open(index, NULL, NULL);
+ }
+ Searcher_init((Searcher*)self, IxReader_Get_Schema(self->reader));
+ self->seg_readers = IxReader_Seg_Readers(self->reader);
+ self->seg_starts = IxReader_Offsets(self->reader);
+ self->doc_reader = (DocReader*)IxReader_Fetch(
+ self->reader, VTable_Get_Name(DOCREADER));
+ self->hl_reader = (HighlightReader*)IxReader_Fetch(
+ self->reader, VTable_Get_Name(HIGHLIGHTREADER));
+ if (self->doc_reader) { INCREF(self->doc_reader); }
+ if (self->hl_reader) { INCREF(self->hl_reader); }
+
+ return self;
+}
+
+void
+IxSearcher_destroy(IndexSearcher *self) {
+ DECREF(self->reader);
+ DECREF(self->doc_reader);
+ DECREF(self->hl_reader);
+ DECREF(self->seg_readers);
+ DECREF(self->seg_starts);
+ SUPER_DESTROY(self, INDEXSEARCHER);
+}
+
+HitDoc*
+IxSearcher_fetch_doc(IndexSearcher *self, int32_t doc_id) {
+ if (!self->doc_reader) { THROW(ERR, "No DocReader"); }
+ return DocReader_Fetch_Doc(self->doc_reader, doc_id);
+}
+
+DocVector*
+IxSearcher_fetch_doc_vec(IndexSearcher *self, int32_t doc_id) {
+ if (!self->hl_reader) { THROW(ERR, "No HighlightReader"); }
+ return HLReader_Fetch_Doc_Vec(self->hl_reader, doc_id);
+}
+
+int32_t
+IxSearcher_doc_max(IndexSearcher *self) {
+ return IxReader_Doc_Max(self->reader);
+}
+
+uint32_t
+IxSearcher_doc_freq(IndexSearcher *self, const CharBuf *field, Obj *term) {
+ LexiconReader *lex_reader
+ = (LexiconReader*)IxReader_Fetch(self->reader,
+ VTable_Get_Name(LEXICONREADER));
+ return lex_reader ? LexReader_Doc_Freq(lex_reader, field, term) : 0;
+}
+
+TopDocs*
+IxSearcher_top_docs(IndexSearcher *self, Query *query, uint32_t num_wanted,
+ SortSpec *sort_spec) {
+ Schema *schema = IxSearcher_Get_Schema(self);
+ uint32_t doc_max = IxSearcher_Doc_Max(self);
+ uint32_t wanted = num_wanted > doc_max ? doc_max : num_wanted;
+ SortCollector *collector = SortColl_new(schema, sort_spec, wanted);
+ IxSearcher_Collect(self, query, (Collector*)collector);
+ {
+ VArray *match_docs = SortColl_Pop_Match_Docs(collector);
+ int32_t total_hits = SortColl_Get_Total_Hits(collector);
+ TopDocs *retval = TopDocs_new(match_docs, total_hits);
+ DECREF(collector);
+ DECREF(match_docs);
+ return retval;
+ }
+}
+
+void
+IxSearcher_collect(IndexSearcher *self, Query *query, Collector *collector) {
+ uint32_t i, max;
+ VArray *const seg_readers = self->seg_readers;
+ I32Array *const seg_starts = self->seg_starts;
+ bool_t need_score = Coll_Need_Score(collector);
+ Compiler *compiler = Query_Is_A(query, COMPILER)
+ ? (Compiler*)INCREF(query)
+ : Query_Make_Compiler(query, (Searcher*)self,
+ Query_Get_Boost(query));
+
+ // Accumulate hits into the Collector.
+ for (i = 0, max = VA_Get_Size(seg_readers); i < max; i++) {
+ SegReader *seg_reader = (SegReader*)VA_Fetch(seg_readers, i);
+ DeletionsReader *del_reader = (DeletionsReader*)SegReader_Fetch(
+ seg_reader,
+ VTable_Get_Name(DELETIONSREADER));
+ Matcher *matcher
+ = Compiler_Make_Matcher(compiler, seg_reader, need_score);
+ if (matcher) {
+ int32_t seg_start = I32Arr_Get(seg_starts, i);
+ Matcher *deletions = DelReader_Iterator(del_reader);
+ Coll_Set_Reader(collector, seg_reader);
+ Coll_Set_Base(collector, seg_start);
+ Coll_Set_Matcher(collector, matcher);
+ Matcher_Collect(matcher, collector, deletions);
+ DECREF(deletions);
+ DECREF(matcher);
+ }
+ }
+
+ DECREF(compiler);
+}
+
+IndexReader*
+IxSearcher_get_reader(IndexSearcher *self) {
+ return self->reader;
+}
+
+void
+IxSearcher_close(IndexSearcher *self) {
+ UNUSED_VAR(self);
+}
+
+
diff --git a/core/Lucy/Search/IndexSearcher.cfh b/core/Lucy/Search/IndexSearcher.cfh
new file mode 100644
index 0000000..f736c52
--- /dev/null
+++ b/core/Lucy/Search/IndexSearcher.cfh
@@ -0,0 +1,78 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Execute searches against a single index.
+ *
+ * Use the IndexSearcher class to perform search queries against an index.
+ * (For searching multiple indexes at once, see
+ * L<PolySearcher|Lucy::Search::PolySearcher>).
+ *
+ * IndexSearchers operate against a single point-in-time view or
+ * L<Snapshot|Lucy::Index::Snapshot> of the index. If an index is
+ * modified, a new IndexSearcher must be opened to access the changes.
+ */
+class Lucy::Search::IndexSearcher cnick IxSearcher
+ inherits Lucy::Search::Searcher {
+
+ IndexReader *reader;
+ DocReader *doc_reader;
+ HighlightReader *hl_reader;
+ VArray *seg_readers;
+ I32Array *seg_starts;
+
+ inert incremented IndexSearcher*
+ new(Obj *index);
+
+ /**
+ * @param index Either a string filepath, a Folder, or an IndexReader.
+ */
+ public inert IndexSearcher*
+ init(IndexSearcher *self, Obj *index);
+
+ public void
+ Destroy(IndexSearcher *self);
+
+ public int32_t
+ Doc_Max(IndexSearcher *self);
+
+ public uint32_t
+ Doc_Freq(IndexSearcher *self, const CharBuf *field, Obj *term);
+
+ public void
+ Collect(IndexSearcher *self, Query *query, Collector *collector);
+
+ incremented TopDocs*
+ Top_Docs(IndexSearcher *self, Query *query, uint32_t num_wanted,
+ SortSpec *sort_spec = NULL);
+
+ public incremented HitDoc*
+ Fetch_Doc(IndexSearcher *self, int32_t doc_id);
+
+ incremented DocVector*
+ Fetch_Doc_Vec(IndexSearcher *self, int32_t doc_id);
+
+ /** Accessor for the object's <code>reader</code> member.
+ */
+ public IndexReader*
+ Get_Reader(IndexSearcher *self);
+
+ void
+ Close(IndexSearcher *self);
+}
+
+
diff --git a/core/Lucy/Search/LeafQuery.c b/core/Lucy/Search/LeafQuery.c
new file mode 100644
index 0000000..6e5f9c5
--- /dev/null
+++ b/core/Lucy/Search/LeafQuery.c
@@ -0,0 +1,114 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LEAFQUERY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/LeafQuery.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+LeafQuery*
+LeafQuery_new(const CharBuf *field, const CharBuf *text) {
+ LeafQuery *self = (LeafQuery*)VTable_Make_Obj(LEAFQUERY);
+ return LeafQuery_init(self, field, text);
+}
+
+LeafQuery*
+LeafQuery_init(LeafQuery *self, const CharBuf *field, const CharBuf *text) {
+ Query_init((Query*)self, 1.0f);
+ self->field = field ? CB_Clone(field) : NULL;
+ self->text = CB_Clone(text);
+ return self;
+}
+
+void
+LeafQuery_destroy(LeafQuery *self) {
+ DECREF(self->field);
+ DECREF(self->text);
+ SUPER_DESTROY(self, LEAFQUERY);
+}
+
+CharBuf*
+LeafQuery_get_field(LeafQuery *self) {
+ return self->field;
+}
+
+CharBuf*
+LeafQuery_get_text(LeafQuery *self) {
+ return self->text;
+}
+
+bool_t
+LeafQuery_equals(LeafQuery *self, Obj *other) {
+ LeafQuery *twin = (LeafQuery*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, LEAFQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!!self->field ^ !!twin->field) { return false; }
+ if (self->field) {
+ if (!CB_Equals(self->field, (Obj*)twin->field)) { return false; }
+ }
+ if (!CB_Equals(self->text, (Obj*)twin->text)) { return false; }
+ return true;
+}
+
+CharBuf*
+LeafQuery_to_string(LeafQuery *self) {
+ if (self->field) {
+ return CB_newf("%o:%o", self->field, self->text);
+ }
+ else {
+ return CB_Clone(self->text);
+ }
+}
+
+void
+LeafQuery_serialize(LeafQuery *self, OutStream *outstream) {
+ if (self->field) {
+ OutStream_Write_U8(outstream, true);
+ CB_Serialize(self->field, outstream);
+ }
+ else {
+ OutStream_Write_U8(outstream, false);
+ }
+ CB_Serialize(self->text, outstream);
+ OutStream_Write_F32(outstream, self->boost);
+}
+
+LeafQuery*
+LeafQuery_deserialize(LeafQuery *self, InStream *instream) {
+ self = self ? self : (LeafQuery*)VTable_Make_Obj(LEAFQUERY);
+ self->field = InStream_Read_U8(instream)
+ ? CB_deserialize(NULL, instream)
+ : NULL;
+ self->text = CB_deserialize(NULL, instream);
+ self->boost = InStream_Read_F32(instream);
+ return self;
+}
+
+Compiler*
+LeafQuery_make_compiler(LeafQuery *self, Searcher *searcher, float boost) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(searcher);
+ UNUSED_VAR(boost);
+ THROW(ERR, "Can't Make_Compiler() from LeafQuery");
+ UNREACHABLE_RETURN(Compiler*);
+}
+
+
diff --git a/core/Lucy/Search/LeafQuery.cfh b/core/Lucy/Search/LeafQuery.cfh
new file mode 100644
index 0000000..1c56632
--- /dev/null
+++ b/core/Lucy/Search/LeafQuery.cfh
@@ -0,0 +1,75 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Leaf node in a tree created by QueryParser.
+ *
+ * LeafQuery objects serve as leaf nodes in the tree structure generated by
+ * L<QueryParser|Lucy::Search::QueryParser>'s Tree() method.
+ * Ultimately, they must be transformed, typically into either
+ * L<TermQuery|Lucy::Search::TermQuery> or
+ * L<PhraseQuery|Lucy::Search::PhraseQuery> objects, as attempting to
+ * search a LeafQuery causes an error.
+ */
+class Lucy::Search::LeafQuery inherits Lucy::Search::Query
+ : dumpable {
+
+ CharBuf *field;
+ CharBuf *text;
+
+ inert incremented LeafQuery*
+ new(const CharBuf *field = NULL, const CharBuf *text);
+
+ /**
+ * @param field Optional field name.
+ * @param text Raw query text.
+ */
+ public inert LeafQuery*
+ init(LeafQuery *self, const CharBuf *field = NULL, const CharBuf *text);
+
+ /** Accessor for object's <code>field</code> attribute.
+ */
+ public nullable CharBuf*
+ Get_Field(LeafQuery *self);
+
+ /** Accessor for object's <code>text</code> attribute.
+ */
+ public CharBuf*
+ Get_Text(LeafQuery *self);
+
+ public bool_t
+ Equals(LeafQuery *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(LeafQuery *self);
+
+ public void
+ Serialize(LeafQuery *self, OutStream *outstream);
+
+ public incremented LeafQuery*
+ Deserialize(LeafQuery *self, InStream *instream);
+
+ /** Throws an error.
+ */
+ public incremented Compiler*
+ Make_Compiler(LeafQuery *self, Searcher *searcher, float boost);
+
+ public void
+ Destroy(LeafQuery *self);
+}
+
+
diff --git a/core/Lucy/Search/MatchAllMatcher.c b/core/Lucy/Search/MatchAllMatcher.c
new file mode 100644
index 0000000..815443d
--- /dev/null
+++ b/core/Lucy/Search/MatchAllMatcher.c
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MATCHALLMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/MatchAllMatcher.h"
+
+MatchAllMatcher*
+MatchAllMatcher_new(float score, int32_t doc_max) {
+ MatchAllMatcher *self
+ = (MatchAllMatcher*)VTable_Make_Obj(MATCHALLMATCHER);
+ return MatchAllMatcher_init(self, score, doc_max);
+}
+
+MatchAllMatcher*
+MatchAllMatcher_init(MatchAllMatcher *self, float score, int32_t doc_max) {
+ Matcher_init((Matcher*)self);
+ self->doc_id = 0;
+ self->score = score;
+ self->doc_max = doc_max;
+ return self;
+}
+
+int32_t
+MatchAllMatcher_next(MatchAllMatcher* self) {
+ if (++self->doc_id <= self->doc_max) {
+ return self->doc_id;
+ }
+ else {
+ self->doc_id--;
+ return 0;
+ }
+}
+
+int32_t
+MatchAllMatcher_advance(MatchAllMatcher* self, int32_t target) {
+ self->doc_id = target - 1;
+ return MatchAllMatcher_next(self);
+}
+
+float
+MatchAllMatcher_score(MatchAllMatcher* self) {
+ return self->score;
+}
+
+int32_t
+MatchAllMatcher_get_doc_id(MatchAllMatcher* self) {
+ return self->doc_id;
+}
+
+
diff --git a/core/Lucy/Search/MatchAllMatcher.cfh b/core/Lucy/Search/MatchAllMatcher.cfh
new file mode 100644
index 0000000..a64a8be
--- /dev/null
+++ b/core/Lucy/Search/MatchAllMatcher.cfh
@@ -0,0 +1,48 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Search::MatchAllMatcher inherits Lucy::Search::Matcher {
+
+ int32_t doc_id;
+ int32_t doc_max;
+ float score;
+
+ /**
+ * @param score Fixed score to be added to each matching document's score.
+ * @param reader An IndexReader.
+ */
+ inert incremented MatchAllMatcher*
+ new(float score, int32_t doc_max);
+
+ inert MatchAllMatcher*
+ init(MatchAllMatcher *self, float score, int32_t doc_max);
+
+ public int32_t
+ Next(MatchAllMatcher* self);
+
+ public int32_t
+ Advance(MatchAllMatcher* self, int32_t target);
+
+ public float
+ Score(MatchAllMatcher* self);
+
+ public int32_t
+ Get_Doc_ID(MatchAllMatcher* self);
+}
+
+
diff --git a/core/Lucy/Search/MatchAllQuery.c b/core/Lucy/Search/MatchAllQuery.c
new file mode 100644
index 0000000..e786153
--- /dev/null
+++ b/core/Lucy/Search/MatchAllQuery.c
@@ -0,0 +1,96 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MATCHALLQUERY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/MatchAllQuery.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Search/MatchAllMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+MatchAllQuery*
+MatchAllQuery_new() {
+ MatchAllQuery *self = (MatchAllQuery*)VTable_Make_Obj(MATCHALLQUERY);
+ return MatchAllQuery_init(self);
+}
+
+MatchAllQuery*
+MatchAllQuery_init(MatchAllQuery *self) {
+ return (MatchAllQuery*)Query_init((Query*)self, 0.0f);
+}
+
+bool_t
+MatchAllQuery_equals(MatchAllQuery *self, Obj *other) {
+ MatchAllQuery *twin = (MatchAllQuery*)other;
+ if (!Obj_Is_A(other, MATCHALLQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ return true;
+}
+
+CharBuf*
+MatchAllQuery_to_string(MatchAllQuery *self) {
+ UNUSED_VAR(self);
+ return CB_new_from_trusted_utf8("[MATCHALL]", 10);
+}
+
+Compiler*
+MatchAllQuery_make_compiler(MatchAllQuery *self, Searcher *searcher,
+ float boost) {
+ return (Compiler*)MatchAllCompiler_new(self, searcher, boost);
+}
+
+/**********************************************************************/
+
+MatchAllCompiler*
+MatchAllCompiler_new(MatchAllQuery *parent, Searcher *searcher,
+ float boost) {
+ MatchAllCompiler *self
+ = (MatchAllCompiler*)VTable_Make_Obj(MATCHALLCOMPILER);
+ return MatchAllCompiler_init(self, parent, searcher, boost);
+}
+
+MatchAllCompiler*
+MatchAllCompiler_init(MatchAllCompiler *self, MatchAllQuery *parent,
+ Searcher *searcher, float boost) {
+ return (MatchAllCompiler*)Compiler_init((Compiler*)self, (Query*)parent,
+ searcher, NULL, boost);
+}
+
+MatchAllCompiler*
+MatchAllCompiler_deserialize(MatchAllCompiler *self, InStream *instream) {
+ self = self
+ ? self
+ : (MatchAllCompiler*)VTable_Make_Obj(MATCHALLCOMPILER);
+ return (MatchAllCompiler*)Compiler_deserialize((Compiler*)self, instream);
+}
+
+Matcher*
+MatchAllCompiler_make_matcher(MatchAllCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ float weight = MatchAllCompiler_Get_Weight(self);
+ UNUSED_VAR(need_score);
+ return (Matcher*)MatchAllMatcher_new(weight, SegReader_Doc_Max(reader));
+}
+
+
diff --git a/core/Lucy/Search/MatchAllQuery.cfh b/core/Lucy/Search/MatchAllQuery.cfh
new file mode 100644
index 0000000..67d36c1
--- /dev/null
+++ b/core/Lucy/Search/MatchAllQuery.cfh
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Query which matches all documents.
+ *
+ * MatchAllQuery is a utility class which matches all documents. Each match
+ * is assigned a score of 0.0, so that in composite queries, any document
+ * which matches against another part of the query will be ranked higher than
+ * a document which matches only via the MatchAllQuery.
+ */
+abstract class Lucy::Search::MatchAllQuery
+ inherits Lucy::Search::Query : dumpable {
+
+ inert incremented MatchAllQuery*
+ new();
+
+ /** Constructor. Takes no arguments.
+ */
+ public inert MatchAllQuery*
+ init(MatchAllQuery *self);
+
+ public bool_t
+ Equals(MatchAllQuery *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(MatchAllQuery *self);
+
+ public incremented Compiler*
+ Make_Compiler(MatchAllQuery *self, Searcher *searcher, float boost);
+}
+
+class Lucy::Search::MatchAllCompiler
+ inherits Lucy::Search::Compiler {
+
+ inert incremented MatchAllCompiler*
+ new(MatchAllQuery *parent, Searcher *searcher, float boost);
+
+ inert MatchAllCompiler*
+ init(MatchAllCompiler *self, MatchAllQuery *parent,
+ Searcher *searcher, float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(MatchAllCompiler *self, SegReader *reader,
+ bool_t need_score);
+
+ public incremented MatchAllCompiler*
+ Deserialize(MatchAllCompiler *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/MatchDoc.c b/core/Lucy/Search/MatchDoc.c
new file mode 100644
index 0000000..279d2d4
--- /dev/null
+++ b/core/Lucy/Search/MatchDoc.c
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MATCHDOC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/MatchDoc.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+MatchDoc*
+MatchDoc_new(int32_t doc_id, float score, VArray *values) {
+ MatchDoc *self = (MatchDoc*)VTable_Make_Obj(MATCHDOC);
+ return MatchDoc_init(self, doc_id, score, values);
+}
+
+MatchDoc*
+MatchDoc_init(MatchDoc *self, int32_t doc_id, float score, VArray *values) {
+ self->doc_id = doc_id;
+ self->score = score;
+ self->values = (VArray*)INCREF(values);
+ return self;
+}
+
+void
+MatchDoc_destroy(MatchDoc *self) {
+ DECREF(self->values);
+ SUPER_DESTROY(self, MATCHDOC);
+}
+
+void
+MatchDoc_serialize(MatchDoc *self, OutStream *outstream) {
+ OutStream_Write_C32(outstream, self->doc_id);
+ OutStream_Write_F32(outstream, self->score);
+ OutStream_Write_U8(outstream, self->values ? 1 : 0);
+ if (self->values) { VA_Serialize(self->values, outstream); }
+}
+
+MatchDoc*
+MatchDoc_deserialize(MatchDoc *self, InStream *instream) {
+ self = self ? self : (MatchDoc*)VTable_Make_Obj(MATCHDOC);
+ self->doc_id = InStream_Read_C32(instream);
+ self->score = InStream_Read_F32(instream);
+ if (InStream_Read_U8(instream)) {
+ self->values = VA_deserialize(NULL, instream);
+ }
+ return self;
+}
+
+int32_t
+MatchDoc_get_doc_id(MatchDoc *self) {
+ return self->doc_id;
+}
+
+float
+MatchDoc_get_score(MatchDoc *self) {
+ return self->score;
+}
+
+VArray*
+MatchDoc_get_values(MatchDoc *self) {
+ return self->values;
+}
+
+void
+MatchDoc_set_doc_id(MatchDoc *self, int32_t doc_id) {
+ self->doc_id = doc_id;
+}
+
+void
+MatchDoc_set_score(MatchDoc *self, float score) {
+ self->score = score;
+}
+
+void
+MatchDoc_set_values(MatchDoc *self, VArray *values) {
+ DECREF(self->values);
+ self->values = (VArray*)INCREF(values);
+}
+
+
diff --git a/core/Lucy/Search/MatchDoc.cfh b/core/Lucy/Search/MatchDoc.cfh
new file mode 100644
index 0000000..b860c68
--- /dev/null
+++ b/core/Lucy/Search/MatchDoc.cfh
@@ -0,0 +1,62 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Representation of a document being evaluated as a hit.
+ */
+
+class Lucy::Search::MatchDoc inherits Lucy::Object::Obj {
+
+ int32_t doc_id;
+ float score;
+ VArray *values;
+
+ inert incremented MatchDoc*
+ new(int32_t doc_id, float score, VArray *values = NULL);
+
+ inert MatchDoc*
+ init(MatchDoc *self, int32_t doc_id, float score, VArray *values = NULL);
+
+ public void
+ Serialize(MatchDoc *self, OutStream *outstream);
+
+ public incremented MatchDoc*
+ Deserialize(MatchDoc *self, InStream *instream);
+
+ int32_t
+ Get_Doc_ID(MatchDoc *self);
+
+ void
+ Set_Doc_ID(MatchDoc *self, int32_t doc_id);
+
+ float
+ Get_Score(MatchDoc *self);
+
+ void
+ Set_Score(MatchDoc *self, float score);
+
+ nullable VArray*
+ Get_Values(MatchDoc *self);
+
+ void
+ Set_Values(MatchDoc *self, VArray *values);
+
+ public void
+ Destroy(MatchDoc *self);
+}
+
+
diff --git a/core/Lucy/Search/Matcher.c b/core/Lucy/Search/Matcher.c
new file mode 100644
index 0000000..c2a8c31
--- /dev/null
+++ b/core/Lucy/Search/Matcher.c
@@ -0,0 +1,91 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MATCHER
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Search/Collector.h"
+
+Matcher*
+Matcher_init(Matcher *self) {
+ ABSTRACT_CLASS_CHECK(self, MATCHER);
+ return self;
+}
+
+int32_t
+Matcher_advance(Matcher *self, int32_t target) {
+ while (1) {
+ int32_t doc_id = Matcher_Next(self);
+ if (doc_id == 0 || doc_id >= target) {
+ return doc_id;
+ }
+ }
+}
+
+void
+Matcher_collect(Matcher *self, Collector *collector, Matcher *deletions) {
+ int32_t doc_id = 0;
+ int32_t next_deletion = deletions ? 0 : I32_MAX;
+
+ Coll_Set_Matcher(collector, self);
+
+ // Execute scoring loop.
+ while (1) {
+ if (doc_id > next_deletion) {
+ next_deletion = Matcher_Advance(deletions, doc_id);
+ if (next_deletion == 0) { next_deletion = I32_MAX; }
+ continue;
+ }
+ else if (doc_id == next_deletion) {
+ // Skip past deletions.
+ while (doc_id == next_deletion) {
+ // Artifically advance matcher.
+ while (doc_id == next_deletion) {
+ doc_id++;
+ next_deletion = Matcher_Advance(deletions, doc_id);
+ if (next_deletion == 0) { next_deletion = I32_MAX; }
+ }
+ // Verify that the artificial advance actually worked.
+ doc_id = Matcher_Advance(self, doc_id);
+ if (doc_id > next_deletion) {
+ next_deletion = Matcher_Advance(deletions, doc_id);
+ }
+ }
+ }
+ else {
+ doc_id = Matcher_Advance(self, doc_id + 1);
+ if (doc_id >= next_deletion) {
+ next_deletion = Matcher_Advance(deletions, doc_id);
+ if (doc_id == next_deletion) { continue; }
+ }
+ }
+
+ if (doc_id) {
+ Coll_Collect(collector, doc_id);
+ }
+ else {
+ break;
+ }
+ }
+
+ Coll_Set_Matcher(collector, NULL);
+}
+
+
diff --git a/core/Lucy/Search/Matcher.cfh b/core/Lucy/Search/Matcher.cfh
new file mode 100644
index 0000000..ac9d91d
--- /dev/null
+++ b/core/Lucy/Search/Matcher.cfh
@@ -0,0 +1,75 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Match a set of document ids.
+ *
+ * A Matcher iterates over a set of ascending document ids. Some Matchers
+ * implement Score() and can assign relevance scores to the docs that they
+ * match. Other implementations may be match-only.
+ */
+
+abstract class Lucy::Search::Matcher inherits Lucy::Object::Obj {
+
+ /** Abstract constructor.
+ */
+ public inert Matcher*
+ init(Matcher* self);
+
+ /** Proceed to the next doc id.
+ *
+ * @return A positive doc id, or 0 once the iterator is exhausted.
+ */
+ public abstract int32_t
+ Next(Matcher *self);
+
+ /** Advance the iterator to the first doc id greater than or equal to
+ * <code>target</code>. The default implementation simply calls Next()
+ * over and over, but subclasses have the option of doing something more
+ * efficient.
+ *
+ * @param target A positive doc id, which must be greater than the current
+ * doc id once the iterator has been initialized.
+ * @return A positive doc id, or 0 once the iterator is exhausted.
+ */
+ public int32_t
+ Advance(Matcher *self, int32_t target);
+
+ /** Return the current doc id. Valid only after a successful call to
+ * Next() or Advance() and must not be called otherwise.
+ */
+ public abstract int32_t
+ Get_Doc_ID(Matcher *self);
+
+ /** Return the score of the current document.
+ *
+ * Only Matchers which are used for scored search need implement Score().
+ */
+ public abstract float
+ Score(Matcher *self);
+
+ /** Collect hits.
+ *
+ * @param collector The Collector to collect hits with.
+ * @param deletions A deletions iterator.
+ */
+ void
+ Collect(Matcher *self, Collector *collector,
+ Matcher *deletions = NULL);
+}
+
+
diff --git a/core/Lucy/Search/NOTMatcher.c b/core/Lucy/Search/NOTMatcher.c
new file mode 100644
index 0000000..25208cd
--- /dev/null
+++ b/core/Lucy/Search/NOTMatcher.c
@@ -0,0 +1,99 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NOTMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/NOTMatcher.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+
+NOTMatcher*
+NOTMatcher_new(Matcher *negated_matcher, int32_t doc_max) {
+ NOTMatcher *self = (NOTMatcher*)VTable_Make_Obj(NOTMATCHER);
+ return NOTMatcher_init(self, negated_matcher, doc_max);
+}
+
+NOTMatcher*
+NOTMatcher_init(NOTMatcher *self, Matcher *negated_matcher, int32_t doc_max) {
+ VArray *children = VA_new(1);
+ VA_Push(children, INCREF(negated_matcher));
+ PolyMatcher_init((PolyMatcher*)self, children, NULL);
+
+ // Init.
+ self->doc_id = 0;
+ self->next_negation = 0;
+
+ // Assign.
+ self->negated_matcher = (Matcher*)INCREF(negated_matcher);
+ self->doc_max = doc_max;
+
+ DECREF(children);
+
+ return self;
+}
+
+void
+NOTMatcher_destroy(NOTMatcher *self) {
+ DECREF(self->negated_matcher);
+ SUPER_DESTROY(self, NOTMATCHER);
+}
+
+int32_t
+NOTMatcher_next(NOTMatcher *self) {
+ while (1) {
+ self->doc_id++;
+
+ // Get next negated doc id.
+ if (self->next_negation < self->doc_id) {
+ self->next_negation
+ = Matcher_Advance(self->negated_matcher, self->doc_id);
+ if (self->next_negation == 0) {
+ DECREF(self->negated_matcher);
+ self->negated_matcher = NULL;
+ self->next_negation = self->doc_max + 1;
+ }
+ }
+
+ if (self->doc_id > self->doc_max) {
+ self->doc_id = self->doc_max; // halt advance
+ return 0;
+ }
+ else if (self->doc_id != self->next_negation) {
+ // Success!
+ return self->doc_id;
+ }
+ }
+}
+
+int32_t
+NOTMatcher_advance(NOTMatcher *self, int32_t target) {
+ self->doc_id = target - 1;
+ return NOTMatcher_next(self);
+}
+
+int32_t
+NOTMatcher_get_doc_id(NOTMatcher *self) {
+ return self->doc_id;
+}
+
+float
+NOTMatcher_score(NOTMatcher *self) {
+ UNUSED_VAR(self);
+ return 0.0f;
+}
+
+
diff --git a/core/Lucy/Search/NOTMatcher.cfh b/core/Lucy/Search/NOTMatcher.cfh
new file mode 100644
index 0000000..4e8feca
--- /dev/null
+++ b/core/Lucy/Search/NOTMatcher.cfh
@@ -0,0 +1,51 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Return the inverse of a Matcher's set. Scores are always 0.
+ */
+
+class Lucy::Search::NOTMatcher inherits Lucy::Search::PolyMatcher {
+
+ Matcher *negated_matcher;
+ int32_t doc_id;
+ int32_t doc_max;
+ int32_t next_negation;
+
+ inert incremented NOTMatcher*
+ new(Matcher* negated_matcher, int32_t doc_max);
+
+ inert NOTMatcher*
+ init(NOTMatcher *self, Matcher *negated_matcher, int32_t doc_max);
+
+ public void
+ Destroy(NOTMatcher *self);
+
+ public int32_t
+ Next(NOTMatcher *self);
+
+ public int32_t
+ Advance(NOTMatcher *self, int32_t target);
+
+ public float
+ Score(NOTMatcher *self);
+
+ public int32_t
+ Get_Doc_ID(NOTMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/NOTQuery.c b/core/Lucy/Search/NOTQuery.c
new file mode 100644
index 0000000..d22be85
--- /dev/null
+++ b/core/Lucy/Search/NOTQuery.c
@@ -0,0 +1,133 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NOTQUERY
+#define C_LUCY_NOTCOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Search/MatchAllMatcher.h"
+#include "Lucy/Search/NOTMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+NOTQuery*
+NOTQuery_new(Query *negated_query) {
+ NOTQuery *self = (NOTQuery*)VTable_Make_Obj(NOTQUERY);
+ return NOTQuery_init(self, negated_query);
+}
+
+NOTQuery*
+NOTQuery_init(NOTQuery *self, Query *negated_query) {
+ self = (NOTQuery*)PolyQuery_init((PolyQuery*)self, NULL);
+ NOTQuery_Set_Boost(self, 0.0f);
+ PolyQuery_add_child((PolyQuery*)self, negated_query);
+ return self;
+}
+
+Query*
+NOTQuery_get_negated_query(NOTQuery *self) {
+ return (Query*)VA_Fetch(self->children, 0);
+}
+
+void
+NOTQuery_set_negated_query(NOTQuery *self, Query *negated_query) {
+ VA_Store(self->children, 0, INCREF(negated_query));
+}
+
+CharBuf*
+NOTQuery_to_string(NOTQuery *self) {
+ CharBuf *neg_string = Obj_To_String(VA_Fetch(self->children, 0));
+ CharBuf *retval = CB_newf("-%o", neg_string);
+ DECREF(neg_string);
+ return retval;
+}
+
+bool_t
+NOTQuery_equals(NOTQuery *self, Obj *other) {
+ if ((NOTQuery*)other == self) { return true; }
+ if (!Obj_Is_A(other, NOTQUERY)) { return false; }
+ return PolyQuery_equals((PolyQuery*)self, other);
+}
+
+Compiler*
+NOTQuery_make_compiler(NOTQuery *self, Searcher *searcher, float boost) {
+ return (Compiler*)NOTCompiler_new(self, searcher, boost);
+}
+
+/**********************************************************************/
+
+NOTCompiler*
+NOTCompiler_new(NOTQuery *parent, Searcher *searcher, float boost) {
+ NOTCompiler *self = (NOTCompiler*)VTable_Make_Obj(NOTCOMPILER);
+ return NOTCompiler_init(self, parent, searcher, boost);
+}
+
+NOTCompiler*
+NOTCompiler_init(NOTCompiler *self, NOTQuery *parent, Searcher *searcher,
+ float boost) {
+ PolyCompiler_init((PolyCompiler*)self, (PolyQuery*)parent, searcher,
+ boost);
+ NOTCompiler_Normalize(self);
+ return self;
+}
+
+float
+NOTCompiler_sum_of_squared_weights(NOTCompiler *self) {
+ UNUSED_VAR(self);
+ return 0.0f;
+}
+
+VArray*
+NOTCompiler_highlight_spans(NOTCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(searcher);
+ UNUSED_VAR(doc_vec);
+ UNUSED_VAR(field);
+ return VA_new(0);
+}
+
+Matcher*
+NOTCompiler_make_matcher(NOTCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ Compiler *negated_compiler
+ = (Compiler*)CERTIFY(VA_Fetch(self->children, 0), COMPILER);
+ Matcher *negated_matcher
+ = Compiler_Make_Matcher(negated_compiler, reader, false);
+ UNUSED_VAR(need_score);
+
+ if (negated_matcher == NULL) {
+ float weight = NOTCompiler_Get_Weight(self);
+ int32_t doc_max = SegReader_Doc_Max(reader);
+ return (Matcher*)MatchAllMatcher_new(weight, doc_max);
+ }
+ else if (Obj_Is_A((Obj*)negated_matcher, MATCHALLMATCHER)) {
+ DECREF(negated_matcher);
+ return NULL;
+ }
+ else {
+ int32_t doc_max = SegReader_Doc_Max(reader);
+ Matcher *retval = (Matcher*)NOTMatcher_new(negated_matcher, doc_max);
+ DECREF(negated_matcher);
+ return retval;
+ }
+}
+
+
diff --git a/core/Lucy/Search/NOTQuery.cfh b/core/Lucy/Search/NOTQuery.cfh
new file mode 100644
index 0000000..ee1109c
--- /dev/null
+++ b/core/Lucy/Search/NOTQuery.cfh
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Invert the result set of another Query.
+ *
+ * A NOTQuery wraps another L<Query|Lucy::Search::Query> and matches
+ * against its inverse document set. All matching docs recieve a score of
+ * 0.0.
+ *
+ * NOTQuery is often used in conjunction with
+ * L<ANDQuery|Lucy::Search::ANDQuery> to provide "a AND NOT b"
+ * semantics.
+ */
+
+class Lucy::Search::NOTQuery inherits Lucy::Search::PolyQuery
+ : dumpable {
+
+ /**
+ * @param negated_query The Query to be inverted.
+ */
+ inert incremented NOTQuery*
+ new(Query *negated_query);
+
+ /**
+ * @param negated_query The Query whose result set should be inverted.
+ */
+ public inert NOTQuery*
+ init(NOTQuery *self, Query *negated_query);
+
+ /** Accessor for the object's negated query. */
+ public Query*
+ Get_Negated_Query(NOTQuery *self);
+
+ /** Setter for the object's negated query. */
+ public void
+ Set_Negated_Query(NOTQuery *self, Query *negated_query);
+
+ public incremented Compiler*
+ Make_Compiler(NOTQuery *self, Searcher *searcher, float boost);
+
+ public incremented CharBuf*
+ To_String(NOTQuery *self);
+
+ public bool_t
+ Equals(NOTQuery *self, Obj *other);
+}
+
+class Lucy::Search::NOTCompiler
+ inherits Lucy::Search::PolyCompiler {
+
+ inert incremented NOTCompiler*
+ new(NOTQuery *parent, Searcher *searcher, float boost);
+
+ inert NOTCompiler*
+ init(NOTCompiler *self, NOTQuery *parent, Searcher *searcher,
+ float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(NOTCompiler *self, SegReader *reader, bool_t need_score);
+
+ public float
+ Sum_Of_Squared_Weights(NOTCompiler *self);
+
+ public incremented VArray*
+ Highlight_Spans(NOTCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field);
+}
+
+
diff --git a/core/Lucy/Search/NoMatchMatcher.c b/core/Lucy/Search/NoMatchMatcher.c
new file mode 100644
index 0000000..53375a8
--- /dev/null
+++ b/core/Lucy/Search/NoMatchMatcher.c
@@ -0,0 +1,49 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NOMATCHMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/NoMatchMatcher.h"
+#include "Lucy/Index/IndexReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+
+NoMatchMatcher*
+NoMatchMatcher_new() {
+ NoMatchMatcher *self = (NoMatchMatcher*)VTable_Make_Obj(NOMATCHMATCHER);
+ return NoMatchMatcher_init(self);
+}
+
+NoMatchMatcher*
+NoMatchMatcher_init(NoMatchMatcher *self) {
+ return (NoMatchMatcher*)Matcher_init((Matcher*)self);
+}
+
+int32_t
+NoMatchMatcher_next(NoMatchMatcher* self) {
+ UNUSED_VAR(self);
+ return 0;
+}
+
+int32_t
+NoMatchMatcher_advance(NoMatchMatcher* self, int32_t target) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(target);
+ return 0;
+}
+
+
diff --git a/core/Lucy/Search/NoMatchMatcher.cfh b/core/Lucy/Search/NoMatchMatcher.cfh
new file mode 100644
index 0000000..1014418
--- /dev/null
+++ b/core/Lucy/Search/NoMatchMatcher.cfh
@@ -0,0 +1,34 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Search::NoMatchMatcher inherits Lucy::Search::Matcher {
+
+ inert incremented NoMatchMatcher*
+ new();
+
+ inert NoMatchMatcher*
+ init(NoMatchMatcher *self);
+
+ public int32_t
+ Next(NoMatchMatcher* self);
+
+ public int32_t
+ Advance(NoMatchMatcher* self, int32_t target);
+}
+
+
diff --git a/core/Lucy/Search/NoMatchQuery.c b/core/Lucy/Search/NoMatchQuery.c
new file mode 100644
index 0000000..eac2fb1
--- /dev/null
+++ b/core/Lucy/Search/NoMatchQuery.c
@@ -0,0 +1,145 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NOMATCHQUERY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/NoMatchQuery.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/NoMatchMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+NoMatchQuery*
+NoMatchQuery_new() {
+ NoMatchQuery *self = (NoMatchQuery*)VTable_Make_Obj(NOMATCHQUERY);
+ return NoMatchQuery_init(self);
+}
+
+NoMatchQuery*
+NoMatchQuery_init(NoMatchQuery *self) {
+ Query_init((Query*)self, 0.0f);
+ self->fails_to_match = true;
+ return self;
+}
+
+bool_t
+NoMatchQuery_equals(NoMatchQuery *self, Obj *other) {
+ NoMatchQuery *twin = (NoMatchQuery*)other;
+ if (!Obj_Is_A(other, NOMATCHQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!!self->fails_to_match != !!twin->fails_to_match) { return false; }
+ return true;
+}
+
+CharBuf*
+NoMatchQuery_to_string(NoMatchQuery *self) {
+ UNUSED_VAR(self);
+ return CB_new_from_trusted_utf8("[NOMATCH]", 9);
+}
+
+Compiler*
+NoMatchQuery_make_compiler(NoMatchQuery *self, Searcher *searcher,
+ float boost) {
+ return (Compiler*)NoMatchCompiler_new(self, searcher, boost);
+}
+
+void
+NoMatchQuery_set_fails_to_match(NoMatchQuery *self, bool_t fails_to_match) {
+ self->fails_to_match = fails_to_match;
+}
+
+bool_t
+NoMatchQuery_get_fails_to_match(NoMatchQuery *self) {
+ return self->fails_to_match;
+}
+
+Obj*
+NoMatchQuery_dump(NoMatchQuery *self) {
+ NoMatchQuery_dump_t super_dump
+ = (NoMatchQuery_dump_t)SUPER_METHOD(NOMATCHQUERY, NoMatchQuery, Dump);
+ Hash *dump = (Hash*)CERTIFY(super_dump(self), HASH);
+ Hash_Store_Str(dump, "fails_to_match", 14,
+ (Obj*)CB_newf("%i64", (int64_t)self->fails_to_match));
+ return (Obj*)dump;
+}
+
+NoMatchQuery*
+NoMatchQuery_load(NoMatchQuery *self, Obj *dump) {
+ Hash *source = (Hash*)CERTIFY(dump, HASH);
+ NoMatchQuery_load_t super_load
+ = (NoMatchQuery_load_t)SUPER_METHOD(NOMATCHQUERY, NoMatchQuery, Load);
+ NoMatchQuery *loaded = super_load(self, dump);
+ Obj *fails = Cfish_Hash_Fetch_Str(source, "fails_to_match", 14);
+ if (fails) {
+ loaded->fails_to_match = (bool_t)!!Obj_To_I64(fails);
+ }
+ else {
+ loaded->fails_to_match = true;
+ }
+ return loaded;
+}
+
+void
+NoMatchQuery_serialize(NoMatchQuery *self, OutStream *outstream) {
+ OutStream_Write_I8(outstream, !!self->fails_to_match);
+}
+
+NoMatchQuery*
+NoMatchQuery_deserialize(NoMatchQuery *self, InStream *instream) {
+ self = self ? self : (NoMatchQuery*)VTable_Make_Obj(NOMATCHQUERY);
+ NoMatchQuery_init(self);
+ self->fails_to_match = !!InStream_Read_I8(instream);
+ return self;
+}
+
+/**********************************************************************/
+
+NoMatchCompiler*
+NoMatchCompiler_new(NoMatchQuery *parent, Searcher *searcher,
+ float boost) {
+ NoMatchCompiler *self
+ = (NoMatchCompiler*)VTable_Make_Obj(NOMATCHCOMPILER);
+ return NoMatchCompiler_init(self, parent, searcher, boost);
+}
+
+NoMatchCompiler*
+NoMatchCompiler_init(NoMatchCompiler *self, NoMatchQuery *parent,
+ Searcher *searcher, float boost) {
+ return (NoMatchCompiler*)Compiler_init((Compiler*)self, (Query*)parent,
+ searcher, NULL, boost);
+}
+
+NoMatchCompiler*
+NoMatchCompiler_deserialize(NoMatchCompiler *self, InStream *instream) {
+ self = self ? self : (NoMatchCompiler*)VTable_Make_Obj(NOMATCHCOMPILER);
+ return (NoMatchCompiler*)Compiler_deserialize((Compiler*)self, instream);
+}
+
+Matcher*
+NoMatchCompiler_make_matcher(NoMatchCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(reader);
+ UNUSED_VAR(need_score);
+ return (Matcher*)NoMatchMatcher_new();
+}
+
+
diff --git a/core/Lucy/Search/NoMatchQuery.cfh b/core/Lucy/Search/NoMatchQuery.cfh
new file mode 100644
index 0000000..0e00626
--- /dev/null
+++ b/core/Lucy/Search/NoMatchQuery.cfh
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Query which matches no documents.
+ *
+ * NoMatchQuery is a utility class representing a query which matches nothing.
+ * Typical usage might include e.g. returning a NoMatchQuery when a
+ * L<QueryParser|Lucy::Search::QueryParser> is asked to parse an empty
+ * string.
+ */
+class Lucy::Search::NoMatchQuery inherits Lucy::Search::Query
+ : dumpable {
+
+ bool_t fails_to_match;
+
+ inert incremented NoMatchQuery*
+ new();
+
+ /** Constructor. Takes no arguments.
+ */
+ public inert NoMatchQuery*
+ init(NoMatchQuery *self);
+
+ void
+ Set_Fails_To_Match(NoMatchQuery *self, bool_t fails_to_match);
+
+ bool_t
+ Get_Fails_To_Match(NoMatchQuery *self);
+
+ public incremented Obj*
+ Dump(NoMatchQuery *self);
+
+ public incremented NoMatchQuery*
+ Load(NoMatchQuery *self, Obj *dump);
+
+ public void
+ Serialize(NoMatchQuery *self, OutStream *outstream);
+
+ public incremented NoMatchQuery*
+ Deserialize(NoMatchQuery *self, InStream *instream);
+
+ public bool_t
+ Equals(NoMatchQuery *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(NoMatchQuery *self);
+
+ public incremented Compiler*
+ Make_Compiler(NoMatchQuery *self, Searcher *searcher, float boost);
+}
+
+class Lucy::Search::NoMatchCompiler
+ inherits Lucy::Search::Compiler {
+
+ inert incremented NoMatchCompiler*
+ new(NoMatchQuery *parent, Searcher *searcher, float boost);
+
+ inert NoMatchCompiler*
+ init(NoMatchCompiler *self, NoMatchQuery *parent,
+ Searcher *searcher, float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(NoMatchCompiler *self, SegReader *reader, bool_t need_score);
+
+ public incremented NoMatchCompiler*
+ Deserialize(NoMatchCompiler *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/ORMatcher.c b/core/Lucy/Search/ORMatcher.c
new file mode 100644
index 0000000..b8b0c3a
--- /dev/null
+++ b/core/Lucy/Search/ORMatcher.c
@@ -0,0 +1,396 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ORMATCHER
+#define C_LUCY_ORSCORER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/ORMatcher.h"
+#include "Lucy/Index/Similarity.h"
+
+// Add an element to the queue. Unsafe -- bounds checking of queue size is
+// left to the caller.
+static void
+S_add_element(ORMatcher *self, Matcher *matcher, int32_t doc_id);
+
+// Empty out the queue.
+static void
+S_clear(ORMatcher *self);
+
+// Call Matcher_Next() on the top queue element and adjust the queue,
+// removing the element if Matcher_Next() returns false.
+static INLINE int32_t
+SI_top_next(ORMatcher *self);
+
+// Call Matcher_Advance() on the top queue element and adjust the queue,
+// removing the element if Matcher_Advance() returns false.
+static INLINE int32_t
+SI_top_advance(ORMatcher *self, int32_t target);
+
+/* React to a change in the top element, or "root" -- presumably the update of
+ * its doc_id resulting from a call to Matcher_Next() or Matcher_Advance().
+ * If the Matcher has been exhausted, remove it from the queue and replace it
+ * with the bottom node (i.e. perform "root replacement"). In either case,
+ * perform a "sift down" to restore the heap property. Return the doc id of
+ * the root node, or 0 if the queue has been emptied.
+ */
+static int32_t
+S_adjust_root(ORMatcher *self);
+
+// Take the bottom node (which probably violates the heap property when this
+// is called) and bubble it up through the heap until the heap property is
+// restored.
+static void
+S_bubble_up(ORMatcher *self);
+
+// Take the top node (which probably violates the heap property when this
+// is called) and sift it down through the heap until the heap property is
+// restored.
+static void
+S_sift_down(ORMatcher *self);
+
+ORMatcher*
+ORMatcher_new(VArray *children) {
+ ORMatcher *self = (ORMatcher*)VTable_Make_Obj(ORMATCHER);
+ return ORMatcher_init(self, children);
+}
+
+static ORMatcher*
+S_ormatcher_init2(ORMatcher *self, VArray *children, Similarity *sim) {
+ size_t amount_to_malloc;
+ uint32_t i;
+
+ // Init.
+ PolyMatcher_init((PolyMatcher*)self, children, sim);
+ self->size = 0;
+
+ // Derive.
+ self->max_size = VA_Get_Size(children);
+
+ // Allocate.
+ self->heap = (HeapedMatcherDoc**)CALLOCATE(self->max_size + 1, sizeof(HeapedMatcherDoc*));
+
+ // Create a pool of HMDs. Encourage CPU cache hits by using a single
+ // allocation for all of them.
+ amount_to_malloc = (self->max_size + 1) * sizeof(HeapedMatcherDoc);
+ self->blob = (char*)MALLOCATE(amount_to_malloc);
+ self->pool = (HeapedMatcherDoc**)CALLOCATE(self->max_size + 1, sizeof(HeapedMatcherDoc*));
+ for (i = 1; i <= self->max_size; i++) {
+ size_t offset = i * sizeof(HeapedMatcherDoc);
+ HeapedMatcherDoc *hmd = (HeapedMatcherDoc*)(self->blob + offset);
+ self->pool[i] = hmd;
+ }
+
+ // Prime queue.
+ for (i = 0; i < self->max_size; i++) {
+ Matcher *matcher = (Matcher*)VA_Fetch(children, i);
+ S_add_element(self, (Matcher*)INCREF(matcher), 0);
+ }
+
+ return self;
+}
+
+ORMatcher*
+ORMatcher_init(ORMatcher *self, VArray *children) {
+ return S_ormatcher_init2(self, children, NULL);
+}
+
+void
+ORMatcher_destroy(ORMatcher *self) {
+ if (self->blob) { S_clear(self); }
+ FREEMEM(self->blob);
+ FREEMEM(self->pool);
+ FREEMEM(self->heap);
+ SUPER_DESTROY(self, ORMATCHER);
+}
+
+int32_t
+ORMatcher_next(ORMatcher *self) {
+ if (self->size == 0) {
+ return 0;
+ }
+ else {
+ int32_t last_doc_id = self->top_hmd->doc;
+ while (self->top_hmd->doc == last_doc_id) {
+ int32_t top_doc_id = SI_top_next(self);
+ if (!top_doc_id && self->size == 0) {
+ return 0;
+ }
+ }
+ return self->top_hmd->doc;
+ }
+}
+
+int32_t
+ORMatcher_advance(ORMatcher *self, int32_t target) {
+ if (!self->size) { return 0; }
+ do {
+ int32_t least = SI_top_advance(self, target);
+ if (least >= target) { return least; }
+ if (!least) {
+ if (!self->size) { return 0; }
+ }
+ } while (true);
+}
+
+int32_t
+ORMatcher_get_doc_id(ORMatcher *self) {
+ return self->top_hmd->doc;
+}
+
+static void
+S_clear(ORMatcher *self) {
+ HeapedMatcherDoc **const heap = self->heap;
+ HeapedMatcherDoc **const pool = self->pool;
+
+ // Node 0 is held empty, to make the algo clearer.
+ for (; self->size > 0; self->size--) {
+ HeapedMatcherDoc *hmd = heap[self->size];
+ heap[self->size] = NULL;
+ DECREF(hmd->matcher);
+
+ // Put HMD back in pool.
+ pool[self->size] = hmd;
+ }
+}
+
+static INLINE int32_t
+SI_top_next(ORMatcher *self) {
+ HeapedMatcherDoc *const top_hmd = self->top_hmd;
+ top_hmd->doc = Matcher_Next(top_hmd->matcher);
+ return S_adjust_root(self);
+}
+
+static INLINE int32_t
+SI_top_advance(ORMatcher *self, int32_t target) {
+ HeapedMatcherDoc *const top_hmd = self->top_hmd;
+ top_hmd->doc = Matcher_Advance(top_hmd->matcher, target);
+ return S_adjust_root(self);
+}
+
+static void
+S_add_element(ORMatcher *self, Matcher *matcher, int32_t doc_id) {
+ HeapedMatcherDoc **const heap = self->heap;
+ HeapedMatcherDoc **const pool = self->pool;
+ HeapedMatcherDoc *hmd;
+
+ // Increment size.
+ self->size++;
+
+ // Put element at the bottom of the heap.
+ hmd = pool[self->size];
+ hmd->matcher = matcher;
+ hmd->doc = doc_id;
+ heap[self->size] = hmd;
+
+ // Adjust heap.
+ S_bubble_up(self);
+}
+
+static int32_t
+S_adjust_root(ORMatcher *self) {
+ HeapedMatcherDoc *const top_hmd = self->top_hmd;
+
+ // Inlined pop.
+ if (!top_hmd->doc) {
+ HeapedMatcherDoc *const last_hmd = self->heap[self->size];
+
+ // Last to first.
+ DECREF(top_hmd->matcher);
+ top_hmd->matcher = last_hmd->matcher;
+ top_hmd->doc = last_hmd->doc;
+ self->heap[self->size] = NULL;
+
+ // Put back in pool.
+ self->pool[self->size] = last_hmd;
+
+ self->size--;
+ if (self->size == 0) {
+ return 0;
+ }
+ }
+
+ // Move queue no matter what.
+ S_sift_down(self);
+
+ return self->top_hmd->doc;
+}
+
+static void
+S_bubble_up(ORMatcher *self) {
+ HeapedMatcherDoc **const heap = self->heap;
+ uint32_t i = self->size;
+ uint32_t j = i >> 1;
+ HeapedMatcherDoc *const node = heap[i]; // save bottom node
+
+ while (j > 0 && node->doc < heap[j]->doc) {
+ heap[i] = heap[j];
+ i = j;
+ j = j >> 1;
+ }
+ heap[i] = node;
+ self->top_hmd = heap[1];
+}
+
+static void
+S_sift_down(ORMatcher *self) {
+ HeapedMatcherDoc **const heap = self->heap;
+ uint32_t i = 1;
+ uint32_t j = i << 1;
+ uint32_t k = j + 1;
+ HeapedMatcherDoc *const node = heap[i]; // save top node
+
+ // Find smaller child.
+ if (k <= self->size && heap[k]->doc < heap[j]->doc) {
+ j = k;
+ }
+
+ while (j <= self->size && heap[j]->doc < node->doc) {
+ heap[i] = heap[j];
+ i = j;
+ j = i << 1;
+ k = j + 1;
+ if (k <= self->size && heap[k]->doc < heap[j]->doc) {
+ j = k;
+ }
+ }
+ heap[i] = node;
+
+ self->top_hmd = heap[1];
+}
+
+/***************************************************************************/
+
+/* When this is called, all children are past the current self->doc_id. The
+ * least doc_id amongst them becomes the new self->doc_id, and they are all
+ * advanced so that they are once again out in front of it. While they are
+ * advancing, their scores are cached in an array, to be summed during
+ * Score().
+ */
+static int32_t
+S_advance_after_current(ORScorer *self);
+
+ORScorer*
+ORScorer_new(VArray *children, Similarity *sim) {
+ ORScorer *self = (ORScorer*)VTable_Make_Obj(ORSCORER);
+ return ORScorer_init(self, children, sim);
+}
+
+ORScorer*
+ORScorer_init(ORScorer *self, VArray *children, Similarity *sim) {
+ S_ormatcher_init2((ORMatcher*)self, children, sim);
+ self->doc_id = 0;
+ self->scores = (float*)MALLOCATE(self->num_kids * sizeof(float));
+
+ // Establish the state of all child matchers being past the current doc
+ // id, by invoking ORMatcher's Next() method.
+ ORMatcher_next((ORMatcher*)self);
+
+ return self;
+}
+
+void
+ORScorer_destroy(ORScorer *self) {
+ FREEMEM(self->scores);
+ SUPER_DESTROY(self, ORSCORER);
+}
+
+int32_t
+ORScorer_next(ORScorer *self) {
+ return S_advance_after_current(self);
+}
+
+static int32_t
+S_advance_after_current(ORScorer *self) {
+ float *const scores = self->scores;
+ Matcher *child;
+
+ // Get the top Matcher, or bail because there are no Matchers left.
+ if (!self->size) { return 0; }
+ else { child = self->top_hmd->matcher; }
+
+ // The top matcher will already be at the correct doc, so start there.
+ self->doc_id = self->top_hmd->doc;
+ scores[0] = Matcher_Score(child);
+ self->matching_kids = 1;
+
+ do {
+ // Attempt to advance past current doc.
+ int32_t top_doc_id = SI_top_next((ORMatcher*)self);
+ if (!top_doc_id) {
+ if (!self->size) {
+ break; // bail, no more to advance
+ }
+ }
+
+ if (top_doc_id != self->doc_id) {
+ // Bail, least doc in queue is now past the one we're scoring.
+ break;
+ }
+ else {
+ // Accumulate score.
+ child = self->top_hmd->matcher;
+ scores[self->matching_kids] = Matcher_Score(child);
+ self->matching_kids++;
+ }
+ } while (true);
+
+ return self->doc_id;
+}
+
+int32_t
+ORScorer_advance(ORScorer *self, int32_t target) {
+ // Return sentinel once exhausted.
+ if (!self->size) { return 0; }
+
+ // Succeed if we're already past and still on a valid doc.
+ if (target <= self->doc_id) {
+ return self->doc_id;
+ }
+
+ do {
+ // If all matchers are caught up, accumulate score and return.
+ if (self->top_hmd->doc >= target) {
+ return S_advance_after_current(self);
+ }
+
+ // Not caught up yet, so keep skipping matchers.
+ if (!SI_top_advance((ORMatcher*)self, target)) {
+ if (!self->size) { return 0; }
+ }
+ } while (true);
+}
+
+int32_t
+ORScorer_get_doc_id(ORScorer *self) {
+ return self->doc_id;
+}
+
+float
+ORScorer_score(ORScorer *self) {
+ float *const scores = self->scores;
+ float score = 0.0f;
+ uint32_t i;
+
+ // Accumulate score, then factor in coord bonus.
+ for (i = 0; i < self->matching_kids; i++) {
+ score += scores[i];
+ }
+ score *= self->coord_factors[self->matching_kids];
+
+ return score;
+}
+
diff --git a/core/Lucy/Search/ORMatcher.cfh b/core/Lucy/Search/ORMatcher.cfh
new file mode 100644
index 0000000..51bb8d8
--- /dev/null
+++ b/core/Lucy/Search/ORMatcher.cfh
@@ -0,0 +1,103 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+#include "Lucy/Search/Matcher.h"
+
+/* A wrapper for a Matcher which caches the result of Matcher_Get_Doc_ID().
+ */
+typedef struct lucy_HeapedMatcherDoc {
+ lucy_Matcher *matcher;
+ int32_t doc;
+} lucy_HeapedMatcherDoc;
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define HeapedMatcherDoc lucy_HeapedMatcherDoc
+#endif
+
+__END_C__
+
+/** Matcher which unions the doc id sets of other Matchers using a priority
+ * queue.
+ */
+
+class Lucy::Search::ORMatcher inherits Lucy::Search::PolyMatcher {
+
+ lucy_HeapedMatcherDoc **heap;
+ lucy_HeapedMatcherDoc **pool; /* Pool of HMDs to minimize mallocs */
+ char *blob; /* single allocation for all HMDs */
+ lucy_HeapedMatcherDoc *top_hmd; /* cached top elem */
+ uint32_t size;
+ uint32_t max_size;
+
+ inert incremented ORMatcher*
+ new(VArray *children);
+
+ /**
+ * @param children An array of Matchers.
+ */
+ inert incremented ORMatcher*
+ init(ORMatcher *self, VArray *children);
+
+ public void
+ Destroy(ORMatcher *self);
+
+ public int32_t
+ Next(ORMatcher *self);
+
+ public int32_t
+ Advance(ORMatcher *self, int32_t target);
+
+ public int32_t
+ Get_Doc_ID(ORMatcher *self);
+}
+
+/**
+ * Union results of multiple Matchers.
+ *
+ * ORScorer collates the output of multiple scoring child Matchers, summing
+ * their scores whenever they match the same document.
+ */
+class Lucy::Search::ORScorer inherits Lucy::Search::ORMatcher {
+
+ float *scores;
+ int32_t doc_id;
+
+ inert incremented ORScorer*
+ new(VArray *children, Similarity *similarity);
+
+ inert ORScorer*
+ init(ORScorer *self, VArray *children, Similarity *similarity);
+
+ public void
+ Destroy(ORScorer *self);
+
+ public int32_t
+ Next(ORScorer *self);
+
+ public int32_t
+ Advance(ORScorer *self, int32_t target);
+
+ public float
+ Score(ORScorer *self);
+
+ public int32_t
+ Get_Doc_ID(ORScorer *self);
+}
+
+
diff --git a/core/Lucy/Search/ORQuery.c b/core/Lucy/Search/ORQuery.c
new file mode 100644
index 0000000..16f0413
--- /dev/null
+++ b/core/Lucy/Search/ORQuery.c
@@ -0,0 +1,139 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ORQUERY
+#define C_LUCY_ORCOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/ORQuery.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Search/ORMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+ORQuery*
+ORQuery_new(VArray *children) {
+ ORQuery *self = (ORQuery*)VTable_Make_Obj(ORQUERY);
+ return ORQuery_init(self, children);
+}
+
+ORQuery*
+ORQuery_init(ORQuery *self, VArray *children) {
+ return (ORQuery*)PolyQuery_init((PolyQuery*)self, children);
+}
+
+Compiler*
+ORQuery_make_compiler(ORQuery *self, Searcher *searcher, float boost) {
+ return (Compiler*)ORCompiler_new(self, searcher, boost);
+}
+
+bool_t
+ORQuery_equals(ORQuery *self, Obj *other) {
+ if ((ORQuery*)other == self) { return true; }
+ if (!Obj_Is_A(other, ORQUERY)) { return false; }
+ return PolyQuery_equals((PolyQuery*)self, other);
+}
+
+CharBuf*
+ORQuery_to_string(ORQuery *self) {
+ uint32_t num_kids = VA_Get_Size(self->children);
+ if (!num_kids) { return CB_new_from_trusted_utf8("()", 2); }
+ else {
+ CharBuf *retval = CB_new_from_trusted_utf8("(", 1);
+ uint32_t i;
+ uint32_t last_kid = num_kids - 1;
+ for (i = 0; i < num_kids; i++) {
+ CharBuf *kid_string = Obj_To_String(VA_Fetch(self->children, i));
+ CB_Cat(retval, kid_string);
+ DECREF(kid_string);
+ if (i == last_kid) {
+ CB_Cat_Trusted_Str(retval, ")", 1);
+ }
+ else {
+ CB_Cat_Trusted_Str(retval, " OR ", 4);
+ }
+ }
+ return retval;
+ }
+}
+
+/**********************************************************************/
+
+ORCompiler*
+ORCompiler_new(ORQuery *parent, Searcher *searcher, float boost) {
+ ORCompiler *self = (ORCompiler*)VTable_Make_Obj(ORCOMPILER);
+ return ORCompiler_init(self, parent, searcher, boost);
+}
+
+ORCompiler*
+ORCompiler_init(ORCompiler *self, ORQuery *parent, Searcher *searcher,
+ float boost) {
+ PolyCompiler_init((PolyCompiler*)self, (PolyQuery*)parent, searcher,
+ boost);
+ ORCompiler_Normalize(self);
+ return self;
+}
+
+Matcher*
+ORCompiler_make_matcher(ORCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ uint32_t num_kids = VA_Get_Size(self->children);
+
+ if (num_kids == 1) {
+ Compiler *only_child = (Compiler*)VA_Fetch(self->children, 0);
+ return Compiler_Make_Matcher(only_child, reader, need_score);
+ }
+ else {
+ VArray *submatchers = VA_new(num_kids);
+ uint32_t i;
+ uint32_t num_submatchers = 0;
+
+ // Accumulate sub-matchers.
+ for (i = 0; i < num_kids; i++) {
+ Compiler *child = (Compiler*)VA_Fetch(self->children, i);
+ Matcher *submatcher
+ = Compiler_Make_Matcher(child, reader, need_score);
+ if (submatcher != NULL) {
+ VA_Push(submatchers, (Obj*)submatcher);
+ num_submatchers++;
+ }
+ }
+
+ if (num_submatchers == 0) {
+ // No possible matches, so return null.
+ DECREF(submatchers);
+ return NULL;
+ }
+ else if (num_submatchers == 1) {
+ // Only one submatcher, so no need for ORScorer wrapper.
+ Matcher *submatcher = (Matcher*)INCREF(VA_Fetch(submatchers, 0));
+ DECREF(submatchers);
+ return submatcher;
+ }
+ else {
+ Similarity *sim = ORCompiler_Get_Similarity(self);
+ Matcher *retval = need_score
+ ? (Matcher*)ORScorer_new(submatchers, sim)
+ : (Matcher*)ORMatcher_new(submatchers);
+ DECREF(submatchers);
+ return retval;
+ }
+ }
+}
+
+
diff --git a/core/Lucy/Search/ORQuery.cfh b/core/Lucy/Search/ORQuery.cfh
new file mode 100644
index 0000000..c15e367
--- /dev/null
+++ b/core/Lucy/Search/ORQuery.cfh
@@ -0,0 +1,63 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Union multiple result sets.
+ *
+ * ORQuery is a composite L<Query|Lucy::Search::Query> which matches
+ * when any of its children match, so its result set is the union of their
+ * result sets. Matching documents recieve a summed score from all matching
+ * child Queries.
+ */
+
+class Lucy::Search::ORQuery inherits Lucy::Search::PolyQuery
+ : dumpable {
+
+ inert incremented ORQuery*
+ new(VArray *children = NULL);
+
+ /**
+ * @param children An array of child Queries.
+ */
+ public inert ORQuery*
+ init(ORQuery *self, VArray *children = NULL);
+
+ public incremented Compiler*
+ Make_Compiler(ORQuery *self, Searcher *searcher, float boost);
+
+ public incremented CharBuf*
+ To_String(ORQuery *self);
+
+ public bool_t
+ Equals(ORQuery *self, Obj *other);
+}
+
+class Lucy::Search::ORCompiler
+ inherits Lucy::Search::PolyCompiler {
+
+ inert incremented ORCompiler*
+ new(ORQuery *parent, Searcher *searcher, float boost);
+
+ inert ORCompiler*
+ init(ORCompiler *self, ORQuery *parent, Searcher *searcher,
+ float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(ORCompiler *self, SegReader *reader, bool_t need_score);
+}
+
+
diff --git a/core/Lucy/Search/PhraseMatcher.c b/core/Lucy/Search/PhraseMatcher.c
new file mode 100644
index 0000000..c10f523
--- /dev/null
+++ b/core/Lucy/Search/PhraseMatcher.c
@@ -0,0 +1,311 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_PHRASEMATCHER
+#define C_LUCY_POSTING
+#define C_LUCY_SCOREPOSTING
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/PhraseMatcher.h"
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Search/Compiler.h"
+
+PhraseMatcher*
+PhraseMatcher_new(Similarity *sim, VArray *plists, Compiler *compiler) {
+ PhraseMatcher *self = (PhraseMatcher*)VTable_Make_Obj(PHRASEMATCHER);
+ return PhraseMatcher_init(self, sim, plists, compiler);
+
+}
+
+PhraseMatcher*
+PhraseMatcher_init(PhraseMatcher *self, Similarity *similarity, VArray *plists,
+ Compiler *compiler) {
+ Matcher_init((Matcher*)self);
+
+ // Init.
+ self->anchor_set = BB_new(0);
+ self->phrase_freq = 0.0;
+ self->phrase_boost = 0.0;
+ self->first_time = true;
+ self->more = true;
+
+ // Extract PostingLists out of VArray into local C array for quick access.
+ self->num_elements = VA_Get_Size(plists);
+ self->plists = (PostingList**)MALLOCATE(
+ self->num_elements * sizeof(PostingList*));
+ for (size_t i = 0; i < self->num_elements; i++) {
+ PostingList *const plist
+ = (PostingList*)CERTIFY(VA_Fetch(plists, i), POSTINGLIST);
+ if (plist == NULL) {
+ THROW(ERR, "Missing element %u32", i);
+ }
+ self->plists[i] = (PostingList*)INCREF(plist);
+ }
+
+ // Assign.
+ self->sim = (Similarity*)INCREF(similarity);
+ self->compiler = (Compiler*)INCREF(compiler);
+ self->weight = Compiler_Get_Weight(compiler);
+
+ return self;
+}
+
+void
+PhraseMatcher_destroy(PhraseMatcher *self) {
+ if (self->plists) {
+ for (size_t i = 0; i < self->num_elements; i++) {
+ DECREF(self->plists[i]);
+ }
+ FREEMEM(self->plists);
+ }
+ DECREF(self->sim);
+ DECREF(self->anchor_set);
+ DECREF(self->compiler);
+ SUPER_DESTROY(self, PHRASEMATCHER);
+}
+
+int32_t
+PhraseMatcher_next(PhraseMatcher *self) {
+ if (self->first_time) {
+ return PhraseMatcher_Advance(self, 1);
+ }
+ else if (self->more) {
+ const int32_t target = PList_Get_Doc_ID(self->plists[0]) + 1;
+ return PhraseMatcher_Advance(self, target);
+ }
+ else {
+ return 0;
+ }
+}
+
+int32_t
+PhraseMatcher_advance(PhraseMatcher *self, int32_t target) {
+ PostingList **const plists = self->plists;
+ const uint32_t num_elements = self->num_elements;
+ int32_t highest = 0;
+
+ // Reset match variables to indicate no match. New values will be
+ // assigned if a match succeeds.
+ self->phrase_freq = 0.0;
+ self->doc_id = 0;
+
+ // Find the lowest possible matching doc ID greater than the current doc
+ // ID. If any one of the PostingLists is exhausted, we're done.
+ if (self->first_time) {
+ self->first_time = false;
+
+ // On the first call to Advance(), advance all PostingLists.
+ for (size_t i = 0, max = self->num_elements; i < max; i++) {
+ int32_t candidate = PList_Advance(plists[i], target);
+ if (!candidate) {
+ self->more = false;
+ return 0;
+ }
+ else if (candidate > highest) {
+ // Remember the highest doc ID so far.
+ highest = candidate;
+ }
+ }
+ }
+ else {
+ // On subsequent iters, advance only one PostingList. Its new doc ID
+ // becomes the minimum target which all the others must move up to.
+ highest = PList_Advance(plists[0], target);
+ if (highest == 0) {
+ self->more = false;
+ return 0;
+ }
+ }
+
+ // Find a doc which contains all the terms.
+ while (1) {
+ bool_t agreement = true;
+
+ // Scoot all posting lists up to at least the current minimum.
+ for (uint32_t i = 0; i < num_elements; i++) {
+ PostingList *const plist = plists[i];
+ int32_t candidate = PList_Get_Doc_ID(plist);
+
+ // Is this PostingList already beyond the minimum? Then raise the
+ // bar for everyone else.
+ if (highest < candidate) { highest = candidate; }
+ if (target < highest) { target = highest; }
+
+ // Scoot this posting list up.
+ if (candidate < target) {
+ candidate = PList_Advance(plist, target);
+
+ // If this PostingList is exhausted, we're done.
+ if (candidate == 0) {
+ self->more = false;
+ return 0;
+ }
+
+ // After calling PList_Advance(), we are guaranteed to be
+ // either at or beyond the minimum, so we can assign without
+ // checking and the minumum will either go up or stay the
+ // same.
+ highest = candidate;
+ }
+ }
+
+ // See whether all the PostingLists have managed to converge on a
+ // single doc ID.
+ for (uint32_t i = 0; i < num_elements; i++) {
+ const int32_t candidate = PList_Get_Doc_ID(plists[i]);
+ if (candidate != highest) { agreement = false; }
+ }
+
+ // If we've found a doc with all terms in it, see if they form a
+ // phrase.
+ if (agreement && highest >= target) {
+ self->phrase_freq = PhraseMatcher_Calc_Phrase_Freq(self);
+ if (self->phrase_freq == 0.0) {
+ // No phrase. Move on to another doc.
+ target += 1;
+ }
+ else {
+ // Success!
+ self->doc_id = highest;
+ return highest;
+ }
+ }
+ }
+}
+
+static INLINE uint32_t
+SI_winnow_anchors(uint32_t *anchors_start, const uint32_t *const anchors_end,
+ const uint32_t *candidates, const uint32_t *const candidates_end,
+ uint32_t offset) {
+ uint32_t *anchors = anchors_start;
+ uint32_t *anchors_found = anchors_start;
+ uint32_t target_anchor;
+ uint32_t target_candidate;
+
+ // Safety check, so there's no chance of a bad dereference.
+ if (anchors_start == anchors_end || candidates == candidates_end) {
+ return 0;
+ }
+
+ /* This function is a loop that finds terms that can continue a phrase.
+ * It overwrites the anchors in place, and returns the number remaining.
+ * The basic algorithm is to alternately increment the candidates' pointer
+ * until it is at or beyond its target position, and then increment the
+ * anchors' pointer until it is at or beyond its target. The non-standard
+ * form is to avoid unnecessary comparisons. This loop has not been
+ * tested for speed, but glancing at the object code produced (objdump -S)
+ * it appears to be significantly faster than the nested loop alternative.
+ * But given the vagaries of modern processors, it merits actual
+ * testing.*/
+
+SPIN_CANDIDATES:
+ target_candidate = *anchors + offset;
+ while (*candidates < target_candidate) {
+ if (++candidates == candidates_end) { goto DONE; }
+ }
+ if (*candidates == target_candidate) { goto MATCH; }
+ goto SPIN_ANCHORS;
+
+SPIN_ANCHORS:
+ target_anchor = *candidates - offset;
+ while (*anchors < target_anchor) {
+ if (++anchors == anchors_end) { goto DONE; }
+ };
+ if (*anchors == target_anchor) { goto MATCH; }
+ goto SPIN_CANDIDATES;
+
+MATCH:
+ *anchors_found++ = *anchors;
+ if (++anchors == anchors_end) { goto DONE; }
+ goto SPIN_CANDIDATES;
+
+DONE:
+ // Return number of anchors remaining.
+ return anchors_found - anchors_start;
+}
+
+float
+PhraseMatcher_calc_phrase_freq(PhraseMatcher *self) {
+ PostingList **const plists = self->plists;
+
+ /* Create a overwriteable "anchor set" from the first posting.
+ *
+ * Each "anchor" is a position, measured in tokens, corresponding to a a
+ * term which might start a phrase. We start off with an "anchor set"
+ * comprised of all positions at which the first term in the phrase occurs
+ * in the field.
+ *
+ * There can never be more phrase matches than instances of this first
+ * term. There may be fewer however, which we will determine by seeing
+ * whether all the other terms line up at subsequent position slots.
+ *
+ * Every time we eliminate an anchor from the anchor set, we splice it out
+ * of the array. So if we begin with an anchor set of (15, 51, 72) and we
+ * discover that phrases occur at the first and last instances of the
+ * first term but not the middle one, the final array will be (15, 72).
+ *
+ * The number of elements in the anchor set when we are finished winnowing
+ * is our phrase freq.
+ */
+ ScorePosting *posting = (ScorePosting*)PList_Get_Posting(plists[0]);
+ uint32_t anchors_remaining = posting->freq;
+ if (!anchors_remaining) { return 0.0f; }
+
+ size_t amount = anchors_remaining * sizeof(uint32_t);
+ uint32_t *anchors_start = (uint32_t*)BB_Grow(self->anchor_set, amount);
+ uint32_t *anchors_end = anchors_start + anchors_remaining;
+ memcpy(anchors_start, posting->prox, amount);
+
+ // Match the positions of other terms against the anchor set.
+ for (uint32_t i = 1, max = self->num_elements; i < max; i++) {
+ // Get the array of positions for the next term. Unlike the anchor
+ // set (which is a copy), these won't be overwritten.
+ ScorePosting *posting = (ScorePosting*)PList_Get_Posting(plists[i]);
+ uint32_t *candidates_start = posting->prox;
+ uint32_t *candidates_end = candidates_start + posting->freq;
+
+ // Splice out anchors that don't match the next term. Bail out if
+ // we've eliminated all possible anchors.
+ anchors_remaining
+ = SI_winnow_anchors(anchors_start, anchors_end,
+ candidates_start, candidates_end, i);
+ if (!anchors_remaining) { return 0.0f; }
+
+ // Adjust end for number of anchors that remain.
+ anchors_end = anchors_start + anchors_remaining;
+ }
+
+ // The number of anchors left is the phrase freq.
+ return (float)anchors_remaining;
+}
+
+int32_t
+PhraseMatcher_get_doc_id(PhraseMatcher *self) {
+ return self->doc_id;
+}
+
+float
+PhraseMatcher_score(PhraseMatcher *self) {
+ ScorePosting *posting = (ScorePosting*)PList_Get_Posting(self->plists[0]);
+ float score = Sim_TF(self->sim, self->phrase_freq)
+ * self->weight
+ * posting->weight;
+ return score;
+}
+
+
diff --git a/core/Lucy/Search/PhraseMatcher.cfh b/core/Lucy/Search/PhraseMatcher.cfh
new file mode 100644
index 0000000..2669b71
--- /dev/null
+++ b/core/Lucy/Search/PhraseMatcher.cfh
@@ -0,0 +1,64 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Matcher for a PhraseQuery.
+ */
+
+class Lucy::Search::PhraseMatcher inherits Lucy::Search::Matcher {
+
+ int32_t doc_id;
+ uint32_t num_elements;
+ Similarity *sim;
+ PostingList **plists;
+ ByteBuf *anchor_set;
+ float phrase_freq;
+ float phrase_boost;
+ Compiler *compiler;
+ float weight;
+ bool_t first_time;
+ bool_t more;
+
+ inert incremented PhraseMatcher*
+ new(Similarity *similarity, VArray *posting_lists, Compiler *compiler);
+
+ inert PhraseMatcher*
+ init(PhraseMatcher *self, Similarity *similarity, VArray *posting_lists,
+ Compiler *compiler);
+
+ public void
+ Destroy(PhraseMatcher *self);
+
+ public int32_t
+ Next(PhraseMatcher *self);
+
+ public int32_t
+ Advance(PhraseMatcher *self, int32_t target);
+
+ public int32_t
+ Get_Doc_ID(PhraseMatcher *self);
+
+ public float
+ Score(PhraseMatcher *self);
+
+ /** Calculate how often the phrase occurs in the current document.
+ */
+ float
+ Calc_Phrase_Freq(PhraseMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/PhraseQuery.c b/core/Lucy/Search/PhraseQuery.c
new file mode 100644
index 0000000..8827b9b
--- /dev/null
+++ b/core/Lucy/Search/PhraseQuery.c
@@ -0,0 +1,403 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_PHRASEQUERY
+#define C_LUCY_PHRASECOMPILER
+#include <stdarg.h>
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/PhraseQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/SegPostingList.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/TermVector.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/PhraseMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+// Shared initialization routine which assumes that it's ok to assume control
+// over [field] and [terms], eating their refcounts.
+static PhraseQuery*
+S_do_init(PhraseQuery *self, CharBuf *field, VArray *terms, float boost);
+
+PhraseQuery*
+PhraseQuery_new(const CharBuf *field, VArray *terms) {
+ PhraseQuery *self = (PhraseQuery*)VTable_Make_Obj(PHRASEQUERY);
+ return PhraseQuery_init(self, field, terms);
+}
+
+PhraseQuery*
+PhraseQuery_init(PhraseQuery *self, const CharBuf *field, VArray *terms) {
+ return S_do_init(self, CB_Clone(field), VA_Clone(terms), 1.0f);
+}
+
+void
+PhraseQuery_destroy(PhraseQuery *self) {
+ DECREF(self->terms);
+ DECREF(self->field);
+ SUPER_DESTROY(self, PHRASEQUERY);
+}
+
+static PhraseQuery*
+S_do_init(PhraseQuery *self, CharBuf *field, VArray *terms, float boost) {
+ uint32_t i, max;
+ Query_init((Query*)self, boost);
+ for (i = 0, max = VA_Get_Size(terms); i < max; i++) {
+ CERTIFY(VA_Fetch(terms, i), OBJ);
+ }
+ self->field = field;
+ self->terms = terms;
+ return self;
+}
+
+void
+PhraseQuery_serialize(PhraseQuery *self, OutStream *outstream) {
+ OutStream_Write_F32(outstream, self->boost);
+ CB_Serialize(self->field, outstream);
+ VA_Serialize(self->terms, outstream);
+}
+
+PhraseQuery*
+PhraseQuery_deserialize(PhraseQuery *self, InStream *instream) {
+ float boost = InStream_Read_F32(instream);
+ CharBuf *field = CB_deserialize(NULL, instream);
+ VArray *terms = VA_deserialize(NULL, instream);
+ self = self ? self : (PhraseQuery*)VTable_Make_Obj(PHRASEQUERY);
+ return S_do_init(self, field, terms, boost);
+}
+
+bool_t
+PhraseQuery_equals(PhraseQuery *self, Obj *other) {
+ PhraseQuery *twin = (PhraseQuery*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, PHRASEQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (self->field && !twin->field) { return false; }
+ if (!self->field && twin->field) { return false; }
+ if (self->field && !CB_Equals(self->field, (Obj*)twin->field)) {
+ return false;
+ }
+ if (!VA_Equals(twin->terms, (Obj*)self->terms)) { return false; }
+ return true;
+}
+
+CharBuf*
+PhraseQuery_to_string(PhraseQuery *self) {
+ uint32_t i;
+ uint32_t num_terms = VA_Get_Size(self->terms);
+ CharBuf *retval = CB_Clone(self->field);
+ CB_Cat_Trusted_Str(retval, ":\"", 2);
+ for (i = 0; i < num_terms; i++) {
+ Obj *term = VA_Fetch(self->terms, i);
+ CharBuf *term_string = Obj_To_String(term);
+ CB_Cat(retval, term_string);
+ DECREF(term_string);
+ if (i < num_terms - 1) {
+ CB_Cat_Trusted_Str(retval, " ", 1);
+ }
+ }
+ CB_Cat_Trusted_Str(retval, "\"", 1);
+ return retval;
+}
+
+Compiler*
+PhraseQuery_make_compiler(PhraseQuery *self, Searcher *searcher,
+ float boost) {
+ if (VA_Get_Size(self->terms) == 1) {
+ // Optimize for one-term "phrases".
+ Obj *term = VA_Fetch(self->terms, 0);
+ TermQuery *term_query = TermQuery_new(self->field, term);
+ TermCompiler *term_compiler;
+ TermQuery_Set_Boost(term_query, self->boost);
+ term_compiler
+ = (TermCompiler*)TermQuery_Make_Compiler(term_query, searcher,
+ boost);
+ DECREF(term_query);
+ return (Compiler*)term_compiler;
+ }
+ else {
+ return (Compiler*)PhraseCompiler_new(self, searcher, boost);
+ }
+}
+
+CharBuf*
+PhraseQuery_get_field(PhraseQuery *self) {
+ return self->field;
+}
+
+VArray*
+PhraseQuery_get_terms(PhraseQuery *self) {
+ return self->terms;
+}
+
+/*********************************************************************/
+
+PhraseCompiler*
+PhraseCompiler_new(PhraseQuery *parent, Searcher *searcher, float boost) {
+ PhraseCompiler *self = (PhraseCompiler*)VTable_Make_Obj(PHRASECOMPILER);
+ return PhraseCompiler_init(self, parent, searcher, boost);
+}
+
+PhraseCompiler*
+PhraseCompiler_init(PhraseCompiler *self, PhraseQuery *parent,
+ Searcher *searcher, float boost) {
+ Schema *schema = Searcher_Get_Schema(searcher);
+ Similarity *sim = Schema_Fetch_Sim(schema, parent->field);
+ VArray *terms = parent->terms;
+ uint32_t i, max;
+
+ // Try harder to find a Similarity if necessary.
+ if (!sim) { sim = Schema_Get_Similarity(schema); }
+
+ // Init.
+ Compiler_init((Compiler*)self, (Query*)parent, searcher, sim, boost);
+
+ // Store IDF for the phrase.
+ self->idf = 0;
+ for (i = 0, max = VA_Get_Size(terms); i < max; i++) {
+ Obj *term = VA_Fetch(terms, i);
+ int32_t doc_max = Searcher_Doc_Max(searcher);
+ int32_t doc_freq = Searcher_Doc_Freq(searcher, parent->field, term);
+ self->idf += Sim_IDF(sim, doc_freq, doc_max);
+ }
+
+ // Calculate raw weight.
+ self->raw_weight = self->idf * self->boost;
+
+ // Make final preparations.
+ PhraseCompiler_Normalize(self);
+
+ return self;
+}
+
+void
+PhraseCompiler_serialize(PhraseCompiler *self, OutStream *outstream) {
+ Compiler_serialize((Compiler*)self, outstream);
+ OutStream_Write_F32(outstream, self->idf);
+ OutStream_Write_F32(outstream, self->raw_weight);
+ OutStream_Write_F32(outstream, self->query_norm_factor);
+ OutStream_Write_F32(outstream, self->normalized_weight);
+}
+
+PhraseCompiler*
+PhraseCompiler_deserialize(PhraseCompiler *self, InStream *instream) {
+ self = self ? self : (PhraseCompiler*)VTable_Make_Obj(PHRASECOMPILER);
+ Compiler_deserialize((Compiler*)self, instream);
+ self->idf = InStream_Read_F32(instream);
+ self->raw_weight = InStream_Read_F32(instream);
+ self->query_norm_factor = InStream_Read_F32(instream);
+ self->normalized_weight = InStream_Read_F32(instream);
+ return self;
+}
+
+bool_t
+PhraseCompiler_equals(PhraseCompiler *self, Obj *other) {
+ PhraseCompiler *twin = (PhraseCompiler*)other;
+ if (!Obj_Is_A(other, PHRASECOMPILER)) { return false; }
+ if (!Compiler_equals((Compiler*)self, other)) { return false; }
+ if (self->idf != twin->idf) { return false; }
+ if (self->raw_weight != twin->raw_weight) { return false; }
+ if (self->query_norm_factor != twin->query_norm_factor) { return false; }
+ if (self->normalized_weight != twin->normalized_weight) { return false; }
+ return true;
+}
+
+float
+PhraseCompiler_get_weight(PhraseCompiler *self) {
+ return self->normalized_weight;
+}
+
+float
+PhraseCompiler_sum_of_squared_weights(PhraseCompiler *self) {
+ return self->raw_weight * self->raw_weight;
+}
+
+void
+PhraseCompiler_apply_norm_factor(PhraseCompiler *self, float factor) {
+ self->query_norm_factor = factor;
+ self->normalized_weight = self->raw_weight * self->idf * factor;
+}
+
+Matcher*
+PhraseCompiler_make_matcher(PhraseCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ UNUSED_VAR(need_score);
+ PhraseQuery *const parent = (PhraseQuery*)self->parent;
+ VArray *const terms = parent->terms;
+ uint32_t num_terms = VA_Get_Size(terms);
+
+ // Bail if there are no terms.
+ if (!num_terms) { return NULL; }
+
+ // Bail unless field is valid and posting type supports positions.
+ Similarity *sim = PhraseCompiler_Get_Similarity(self);
+ Posting *posting = Sim_Make_Posting(sim);
+ if (posting == NULL || !Obj_Is_A((Obj*)posting, SCOREPOSTING)) {
+ DECREF(posting);
+ return NULL;
+ }
+ DECREF(posting);
+
+ // Bail if there's no PostingListReader for this segment.
+ PostingListReader *const plist_reader
+ = (PostingListReader*)SegReader_Fetch(
+ reader, VTable_Get_Name(POSTINGLISTREADER));
+ if (!plist_reader) { return NULL; }
+
+ // Look up each term.
+ VArray *plists = VA_new(num_terms);
+ for (uint32_t i = 0; i < num_terms; i++) {
+ Obj *term = VA_Fetch(terms, i);
+ PostingList *plist
+ = PListReader_Posting_List(plist_reader, parent->field, term);
+
+ // Bail if any one of the terms isn't in the index.
+ if (!plist || !PList_Get_Doc_Freq(plist)) {
+ DECREF(plist);
+ DECREF(plists);
+ return NULL;
+ }
+ VA_Push(plists, (Obj*)plist);
+ }
+
+ Matcher *retval
+ = (Matcher*)PhraseMatcher_new(sim, plists, (Compiler*)self);
+ DECREF(plists);
+ return retval;
+}
+
+VArray*
+PhraseCompiler_highlight_spans(PhraseCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field) {
+ PhraseQuery *const parent = (PhraseQuery*)self->parent;
+ VArray *const terms = parent->terms;
+ VArray *const spans = VA_new(0);
+ VArray *term_vectors;
+ BitVector *posit_vec;
+ BitVector *other_posit_vec;
+ uint32_t i;
+ const uint32_t num_terms = VA_Get_Size(terms);
+ uint32_t num_tvs;
+ UNUSED_VAR(searcher);
+
+ // Bail if no terms or field doesn't match.
+ if (!num_terms) { return spans; }
+ if (!CB_Equals(field, (Obj*)parent->field)) { return spans; }
+
+ term_vectors = VA_new(num_terms);
+ posit_vec = BitVec_new(0);
+ other_posit_vec = BitVec_new(0);
+ for (i = 0; i < num_terms; i++) {
+ Obj *term = VA_Fetch(terms, i);
+ TermVector *term_vector
+ = DocVec_Term_Vector(doc_vec, field, (CharBuf*)term);
+
+ // Bail if any term is missing.
+ if (!term_vector) {
+ break;
+ }
+
+ VA_Push(term_vectors, (Obj*)term_vector);
+
+ if (i == 0) {
+ // Set initial positions from first term.
+ uint32_t j;
+ I32Array *positions = TV_Get_Positions(term_vector);
+ for (j = I32Arr_Get_Size(positions); j > 0; j--) {
+ BitVec_Set(posit_vec, I32Arr_Get(positions, j - 1));
+ }
+ }
+ else {
+ // Filter positions using logical "and".
+ uint32_t j;
+ I32Array *positions = TV_Get_Positions(term_vector);
+
+ BitVec_Clear_All(other_posit_vec);
+ for (j = I32Arr_Get_Size(positions); j > 0; j--) {
+ int32_t pos = I32Arr_Get(positions, j - 1) - i;
+ if (pos >= 0) {
+ BitVec_Set(other_posit_vec, pos);
+ }
+ }
+ BitVec_And(posit_vec, other_posit_vec);
+ }
+ }
+
+ // Proceed only if all terms are present.
+ num_tvs = VA_Get_Size(term_vectors);
+ if (num_tvs == num_terms) {
+ TermVector *first_tv = (TermVector*)VA_Fetch(term_vectors, 0);
+ TermVector *last_tv
+ = (TermVector*)VA_Fetch(term_vectors, num_tvs - 1);
+ I32Array *tv_start_positions = TV_Get_Positions(first_tv);
+ I32Array *tv_end_positions = TV_Get_Positions(last_tv);
+ I32Array *tv_start_offsets = TV_Get_Start_Offsets(first_tv);
+ I32Array *tv_end_offsets = TV_Get_End_Offsets(last_tv);
+ uint32_t terms_max = num_terms - 1;
+ I32Array *valid_posits = BitVec_To_Array(posit_vec);
+ uint32_t num_valid_posits = I32Arr_Get_Size(valid_posits);
+ uint32_t j = 0;
+ uint32_t posit_tick;
+ float weight = PhraseCompiler_Get_Weight(self);
+ i = 0;
+
+ // Add only those starts/ends that belong to a valid position.
+ for (posit_tick = 0; posit_tick < num_valid_posits; posit_tick++) {
+ int32_t valid_start_posit = I32Arr_Get(valid_posits, posit_tick);
+ int32_t valid_end_posit = valid_start_posit + terms_max;
+ int32_t start_offset = 0, end_offset = 0;
+ uint32_t max;
+
+ for (max = I32Arr_Get_Size(tv_start_positions); i < max; i++) {
+ if (I32Arr_Get(tv_start_positions, i) == valid_start_posit) {
+ start_offset = I32Arr_Get(tv_start_offsets, i);
+ break;
+ }
+ }
+ for (max = I32Arr_Get_Size(tv_end_positions); j < max; j++) {
+ if (I32Arr_Get(tv_end_positions, j) == valid_end_posit) {
+ end_offset = I32Arr_Get(tv_end_offsets, j);
+ break;
+ }
+ }
+
+ VA_Push(spans, (Obj*)Span_new(start_offset,
+ end_offset - start_offset, weight));
+
+ i++, j++;
+ }
+
+ DECREF(valid_posits);
+ }
+
+ DECREF(other_posit_vec);
+ DECREF(posit_vec);
+ DECREF(term_vectors);
+ return spans;
+}
+
+
diff --git a/core/Lucy/Search/PhraseQuery.cfh b/core/Lucy/Search/PhraseQuery.cfh
new file mode 100644
index 0000000..4015d97
--- /dev/null
+++ b/core/Lucy/Search/PhraseQuery.cfh
@@ -0,0 +1,111 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Query matching an ordered list of terms.
+ *
+ * PhraseQuery is a subclass of L<Lucy::Search::Query> for matching
+ * against an ordered sequence of terms.
+ */
+
+class Lucy::Search::PhraseQuery inherits Lucy::Search::Query
+ : dumpable {
+
+ CharBuf *field;
+ VArray *terms;
+
+ inert incremented PhraseQuery*
+ new(const CharBuf *field, VArray *terms);
+
+ /**
+ * @param field The field that the phrase must occur in.
+ * @param terms The ordered array of terms that must match.
+ */
+ public inert PhraseQuery*
+ init(PhraseQuery *self, const CharBuf *field, VArray *terms);
+
+ /** Accessor for object's field attribute.
+ */
+ public CharBuf*
+ Get_Field(PhraseQuery *self);
+
+ /** Accessor for object's array of terms.
+ */
+ public VArray*
+ Get_Terms(PhraseQuery *self);
+
+ public incremented Compiler*
+ Make_Compiler(PhraseQuery *self, Searcher *searcher, float boost);
+
+ public bool_t
+ Equals(PhraseQuery *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(PhraseQuery *self);
+
+ public void
+ Serialize(PhraseQuery *self, OutStream *outstream);
+
+ public incremented PhraseQuery*
+ Deserialize(PhraseQuery *self, InStream *instream);
+
+ public void
+ Destroy(PhraseQuery *self);
+}
+
+class Lucy::Search::PhraseCompiler
+ inherits Lucy::Search::Compiler {
+
+ float idf;
+ float raw_weight;
+ float query_norm_factor;
+ float normalized_weight;
+
+ inert incremented PhraseCompiler*
+ new(PhraseQuery *parent, Searcher *searcher, float boost);
+
+ inert PhraseCompiler*
+ init(PhraseCompiler *self, PhraseQuery *parent, Searcher *searcher,
+ float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(PhraseCompiler *self, SegReader *reader, bool_t need_score);
+
+ public float
+ Get_Weight(PhraseCompiler *self);
+
+ public float
+ Sum_Of_Squared_Weights(PhraseCompiler *self);
+
+ public void
+ Apply_Norm_Factor(PhraseCompiler *self, float factor);
+
+ public incremented VArray*
+ Highlight_Spans(PhraseCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field);
+
+ public bool_t
+ Equals(PhraseCompiler *self, Obj *other);
+
+ public void
+ Serialize(PhraseCompiler *self, OutStream *outstream);
+
+ public incremented PhraseCompiler*
+ Deserialize(PhraseCompiler *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/PolyMatcher.c b/core/Lucy/Search/PolyMatcher.c
new file mode 100644
index 0000000..3b166df
--- /dev/null
+++ b/core/Lucy/Search/PolyMatcher.c
@@ -0,0 +1,54 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POLYMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/PolyMatcher.h"
+#include "Lucy/Index/Similarity.h"
+
+PolyMatcher*
+PolyMatcher_new(VArray *children, Similarity *sim) {
+ PolyMatcher *self = (PolyMatcher*)VTable_Make_Obj(POLYMATCHER);
+ return PolyMatcher_init(self, children, sim);
+}
+
+PolyMatcher*
+PolyMatcher_init(PolyMatcher *self, VArray *children, Similarity *similarity) {
+ uint32_t i;
+
+ Matcher_init((Matcher*)self);
+ self->num_kids = VA_Get_Size(children);
+ self->sim = (Similarity*)INCREF(similarity);
+ self->children = (VArray*)INCREF(children);
+ self->coord_factors = (float*)MALLOCATE((self->num_kids + 1) * sizeof(float));
+ for (i = 0; i <= self->num_kids; i++) {
+ self->coord_factors[i] = similarity
+ ? Sim_Coord(similarity, i, self->num_kids)
+ : 1.0f;
+ }
+ return self;
+}
+
+void
+PolyMatcher_destroy(PolyMatcher *self) {
+ DECREF(self->children);
+ DECREF(self->sim);
+ FREEMEM(self->coord_factors);
+ SUPER_DESTROY(self, POLYMATCHER);
+}
+
+
diff --git a/core/Lucy/Search/PolyMatcher.cfh b/core/Lucy/Search/PolyMatcher.cfh
new file mode 100644
index 0000000..006a1d0
--- /dev/null
+++ b/core/Lucy/Search/PolyMatcher.cfh
@@ -0,0 +1,40 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Base class for composite Matchers.
+ */
+
+class Lucy::Search::PolyMatcher inherits Lucy::Search::Matcher {
+
+ VArray *children;
+ Similarity *sim;
+ uint32_t num_kids;
+ uint32_t matching_kids;
+ float *coord_factors;
+
+ inert incremented PolyMatcher*
+ new(VArray *children, Similarity *similarity);
+
+ inert PolyMatcher*
+ init(PolyMatcher *self, VArray *children, Similarity *similarity);
+
+ public void
+ Destroy(PolyMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/PolyQuery.c b/core/Lucy/Search/PolyQuery.c
new file mode 100644
index 0000000..39ac1b0
--- /dev/null
+++ b/core/Lucy/Search/PolyQuery.c
@@ -0,0 +1,196 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POLYQUERY
+#define C_LUCY_POLYCOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/PolyQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+PolyQuery*
+PolyQuery_init(PolyQuery *self, VArray *children) {
+ uint32_t i;
+ const uint32_t num_kids = children ? VA_Get_Size(children) : 0;
+ Query_init((Query*)self, 1.0f);
+ self->children = VA_new(num_kids);
+ for (i = 0; i < num_kids; i++) {
+ PolyQuery_Add_Child(self, (Query*)VA_Fetch(children, i));
+ }
+ return self;
+}
+
+void
+PolyQuery_destroy(PolyQuery *self) {
+ DECREF(self->children);
+ SUPER_DESTROY(self, POLYQUERY);
+}
+
+void
+PolyQuery_add_child(PolyQuery *self, Query *query) {
+ CERTIFY(query, QUERY);
+ VA_Push(self->children, INCREF(query));
+}
+
+void
+PolyQuery_set_children(PolyQuery *self, VArray *children) {
+ DECREF(self->children);
+ self->children = (VArray*)INCREF(children);
+}
+
+VArray*
+PolyQuery_get_children(PolyQuery *self) {
+ return self->children;
+}
+
+void
+PolyQuery_serialize(PolyQuery *self, OutStream *outstream) {
+ const uint32_t num_kids = VA_Get_Size(self->children);
+ uint32_t i;
+ OutStream_Write_F32(outstream, self->boost);
+ OutStream_Write_U32(outstream, num_kids);
+ for (i = 0; i < num_kids; i++) {
+ Query *child = (Query*)VA_Fetch(self->children, i);
+ FREEZE(child, outstream);
+ }
+}
+
+PolyQuery*
+PolyQuery_deserialize(PolyQuery *self, InStream *instream) {
+ float boost = InStream_Read_F32(instream);
+ uint32_t num_children = InStream_Read_U32(instream);
+
+ if (!self) { THROW(ERR, "Abstract class"); }
+ PolyQuery_init(self, NULL);
+ PolyQuery_Set_Boost(self, boost);
+
+ VA_Grow(self->children, num_children);
+ while (num_children--) {
+ VA_Push(self->children, THAW(instream));
+ }
+
+ return self;
+}
+
+bool_t
+PolyQuery_equals(PolyQuery *self, Obj *other) {
+ PolyQuery *twin = (PolyQuery*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, POLYQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!VA_Equals(twin->children, (Obj*)self->children)) { return false; }
+ return true;
+}
+
+/**********************************************************************/
+
+
+PolyCompiler*
+PolyCompiler_init(PolyCompiler *self, PolyQuery *parent,
+ Searcher *searcher, float boost) {
+ uint32_t i;
+ const uint32_t num_kids = VA_Get_Size(parent->children);
+
+ Compiler_init((Compiler*)self, (Query*)parent, searcher, NULL, boost);
+ self->children = VA_new(num_kids);
+
+ // Iterate over the children, creating a Compiler for each one.
+ for (i = 0; i < num_kids; i++) {
+ Query *child_query = (Query*)VA_Fetch(parent->children, i);
+ float sub_boost = boost * Query_Get_Boost(child_query);
+ VA_Push(self->children,
+ (Obj*)Query_Make_Compiler(child_query, searcher, sub_boost));
+ }
+
+ return self;
+}
+
+void
+PolyCompiler_destroy(PolyCompiler *self) {
+ DECREF(self->children);
+ SUPER_DESTROY(self, POLYCOMPILER);
+}
+
+float
+PolyCompiler_sum_of_squared_weights(PolyCompiler *self) {
+ float sum = 0;
+ uint32_t i, max;
+ float my_boost = PolyCompiler_Get_Boost(self);
+
+ for (i = 0, max = VA_Get_Size(self->children); i < max; i++) {
+ Compiler *child = (Compiler*)VA_Fetch(self->children, i);
+ sum += Compiler_Sum_Of_Squared_Weights(child);
+ }
+
+ // Compound the weight of each child.
+ sum *= my_boost * my_boost;
+
+ return sum;
+}
+
+void
+PolyCompiler_apply_norm_factor(PolyCompiler *self, float factor) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->children); i < max; i++) {
+ Compiler *child = (Compiler*)VA_Fetch(self->children, i);
+ Compiler_Apply_Norm_Factor(child, factor);
+ }
+}
+
+VArray*
+PolyCompiler_highlight_spans(PolyCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field) {
+ VArray *spans = VA_new(0);
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(self->children); i < max; i++) {
+ Compiler *child = (Compiler*)VA_Fetch(self->children, i);
+ VArray *child_spans = Compiler_Highlight_Spans(child, searcher,
+ doc_vec, field);
+ if (child_spans) {
+ VA_Push_VArray(spans, child_spans);
+ VA_Dec_RefCount(child_spans);
+ }
+ }
+ return spans;
+}
+
+void
+PolyCompiler_serialize(PolyCompiler *self, OutStream *outstream) {
+ CB_Serialize(PolyCompiler_Get_Class_Name(self), outstream);
+ VA_Serialize(self->children, outstream);
+ Compiler_serialize((Compiler*)self, outstream);
+}
+
+PolyCompiler*
+PolyCompiler_deserialize(PolyCompiler *self, InStream *instream) {
+ CharBuf *class_name = CB_deserialize(NULL, instream);
+ if (!self) {
+ VTable *vtable = VTable_singleton(class_name, NULL);
+ self = (PolyCompiler*)VTable_Make_Obj(vtable);
+ }
+ DECREF(class_name);
+ self->children = VA_deserialize(NULL, instream);
+ return (PolyCompiler*)Compiler_deserialize((Compiler*)self, instream);
+}
+
+
diff --git a/core/Lucy/Search/PolyQuery.cfh b/core/Lucy/Search/PolyQuery.cfh
new file mode 100644
index 0000000..e693ebe
--- /dev/null
+++ b/core/Lucy/Search/PolyQuery.cfh
@@ -0,0 +1,98 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Base class for composite Query objects.
+ *
+ * PolyQuery serves as a shared base class for
+ * L<ANDQuery|Lucy::Search::ANDQuery>,
+ * L<ORQuery|Lucy::Search::ORQuery>,
+ * L<NOTQuery|Lucy::Search::NOTQuery>, and
+ * L<RequiredOptionalQuery|Lucy::Search::RequiredOptionalQuery>. All of
+ * these classes may serve as nodes in composite Query with a tree structure
+ * which may be walked.
+ */
+abstract class Lucy::Search::PolyQuery
+ inherits Lucy::Search::Query : dumpable {
+
+ VArray *children;
+
+ /**
+ * @param children An array of child Queries.
+ */
+ public inert PolyQuery*
+ init(PolyQuery *self, VArray *children = NULL);
+
+ /** Add a child Query node.
+ */
+ public void
+ Add_Child(PolyQuery *self, Query *query);
+
+ void
+ Set_Children(PolyQuery *self, VArray *children);
+
+ VArray*
+ Get_Children(PolyQuery *self);
+
+ public void
+ Serialize(PolyQuery *self, OutStream *outstream);
+
+ public incremented PolyQuery*
+ Deserialize(PolyQuery *self, InStream *instream);
+
+ public bool_t
+ Equals(PolyQuery *self, Obj *other);
+
+ public void
+ Destroy(PolyQuery *self);
+}
+
+class Lucy::Search::PolyCompiler inherits Lucy::Search::Compiler {
+
+ VArray *children;
+
+ inert incremented PolyCompiler*
+ new(PolyQuery *parent, Searcher *searcher, float boost);
+
+ /** Initialize the Compiler, creating a Compiler child for each child
+ * Query. Normalization is left to the subclass.
+ */
+ inert PolyCompiler*
+ init(PolyCompiler *self, PolyQuery *parent, Searcher *searcher,
+ float boost);
+
+ public float
+ Sum_Of_Squared_Weights(PolyCompiler *self);
+
+ public void
+ Apply_Norm_Factor(PolyCompiler *self, float factor);
+
+ public incremented VArray*
+ Highlight_Spans(PolyCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field);
+
+ public void
+ Destroy(PolyCompiler *self);
+
+ public void
+ Serialize(PolyCompiler *self, OutStream *outstream);
+
+ public incremented PolyCompiler*
+ Deserialize(PolyCompiler *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/PolySearcher.c b/core/Lucy/Search/PolySearcher.c
new file mode 100644
index 0000000..07737d9
--- /dev/null
+++ b/core/Lucy/Search/PolySearcher.c
@@ -0,0 +1,188 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_POLYSEARCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/PolySearcher.h"
+
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Collector.h"
+#include "Lucy/Search/HitQueue.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/MatchDoc.h"
+#include "Lucy/Search/Matcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/SortSpec.h"
+#include "Lucy/Search/TopDocs.h"
+#include "Lucy/Search/Compiler.h"
+
+PolySearcher*
+PolySearcher_init(PolySearcher *self, Schema *schema, VArray *searchers) {
+ const uint32_t num_searchers = VA_Get_Size(searchers);
+ uint32_t i;
+ int32_t *starts_array = (int32_t*)MALLOCATE(num_searchers * sizeof(int32_t));
+ int32_t doc_max = 0;
+
+ Searcher_init((Searcher*)self, schema);
+ self->searchers = (VArray*)INCREF(searchers);
+ self->starts = NULL; // Safe cleanup.
+
+ for (i = 0; i < num_searchers; i++) {
+ Searcher *searcher
+ = (Searcher*)CERTIFY(VA_Fetch(searchers, i), SEARCHER);
+ Schema *candidate = Searcher_Get_Schema(searcher);
+ VTable *orig_vt = Schema_Get_VTable(schema);
+ VTable *candidate_vt = Schema_Get_VTable(candidate);
+
+ // Confirm that searchers all use the same schema.
+ if (orig_vt != candidate_vt) {
+ THROW(ERR, "Conflicting schemas: '%o', '%o'",
+ Schema_Get_Class_Name(schema),
+ Schema_Get_Class_Name(candidate));
+ }
+
+ // Derive doc_max and relative start offsets.
+ starts_array[i] = (int32_t)doc_max;
+ doc_max += Searcher_Doc_Max(searcher);
+ }
+
+ self->doc_max = doc_max;
+ self->starts = I32Arr_new_steal(starts_array, num_searchers);
+
+ return self;
+}
+
+void
+PolySearcher_destroy(PolySearcher *self) {
+ DECREF(self->searchers);
+ DECREF(self->starts);
+ SUPER_DESTROY(self, POLYSEARCHER);
+}
+
+HitDoc*
+PolySearcher_fetch_doc(PolySearcher *self, int32_t doc_id) {
+ uint32_t tick = PolyReader_sub_tick(self->starts, doc_id);
+ Searcher *searcher = (Searcher*)VA_Fetch(self->searchers, tick);
+ int32_t offset = I32Arr_Get(self->starts, tick);
+ if (!searcher) { THROW(ERR, "Invalid doc id: %i32", doc_id); }
+ HitDoc *hit_doc = Searcher_Fetch_Doc(searcher, doc_id - offset);
+ HitDoc_Set_Doc_ID(hit_doc, doc_id);
+ return hit_doc;
+}
+
+DocVector*
+PolySearcher_fetch_doc_vec(PolySearcher *self, int32_t doc_id) {
+ uint32_t tick = PolyReader_sub_tick(self->starts, doc_id);
+ Searcher *searcher = (Searcher*)VA_Fetch(self->searchers, tick);
+ int32_t start = I32Arr_Get(self->starts, tick);
+ if (!searcher) { THROW(ERR, "Invalid doc id: %i32", doc_id); }
+ return Searcher_Fetch_Doc_Vec(searcher, doc_id - start);
+}
+
+int32_t
+PolySearcher_doc_max(PolySearcher *self) {
+ return self->doc_max;
+}
+
+uint32_t
+PolySearcher_doc_freq(PolySearcher *self, const CharBuf *field, Obj *term) {
+ uint32_t i, max;
+ uint32_t doc_freq = 0;
+ for (i = 0, max = VA_Get_Size(self->searchers); i < max; i++) {
+ Searcher *searcher = (Searcher*)VA_Fetch(self->searchers, i);
+ doc_freq += Searcher_Doc_Freq(searcher, field, term);
+ }
+ return doc_freq;
+}
+
+static void
+S_modify_doc_ids(VArray *match_docs, int32_t base) {
+ uint32_t i, max;
+ for (i = 0, max = VA_Get_Size(match_docs); i < max; i++) {
+ MatchDoc *match_doc = (MatchDoc*)VA_Fetch(match_docs, i);
+ int32_t new_doc_id = MatchDoc_Get_Doc_ID(match_doc) + base;
+ MatchDoc_Set_Doc_ID(match_doc, new_doc_id);
+ }
+}
+
+TopDocs*
+PolySearcher_top_docs(PolySearcher *self, Query *query, uint32_t num_wanted,
+ SortSpec *sort_spec) {
+ Schema *schema = PolySearcher_Get_Schema(self);
+ VArray *searchers = self->searchers;
+ I32Array *starts = self->starts;
+ HitQueue *hit_q = sort_spec
+ ? HitQ_new(schema, sort_spec, num_wanted)
+ : HitQ_new(NULL, NULL, num_wanted);
+ uint32_t total_hits = 0;
+ Compiler *compiler = Query_Is_A(query, COMPILER)
+ ? ((Compiler*)INCREF(query))
+ : Query_Make_Compiler(query, (Searcher*)self,
+ Query_Get_Boost(query));
+ uint32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(searchers); i < max; i++) {
+ Searcher *searcher = (Searcher*)VA_Fetch(searchers, i);
+ int32_t base = I32Arr_Get(starts, i);
+ TopDocs *top_docs = Searcher_Top_Docs(searcher, (Query*)compiler,
+ num_wanted, sort_spec);
+ VArray *sub_match_docs = TopDocs_Get_Match_Docs(top_docs);
+ uint32_t j, jmax;
+
+ total_hits += TopDocs_Get_Total_Hits(top_docs);
+
+ S_modify_doc_ids(sub_match_docs, base);
+ for (j = 0, jmax = VA_Get_Size(sub_match_docs); j < jmax; j++) {
+ MatchDoc *match_doc = (MatchDoc*)VA_Fetch(sub_match_docs, j);
+ if (!HitQ_Insert(hit_q, INCREF(match_doc))) { break; }
+ }
+
+ DECREF(top_docs);
+ }
+
+ {
+ VArray *match_docs = HitQ_Pop_All(hit_q);
+ TopDocs *retval = TopDocs_new(match_docs, total_hits);
+
+ DECREF(match_docs);
+ DECREF(compiler);
+ DECREF(hit_q);
+ return retval;
+ }
+}
+
+
+void
+PolySearcher_collect(PolySearcher *self, Query *query,
+ Collector *collector) {
+ uint32_t i, max;
+ VArray *const searchers = self->searchers;
+ I32Array *starts = self->starts;
+
+ for (i = 0, max = VA_Get_Size(searchers); i < max; i++) {
+ int32_t start = I32Arr_Get(starts, i);
+ Searcher *searcher = (Searcher*)VA_Fetch(searchers, i);
+ OffsetCollector *offset_coll = OffsetColl_new(collector, start);
+ Searcher_Collect(searcher, query, (Collector*)offset_coll);
+ DECREF(offset_coll);
+ }
+}
+
+
diff --git a/core/Lucy/Search/PolySearcher.cfh b/core/Lucy/Search/PolySearcher.cfh
new file mode 100644
index 0000000..e8980b9
--- /dev/null
+++ b/core/Lucy/Search/PolySearcher.cfh
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Aggregate results from multiple Searchers.
+ *
+ * The primary use for PolySearcher is to aggregate results from several
+ * remote L<Searchers|Lucy::Search::Searcher> via
+ * L<LucyX::Remote::SearchClient>, diffusing the cost of searching a large
+ * corpus over multiple machines. It is also possible to aggregate results
+ * from multiple Searchers on a single machine.
+ */
+
+class Lucy::Search::PolySearcher
+ inherits Lucy::Search::Searcher {
+
+ VArray *searchers;
+ I32Array *starts;
+ int32_t doc_max;
+
+ inert incremented PolySearcher*
+ new(Schema *schema, VArray *searchers);
+
+ /**
+ * @param schema A Schema.
+ * @param searchers An array of Searchers.
+ */
+ public inert PolySearcher*
+ init(PolySearcher *self, Schema *schema, VArray *searchers);
+
+ public void
+ Destroy(PolySearcher *self);
+
+ public int32_t
+ Doc_Max(PolySearcher *self);
+
+ public uint32_t
+ Doc_Freq(PolySearcher *self, const CharBuf *field, Obj *term);
+
+ public void
+ Collect(PolySearcher *self, Query *query, Collector *collector);
+
+ incremented TopDocs*
+ Top_Docs(PolySearcher *self, Query *query, uint32_t num_wanted,
+ SortSpec *sort_spec = NULL);
+
+ public incremented HitDoc*
+ Fetch_Doc(PolySearcher *self, int32_t doc_id);
+
+ incremented DocVector*
+ Fetch_Doc_Vec(PolySearcher *self, int32_t doc_id);
+}
+
+
diff --git a/core/Lucy/Search/Query.c b/core/Lucy/Search/Query.c
new file mode 100644
index 0000000..edfef17
--- /dev/null
+++ b/core/Lucy/Search/Query.c
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_QUERY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+Query*
+Query_init(Query *self, float boost) {
+ self->boost = boost;
+ ABSTRACT_CLASS_CHECK(self, QUERY);
+ return self;
+}
+
+void
+Query_set_boost(Query *self, float boost) {
+ self->boost = boost;
+}
+
+float
+Query_get_boost(Query *self) {
+ return self->boost;
+}
+
+void
+Query_serialize(Query *self, OutStream *outstream) {
+ OutStream_Write_F32(outstream, self->boost);
+}
+
+Query*
+Query_deserialize(Query *self, InStream *instream) {
+ float boost = InStream_Read_F32(instream);
+ self = self ? self : (Query*)VTable_Make_Obj(QUERY);
+ Query_init(self, boost);
+ return self;
+}
+
+
diff --git a/core/Lucy/Search/Query.cfh b/core/Lucy/Search/Query.cfh
new file mode 100644
index 0000000..d6ca358
--- /dev/null
+++ b/core/Lucy/Search/Query.cfh
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** A specification for a search query.
+ *
+ * Query objects are simple containers which contain the minimum information
+ * necessary to define a search query.
+ *
+ * The most common way to generate Query objects is to feed a search string
+ * such as 'foo AND bar' to a L<QueryParser's|Lucy::Search::QueryParser>
+ * Parse() method, which outputs an abstract syntax tree built up from various
+ * Query subclasses such as L<ANDQuery|Lucy::Search::ANDQuery> and
+ * L<TermQuery|Lucy::Search::TermQuery>. However, it is also possible
+ * to use custom Query objects to build a search specification which cannot be
+ * easily represented using a search string.
+ *
+ * Subclasses of Query must implement Make_Compiler(), which is the first step
+ * in compiling a Query down to a L<Matcher|Lucy::Search::Matcher> which
+ * can actually match and score documents.
+ */
+
+class Lucy::Search::Query inherits Lucy::Object::Obj : dumpable {
+
+ float boost;
+
+ /** Abstract constructor.
+ *
+ * @param boost A scoring multiplier, affecting the Query's relative
+ * contribution to each document's score. Typically defaults to 1.0, but
+ * subclasses which do not contribute to document scores such as NOTQuery
+ * and MatchAllQuery default to 0.0 instead.
+ */
+ public inert Query*
+ init(Query *self, float boost = 1.0);
+
+ /** Abstract factory method returning a Compiler derived from this Query.
+ *
+ * @param searcher A Searcher.
+ * @param boost A scoring multiplier. Defaults to the Query's own boost.
+ */
+ public abstract incremented Compiler*
+ Make_Compiler(Query *self, Searcher *searcher, float boost);
+
+ /** Set the Query's boost.
+ */
+ public void
+ Set_Boost(Query *self, float boost);
+
+ /** Get the Query's boost.
+ */
+ public float
+ Get_Boost(Query *self);
+
+ public void
+ Serialize(Query *self, OutStream *outstream);
+
+ public incremented Query*
+ Deserialize(Query *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/QueryParser.c b/core/Lucy/Search/QueryParser.c
new file mode 100644
index 0000000..572472f
--- /dev/null
+++ b/core/Lucy/Search/QueryParser.c
@@ -0,0 +1,1249 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_QUERYPARSER
+#define C_LUCY_PARSERCLAUSE
+#define C_LUCY_PARSERTOKEN
+#define C_LUCY_VIEWCHARBUF
+#include <stdlib.h>
+#include <ctype.h>
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/QueryParser.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/LeafQuery.h"
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Search/MatchAllQuery.h"
+#include "Lucy/Search/NoMatchQuery.h"
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Search/ORQuery.h"
+#include "Lucy/Search/PhraseQuery.h"
+#include "Lucy/Search/RequiredOptionalQuery.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Search/Query.h"
+
+#define SHOULD 0x00000001
+#define MUST 0x00000002
+#define MUST_NOT 0x00000004
+#define TOKEN_OPEN_PAREN 0x00000008
+#define TOKEN_CLOSE_PAREN 0x00000010
+#define TOKEN_MINUS 0x00000020
+#define TOKEN_PLUS 0x00000040
+#define TOKEN_NOT 0x00000080
+#define TOKEN_OR 0x00000100
+#define TOKEN_AND 0x00000200
+#define TOKEN_FIELD 0x00000400
+#define TOKEN_QUERY 0x00000800
+
+// Recursing helper function for Tree().
+static Query*
+S_do_tree(QueryParser *self, CharBuf *query_string, CharBuf *default_field,
+ Hash *extractions);
+
+// A function that attempts to match a substring and if successful, stores the
+// begin and end of the match in the supplied pointers and returns true.
+typedef bool_t
+(*lucy_QueryParser_match_t)(CharBuf *input, char **begin_match,
+ char **end_match);
+#define match_t lucy_QueryParser_match_t
+
+// Find a quote/end-of-string -delimited phrase.
+static bool_t
+S_match_phrase(CharBuf *input, char**begin_match, char **end_match);
+
+// Find a non-nested parethetical group.
+static bool_t
+S_match_bool_group(CharBuf *input, char**begin_match, char **end_match);
+
+// Replace whatever match() matches with a label, storing the matched text as
+// a CharBuf in the supplied storage Hash.
+static CharBuf*
+S_extract_something(QueryParser *self, const CharBuf *query_string,
+ CharBuf *label, Hash *extractions, match_t match);
+
+// Symbolically replace phrases in a query string.
+static CharBuf*
+S_extract_phrases(QueryParser *self, const CharBuf *query_string,
+ Hash *extractions);
+
+// Symbolically replace parenthetical groupings in a query string.
+static CharBuf*
+S_extract_paren_groups(QueryParser *self, const CharBuf *query_string,
+ Hash *extractions);
+
+// Consume text and possibly following whitespace, if there's a match and the
+// matching is bordered on the right by either whitespace or the end of the
+// string.
+static bool_t
+S_consume_ascii_token(ViewCharBuf *qstring, char *ptr, size_t size);
+
+// Consume the supplied text if there's a match.
+static bool_t
+S_consume_ascii(ViewCharBuf *qstring, char *ptr, size_t size);
+
+// Consume what looks like a field name followed by a colon.
+static bool_t
+S_consume_field(ViewCharBuf *qstring, ViewCharBuf *target);
+
+// Consume non-whitespace from qstring and store the match in target.
+static bool_t
+S_consume_non_whitespace(ViewCharBuf *qstring, ViewCharBuf *target);
+
+#define RAND_STRING_LEN 16
+#define PHRASE_LABEL_LEN (RAND_STRING_LEN + sizeof("_phrase") - 1)
+#define BOOL_GROUP_LABEL_LEN (RAND_STRING_LEN + sizeof("_bool_group") - 1)
+
+QueryParser*
+QParser_new(Schema *schema, Analyzer *analyzer, const CharBuf *default_boolop,
+ VArray *fields) {
+ QueryParser *self = (QueryParser*)VTable_Make_Obj(QUERYPARSER);
+ return QParser_init(self, schema, analyzer, default_boolop, fields);
+}
+
+QueryParser*
+QParser_init(QueryParser *self, Schema *schema, Analyzer *analyzer,
+ const CharBuf *default_boolop, VArray *fields) {
+ uint32_t i;
+
+ // Init.
+ self->heed_colons = false;
+ self->label_inc = 0;
+
+ // Assign.
+ self->schema = (Schema*)INCREF(schema);
+ self->analyzer = (Analyzer*)INCREF(analyzer);
+ self->default_boolop = default_boolop
+ ? CB_Clone(default_boolop)
+ : CB_new_from_trusted_utf8("OR", 2);
+
+ if (fields) {
+ self->fields = VA_Shallow_Copy(fields);
+ for (uint32_t i = 0, max = VA_Get_Size(fields); i < max; i++) {
+ CERTIFY(VA_Fetch(fields, i), CHARBUF);
+ }
+ VA_Sort(self->fields, NULL, NULL);
+ }
+ else {
+ VArray *all_fields = Schema_All_Fields(schema);
+ uint32_t num_fields = VA_Get_Size(all_fields);
+ self->fields = VA_new(num_fields);
+ for (uint32_t i = 0; i < num_fields; i++) {
+ CharBuf *field = (CharBuf*)VA_Fetch(all_fields, i);
+ FieldType *type = Schema_Fetch_Type(schema, field);
+ if (type && FType_Indexed(type)) {
+ VA_Push(self->fields, INCREF(field));
+ }
+ }
+ DECREF(all_fields);
+ }
+ VA_Sort(self->fields, NULL, NULL);
+
+ if (!(CB_Equals_Str(self->default_boolop, "OR", 2)
+ || CB_Equals_Str(self->default_boolop, "AND", 3))
+ ) {
+ THROW(ERR, "Invalid value for default_boolop: %o", self->default_boolop);
+ }
+
+ // Create string labels that presumably won't appear in a search.
+ self->phrase_label = CB_new_from_trusted_utf8("_phrase", 7);
+ self->bool_group_label = CB_new_from_trusted_utf8("_bool_group", 11);
+ CB_Grow(self->phrase_label, PHRASE_LABEL_LEN + 5);
+ CB_Grow(self->bool_group_label, BOOL_GROUP_LABEL_LEN + 5);
+ for (i = 0; i < RAND_STRING_LEN; i++) {
+ char rand_char = (rand() % 26) + 'A';
+ CB_Cat_Trusted_Str(self->phrase_label, &rand_char, 1);
+ CB_Cat_Trusted_Str(self->bool_group_label, &rand_char, 1);
+ }
+
+ return self;
+}
+
+void
+QParser_destroy(QueryParser *self) {
+ DECREF(self->schema);
+ DECREF(self->analyzer);
+ DECREF(self->default_boolop);
+ DECREF(self->fields);
+ DECREF(self->phrase_label);
+ DECREF(self->bool_group_label);
+ SUPER_DESTROY(self, QUERYPARSER);
+}
+
+Analyzer*
+QParser_get_analyzer(QueryParser *self) {
+ return self->analyzer;
+}
+
+Schema*
+QParser_get_schema(QueryParser *self) {
+ return self->schema;
+}
+
+CharBuf*
+QParser_get_default_boolop(QueryParser *self) {
+ return self->default_boolop;
+}
+
+VArray*
+QParser_get_fields(QueryParser *self) {
+ return self->fields;
+}
+
+bool_t
+QParser_heed_colons(QueryParser *self) {
+ return self->heed_colons;
+}
+
+void
+QParser_set_heed_colons(QueryParser *self, bool_t heed_colons) {
+ self->heed_colons = heed_colons;
+}
+
+
+Query*
+QParser_parse(QueryParser *self, const CharBuf *query_string) {
+ CharBuf *qstring = query_string
+ ? CB_Clone(query_string)
+ : CB_new_from_trusted_utf8("", 0);
+ Query *tree = QParser_Tree(self, qstring);
+ Query *expanded = QParser_Expand(self, tree);
+ Query *pruned = QParser_Prune(self, expanded);
+ DECREF(expanded);
+ DECREF(tree);
+ DECREF(qstring);
+ return pruned;
+}
+
+Query*
+QParser_tree(QueryParser *self, const CharBuf *query_string) {
+ Hash *extractions = Hash_new(0);
+ CharBuf *mod1 = S_extract_phrases(self, query_string, extractions);
+ CharBuf *mod2 = S_extract_paren_groups(self, mod1, extractions);
+ Query *retval = S_do_tree(self, mod2, NULL, extractions);
+ DECREF(mod2);
+ DECREF(mod1);
+ DECREF(extractions);
+ return retval;
+}
+
+static VArray*
+S_parse_flat_string(QueryParser *self, CharBuf *query_string) {
+ VArray *parse_tree = VA_new(0);
+ CharBuf *qstring_copy = CB_Clone(query_string);
+ ViewCharBuf *qstring = (ViewCharBuf*)ZCB_WRAP(qstring_copy);
+ bool_t need_close_paren = false;
+
+ ViewCB_Trim(qstring);
+
+ if (S_consume_ascii(qstring, "(", 1)) {
+ VA_Push(parse_tree, (Obj*)ParserToken_new(TOKEN_OPEN_PAREN, NULL, 0));
+ if (ViewCB_Code_Point_From(qstring, 1) == ')') {
+ need_close_paren = true;
+ ViewCB_Chop(qstring, 1);
+ }
+ }
+
+ ViewCharBuf *temp = (ViewCharBuf*)ZCB_BLANK();
+ while (ViewCB_Get_Size(qstring)) {
+ ParserToken *token = NULL;
+
+ if (ViewCB_Trim_Top(qstring)) {
+ // Fast-forward past whitespace.
+ continue;
+ }
+ else if (S_consume_ascii(qstring, "+", 1)) {
+ if (ViewCB_Trim_Top(qstring)) {
+ token = ParserToken_new(TOKEN_QUERY, "+", 1);
+ }
+ else {
+ token = ParserToken_new(TOKEN_PLUS, NULL, 0);
+ }
+ }
+ else if (S_consume_ascii(qstring, "-", 1)) {
+ if (ViewCB_Trim_Top(qstring)) {
+ token = ParserToken_new(TOKEN_QUERY, "-", 1);
+ }
+ else {
+ token = ParserToken_new(TOKEN_MINUS, NULL, 0);
+ }
+ }
+ else if (S_consume_ascii_token(qstring, "AND", 3)) {
+ token = ParserToken_new(TOKEN_AND, NULL, 0);
+ }
+ else if (S_consume_ascii_token(qstring, "OR", 2)) {
+ token = ParserToken_new(TOKEN_OR, NULL, 0);
+ }
+ else if (S_consume_ascii_token(qstring, "NOT", 3)) {
+ token = ParserToken_new(TOKEN_NOT, NULL, 0);
+ }
+ else if (self->heed_colons && S_consume_field(qstring, temp)) {
+ token = ParserToken_new(TOKEN_FIELD, (char*)ViewCB_Get_Ptr8(temp),
+ ViewCB_Get_Size(temp));
+ }
+ else if (S_consume_non_whitespace(qstring, temp)) {
+ token = ParserToken_new(TOKEN_QUERY, (char*)ViewCB_Get_Ptr8(temp),
+ ViewCB_Get_Size(temp));
+ }
+ else {
+ THROW(ERR, "Failed to parse '%o'", qstring);
+ }
+
+ VA_Push(parse_tree, (Obj*)token);
+ }
+
+ if (need_close_paren) {
+ VA_Push(parse_tree,
+ (Obj*)ParserToken_new(TOKEN_CLOSE_PAREN, NULL, 0));
+ }
+
+ // Clean up.
+ DECREF(qstring_copy);
+
+ return parse_tree;
+}
+
+static void
+S_splice_out_token_type(VArray *elems, uint32_t token_type_mask) {
+ for (uint32_t i = VA_Get_Size(elems); i--;) {
+ ParserToken *token = (ParserToken*)VA_Fetch(elems, i);
+ if (Obj_Is_A((Obj*)token, PARSERTOKEN)) {
+ if (token->type & token_type_mask) { VA_Excise(elems, i, 1); }
+ }
+ }
+}
+
+static Query*
+S_do_tree(QueryParser *self, CharBuf *query_string, CharBuf *default_field,
+ Hash *extractions) {
+ Query *retval;
+ bool_t apply_parens = false;
+ uint32_t default_occur = CB_Equals_Str(self->default_boolop, "AND", 3)
+ ? MUST
+ : SHOULD;
+ VArray *elems = S_parse_flat_string(self, query_string);
+ uint32_t i, max;
+
+ // Determine whether this subclause is bracketed by parens.
+ {
+ ParserToken *maybe_open_paren = (ParserToken*)VA_Fetch(elems, 0);
+ if (maybe_open_paren != NULL
+ && maybe_open_paren->type == TOKEN_OPEN_PAREN
+ ) {
+ uint32_t num_elems;
+ apply_parens = true;
+ VA_Excise(elems, 0, 1);
+ num_elems = VA_Get_Size(elems);
+ if (num_elems) {
+ ParserToken *maybe_close_paren
+ = (ParserToken*)VA_Fetch(elems, num_elems - 1);
+ if (maybe_close_paren->type == TOKEN_CLOSE_PAREN) {
+ VA_Excise(elems, num_elems - 1, 1);
+ }
+ }
+ }
+ }
+
+ // Generate all queries. Apply any fields.
+ for (i = VA_Get_Size(elems); i--;) {
+ CharBuf *field = default_field;
+ ParserToken *token = (ParserToken*)VA_Fetch(elems, i);
+
+ // Apply field.
+ if (i > 0) {
+ // Field specifier must immediately precede any query.
+ ParserToken* maybe_field_token
+ = (ParserToken*)VA_Fetch(elems, i - 1);
+ if (maybe_field_token->type == TOKEN_FIELD) {
+ field = maybe_field_token->text;
+ }
+ }
+
+ if (token->type == TOKEN_QUERY) {
+ // Generate a LeafQuery from a Phrase.
+ if (CB_Starts_With(token->text, self->phrase_label)) {
+ CharBuf *inner_text
+ = (CharBuf*)Hash_Fetch(extractions, (Obj*)token->text);
+ Query *query = (Query*)LeafQuery_new(field, inner_text);
+ ParserClause *clause = ParserClause_new(query, default_occur);
+ DECREF(Hash_Delete(extractions, (Obj*)token->text));
+ VA_Store(elems, i, (Obj*)clause);
+ DECREF(query);
+ }
+ // Recursively parse parenthetical groupings.
+ else if (CB_Starts_With(token->text, self->bool_group_label)) {
+ CharBuf *inner_text
+ = (CharBuf*)Hash_Fetch(extractions, (Obj*)token->text);
+ Query *query
+ = S_do_tree(self, inner_text, field, extractions);
+ DECREF(Hash_Delete(extractions, (Obj*)token->text));
+ if (query) {
+ ParserClause *clause
+ = ParserClause_new(query, default_occur);
+ VA_Store(elems, i, (Obj*)clause);
+ DECREF(query);
+ }
+ }
+ // What's left is probably a term, so generate a LeafQuery.
+ else {
+ Query *query = (Query*)LeafQuery_new(field, token->text);
+ ParserClause *clause = ParserClause_new(query, default_occur);
+ VA_Store(elems, i, (Obj*)clause);
+ DECREF(query);
+ }
+ }
+ }
+ S_splice_out_token_type(elems, TOKEN_FIELD | TOKEN_QUERY);
+
+ // Apply +, -, NOT.
+ for (i = VA_Get_Size(elems); i--;) {
+ ParserClause *clause = (ParserClause*)VA_Fetch(elems, i);
+ if (Obj_Is_A((Obj*)clause, PARSERCLAUSE)) {
+ uint32_t j;
+ for (j = i; j--;) {
+ ParserToken *token = (ParserToken*)VA_Fetch(elems, j);
+ if (Obj_Is_A((Obj*)token, PARSERTOKEN)) {
+ if (token->type == TOKEN_MINUS
+ || token->type == TOKEN_NOT
+ ) {
+ clause->occur = clause->occur == MUST_NOT
+ ? MUST
+ : MUST_NOT;
+ }
+ else if (token->type == TOKEN_PLUS) {
+ if (clause->occur == SHOULD) {
+ clause->occur = MUST;
+ }
+ }
+ }
+ else {
+ break;
+ }
+ }
+ }
+ }
+ S_splice_out_token_type(elems, TOKEN_PLUS | TOKEN_MINUS | TOKEN_NOT);
+
+ // Wrap negated queries with NOTQuery objects.
+ for (i = 0, max = VA_Get_Size(elems); i < max; i++) {
+ ParserClause *clause = (ParserClause*)VA_Fetch(elems, i);
+ if (Obj_Is_A((Obj*)clause, PARSERCLAUSE) && clause->occur == MUST_NOT) {
+ Query *not_query = QParser_Make_NOT_Query(self, clause->query);
+ DECREF(clause->query);
+ clause->query = not_query;
+ }
+ }
+
+ // Silently discard non-sensical combos of AND and OR, e.g.
+ // 'OR a AND AND OR b AND'.
+ for (i = 0; i < VA_Get_Size(elems); i++) {
+ ParserToken *token = (ParserToken*)VA_Fetch(elems, i);
+ if (Obj_Is_A((Obj*)token, PARSERTOKEN)) {
+ uint32_t j, jmax;
+ uint32_t num_to_zap = 0;
+ ParserClause *preceding = (ParserClause*)VA_Fetch(elems, i - 1);
+ ParserClause *following = (ParserClause*)VA_Fetch(elems, i + 1);
+ if (!preceding || !Obj_Is_A((Obj*)preceding, PARSERCLAUSE)) {
+ num_to_zap = 1;
+ }
+ if (!following || !Obj_Is_A((Obj*)following, PARSERCLAUSE)) {
+ num_to_zap = 1;
+ }
+ for (j = i + 1, jmax = VA_Get_Size(elems); j < jmax; j++) {
+ ParserClause *clause = (ParserClause*)VA_Fetch(elems, j);
+ if (Obj_Is_A((Obj*)clause, PARSERCLAUSE)) { break; }
+ else { num_to_zap++; }
+ }
+ if (num_to_zap) { VA_Excise(elems, i, num_to_zap); }
+ }
+ }
+
+ // Apply AND.
+ for (i = 0; i + 2 < VA_Get_Size(elems); i++) {
+ ParserToken *token = (ParserToken*)VA_Fetch(elems, i + 1);
+ if (Obj_Is_A((Obj*)token, PARSERTOKEN) && token->type == TOKEN_AND) {
+ ParserClause *preceding = (ParserClause*)VA_Fetch(elems, i);
+ VArray *children = VA_new(2);
+ uint32_t num_to_zap = 0;
+ uint32_t j, jmax;
+
+ // Add first clause.
+ VA_Push(children, INCREF(preceding->query));
+
+ // Add following clauses.
+ for (j = i + 1, jmax = VA_Get_Size(elems);
+ j < jmax;
+ j += 2, num_to_zap += 2
+ ) {
+ ParserToken *maybe_and = (ParserToken*)VA_Fetch(elems, j);
+ ParserClause *following
+ = (ParserClause*)VA_Fetch(elems, j + 1);
+ if (!Obj_Is_A((Obj*)maybe_and, PARSERTOKEN)
+ || maybe_and->type != TOKEN_AND
+ ) {
+ break;
+ }
+ else {
+ CERTIFY(following, PARSERCLAUSE);
+ }
+ VA_Push(children, INCREF(following->query));
+ }
+ DECREF(preceding->query);
+ preceding->query = QParser_Make_AND_Query(self, children);
+ preceding->occur = default_occur;
+ DECREF(children);
+
+ VA_Excise(elems, i + 1, num_to_zap);
+
+ // Don't double wrap '(a AND b)'.
+ if (VA_Get_Size(elems) == 1) { apply_parens = false; }
+ }
+ }
+
+ // Apply OR.
+ for (i = 0; i + 2 < VA_Get_Size(elems); i++) {
+ ParserToken *token = (ParserToken*)VA_Fetch(elems, i + 1);
+ if (Obj_Is_A((Obj*)token, PARSERTOKEN) && token->type == TOKEN_OR) {
+ ParserClause *preceding = (ParserClause*)VA_Fetch(elems, i);
+ VArray *children = VA_new(2);
+ uint32_t num_to_zap = 0;
+ uint32_t j, jmax;
+
+ // Add first clause.
+ VA_Push(children, INCREF(preceding->query));
+
+ // Add following clauses.
+ for (j = i + 1, jmax = VA_Get_Size(elems);
+ j < jmax;
+ j += 2, num_to_zap += 2
+ ) {
+ ParserToken *maybe_or = (ParserToken*)VA_Fetch(elems, j);
+ ParserClause *following
+ = (ParserClause*)VA_Fetch(elems, j + 1);
+ if (!Obj_Is_A((Obj*)maybe_or, PARSERTOKEN)
+ || maybe_or->type != TOKEN_OR
+ ) {
+ break;
+ }
+ else {
+ CERTIFY(following, PARSERCLAUSE);
+ }
+ VA_Push(children, INCREF(following->query));
+ }
+ DECREF(preceding->query);
+ preceding->query = QParser_Make_OR_Query(self, children);
+ preceding->occur = default_occur;
+ DECREF(children);
+
+ VA_Excise(elems, i + 1, num_to_zap);
+
+ // Don't double wrap '(a OR b)'.
+ if (VA_Get_Size(elems) == 1) { apply_parens = false; }
+ }
+ }
+
+ if (VA_Get_Size(elems) == 0) {
+ // No elems means no query. Maybe the search string was something
+ // like 'NOT AND'
+ if (apply_parens) {
+ retval = default_occur == SHOULD
+ ? QParser_Make_OR_Query(self, NULL)
+ : QParser_Make_AND_Query(self, NULL);
+ }
+ else {
+ retval = (Query*)NoMatchQuery_new();
+ }
+ }
+ else if (VA_Get_Size(elems) == 1 && !apply_parens) {
+ ParserClause *clause = (ParserClause*)VA_Fetch(elems, 0);
+ retval = (Query*)INCREF(clause->query);
+ }
+ else {
+ uint32_t num_elems = VA_Get_Size(elems);
+ VArray *required = VA_new(num_elems);
+ VArray *optional = VA_new(num_elems);
+ VArray *negated = VA_new(num_elems);
+ Query *req_query = NULL;
+ Query *opt_query = NULL;
+ uint32_t i, num_required, num_negated, num_optional;
+
+ // Demux elems into bins.
+ for (i = 0; i < num_elems; i++) {
+ ParserClause *clause = (ParserClause*)VA_Fetch(elems, i);
+ if (clause->occur == MUST) {
+ VA_Push(required, INCREF(clause->query));
+ }
+ else if (clause->occur == SHOULD) {
+ VA_Push(optional, INCREF(clause->query));
+ }
+ else if (clause->occur == MUST_NOT) {
+ VA_Push(negated, INCREF(clause->query));
+ }
+ }
+ num_required = VA_Get_Size(required);
+ num_negated = VA_Get_Size(negated);
+ num_optional = VA_Get_Size(optional);
+
+ // Bind all mandatory matchers together in one Query.
+ if (num_required || num_negated) {
+ if (apply_parens || num_required + num_negated > 1) {
+ VArray *children = VA_Shallow_Copy(required);
+ VA_Push_VArray(children, negated);
+ req_query = QParser_Make_AND_Query(self, children);
+ DECREF(children);
+ }
+ else if (num_required) {
+ req_query = (Query*)INCREF(VA_Fetch(required, 0));
+ }
+ else if (num_negated) {
+ req_query = (Query*)INCREF(VA_Fetch(negated, 0));
+ }
+ }
+
+ // Bind all optional matchers together in one Query.
+ if (num_optional) {
+ if (!apply_parens && num_optional == 1) {
+ opt_query = (Query*)INCREF(VA_Fetch(optional, 0));
+ }
+ else {
+ opt_query = QParser_Make_OR_Query(self, optional);
+ }
+ }
+
+ // Unify required and optional.
+ if (req_query && opt_query) {
+ if (num_required) { // not just negated elems
+ retval = QParser_Make_Req_Opt_Query(self, req_query,
+ opt_query);
+ }
+ else {
+ // req_query has only negated queries.
+ VArray *children = VA_new(2);
+ VA_Push(children, INCREF(req_query));
+ VA_Push(children, INCREF(opt_query));
+ retval = QParser_Make_AND_Query(self, children);
+ DECREF(children);
+ }
+ }
+ else if (opt_query) {
+ // Only optional elems.
+ retval = (Query*)INCREF(opt_query);
+ }
+ else if (req_query) {
+ // Only required elems.
+ retval = (Query*)INCREF(req_query);
+ }
+ else {
+ retval = NULL; // kill "uninitialized" compiler warning
+ THROW(ERR, "Unexpected error");
+ }
+
+ DECREF(opt_query);
+ DECREF(req_query);
+ DECREF(negated);
+ DECREF(optional);
+ DECREF(required);
+ }
+
+ DECREF(elems);
+
+ return retval;
+}
+
+static bool_t
+S_has_valid_clauses(Query *query) {
+ if (Query_Is_A(query, NOTQUERY)) {
+ return false;
+ }
+ else if (Query_Is_A(query, MATCHALLQUERY)) {
+ return false;
+ }
+ else if (Query_Is_A(query, ORQUERY) || Query_Is_A(query, ANDQUERY)) {
+ PolyQuery *polyquery = (PolyQuery*)query;
+ VArray *children = PolyQuery_Get_Children(polyquery);
+ for (uint32_t i = 0, max = VA_Get_Size(children); i < max; i++) {
+ Query *child = (Query*)VA_Fetch(children, i);
+ if (S_has_valid_clauses(child)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ return true;
+}
+
+static void
+S_do_prune(QueryParser *self, Query *query) {
+ if (Query_Is_A(query, NOTQUERY)) {
+ // Don't allow double negatives.
+ NOTQuery *not_query = (NOTQuery*)query;
+ Query *neg_query = NOTQuery_Get_Negated_Query(not_query);
+ if (!Query_Is_A(neg_query, MATCHALLQUERY)
+ && !S_has_valid_clauses(neg_query)
+ ) {
+ MatchAllQuery *matchall = MatchAllQuery_new();
+ NOTQuery_Set_Negated_Query(not_query, (Query*)matchall);
+ DECREF(matchall);
+ }
+ }
+ else if (Query_Is_A(query, POLYQUERY)) {
+ PolyQuery *polyquery = (PolyQuery*)query;
+ VArray *children = PolyQuery_Get_Children(polyquery);
+
+ // Recurse.
+ for (uint32_t i = 0, max = VA_Get_Size(children); i < max; i++) {
+ Query *child = (Query*)VA_Fetch(children, i);
+ S_do_prune(self, child);
+ }
+
+ if (PolyQuery_Is_A(polyquery, REQUIREDOPTIONALQUERY)
+ || PolyQuery_Is_A(polyquery, ORQUERY)
+ ) {
+ // Don't allow 'foo OR (-bar)'.
+ VArray *children = PolyQuery_Get_Children(polyquery);
+ for (uint32_t i = 0, max = VA_Get_Size(children); i < max; i++) {
+ Query *child = (Query*)VA_Fetch(children, i);
+ if (!S_has_valid_clauses(child)) {
+ VA_Store(children, i, (Obj*)NoMatchQuery_new());
+ }
+ }
+ }
+ else if (PolyQuery_Is_A(polyquery, ANDQUERY)) {
+ // Don't allow '(-bar AND -baz)'.
+ if (!S_has_valid_clauses((Query*)polyquery)) {
+ VArray *children = PolyQuery_Get_Children(polyquery);
+ VA_Clear(children);
+ }
+ }
+ }
+}
+
+Query*
+QParser_prune(QueryParser *self, Query *query) {
+ if (!query
+ || Query_Is_A(query, NOTQUERY)
+ || Query_Is_A(query, MATCHALLQUERY)
+ ) {
+ return (Query*)NoMatchQuery_new();
+ }
+ else if (Query_Is_A(query, POLYQUERY)) {
+ S_do_prune(self, query);
+ }
+ return (Query*)INCREF(query);
+}
+
+static bool_t
+S_consume_ascii(ViewCharBuf *qstring, char *ptr, size_t len) {
+ if (ViewCB_Starts_With_Str(qstring, ptr, len)) {
+ ViewCB_Nip(qstring, len);
+ return true;
+ }
+ return false;
+}
+
+static bool_t
+S_consume_ascii_token(ViewCharBuf *qstring, char *ptr, size_t len) {
+ if (ViewCB_Starts_With_Str(qstring, ptr, len)) {
+ if (len == ViewCB_Get_Size(qstring)
+ || StrHelp_is_whitespace(ViewCB_Code_Point_At(qstring, len))
+ ) {
+ ViewCB_Nip(qstring, len);
+ ViewCB_Trim_Top(qstring);
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool_t
+S_consume_field(ViewCharBuf *qstring, ViewCharBuf *target) {
+ size_t tick = 0;
+
+ // Field names constructs must start with a letter or underscore.
+ uint32_t code_point = ViewCB_Code_Point_At(qstring, tick);
+ if (isalpha(code_point) || code_point == '_') {
+ tick++;
+ }
+ else {
+ return false;
+ }
+
+ // Only alphanumerics and underscores are allowed in field names.
+ while (1) {
+ code_point = ViewCB_Code_Point_At(qstring, tick);
+ if (isalnum(code_point) || code_point == '_') {
+ tick++;
+ }
+ else if (code_point == ':') {
+ tick++;
+ break;
+ }
+ else {
+ return false;
+ }
+ }
+
+ // Field name constructs must be followed by something sensible.
+ uint32_t lookahead = ViewCB_Code_Point_At(qstring, tick);
+ if (!(isalnum(lookahead)
+ || lookahead == '_'
+ || lookahead > 127
+ || lookahead == '"'
+ || lookahead == '('
+ )
+ ) {
+ return false;
+ }
+
+ // Consume string data.
+ ViewCB_Assign(target, (CharBuf*)qstring);
+ ViewCB_Set_Size(target, tick - 1);
+ ViewCB_Nip(qstring, tick);
+ return true;
+}
+
+static bool_t
+S_consume_non_whitespace(ViewCharBuf *qstring, ViewCharBuf *target) {
+ uint32_t code_point = ViewCB_Code_Point_At(qstring, 0);
+ bool_t success = false;
+ ViewCB_Assign(target, (CharBuf*)qstring);
+ while (code_point && !StrHelp_is_whitespace(code_point)) {
+ ViewCB_Nip_One(qstring);
+ code_point = ViewCB_Code_Point_At(qstring, 0);
+ success = true;
+ }
+ if (!success) {
+ return false;
+ }
+ else {
+ uint32_t new_size = ViewCB_Get_Size(target) - ViewCB_Get_Size(qstring);
+ ViewCB_Set_Size(target, new_size);
+ return true;
+ }
+}
+
+Query*
+QParser_expand(QueryParser *self, Query *query) {
+ Query *retval = NULL;
+
+ if (Query_Is_A(query, LEAFQUERY)) {
+ retval = QParser_Expand_Leaf(self, query);
+ }
+ else if (Query_Is_A(query, ORQUERY) || Query_Is_A(query, ANDQUERY)) {
+ PolyQuery *polyquery = (PolyQuery*)query;
+ VArray *children = PolyQuery_Get_Children(polyquery);
+ VArray *new_kids = VA_new(VA_Get_Size(children));
+
+ for (uint32_t i = 0, max = VA_Get_Size(children); i < max; i++) {
+ Query *child = (Query*)VA_Fetch(children, i);
+ Query *new_child = QParser_Expand(self, child); // recurse
+ if (new_child) {
+ if (Query_Is_A(new_child, NOMATCHQUERY)) {
+ bool_t fails = NoMatchQuery_Get_Fails_To_Match(
+ (NoMatchQuery*)new_child);
+ if (fails) {
+ VA_Push(new_kids, (Obj*)new_child);
+ }
+ else {
+ DECREF(new_child);
+ }
+ }
+ else {
+ VA_Push(new_kids, (Obj*)new_child);
+ }
+ }
+ }
+
+ if (VA_Get_Size(new_kids) == 0) {
+ retval = (Query*)NoMatchQuery_new();
+ }
+ else if (VA_Get_Size(new_kids) == 1) {
+ retval = (Query*)INCREF(VA_Fetch(new_kids, 0));
+ }
+ else {
+ PolyQuery_Set_Children(polyquery, new_kids);
+ retval = (Query*)INCREF(query);
+ }
+
+ DECREF(new_kids);
+ }
+ else if (Query_Is_A(query, NOTQUERY)) {
+ NOTQuery *not_query = (NOTQuery*)query;
+ Query *negated_query = NOTQuery_Get_Negated_Query(not_query);
+ negated_query = QParser_Expand(self, negated_query);
+ if (negated_query) {
+ NOTQuery_Set_Negated_Query(not_query, negated_query);
+ DECREF(negated_query);
+ retval = (Query*)INCREF(query);
+ }
+ else {
+ retval = (Query*)MatchAllQuery_new();
+ }
+ }
+ else if (Query_Is_A(query, REQUIREDOPTIONALQUERY)) {
+ RequiredOptionalQuery *req_opt_query = (RequiredOptionalQuery*)query;
+ Query *req_query = ReqOptQuery_Get_Required_Query(req_opt_query);
+ Query *opt_query = ReqOptQuery_Get_Optional_Query(req_opt_query);
+
+ req_query = QParser_Expand(self, req_query);
+ opt_query = QParser_Expand(self, opt_query);
+
+ if (req_query && opt_query) {
+ ReqOptQuery_Set_Required_Query(req_opt_query, req_query);
+ ReqOptQuery_Set_Optional_Query(req_opt_query, opt_query);
+ retval = (Query*)INCREF(query);
+ }
+ else if (req_query) { retval = (Query*)INCREF(req_query); }
+ else if (opt_query) { retval = (Query*)INCREF(opt_query); }
+ else { retval = (Query*)NoMatchQuery_new(); }
+
+ DECREF(opt_query);
+ DECREF(req_query);
+ }
+ else {
+ retval = (Query*)INCREF(query);
+ }
+
+ return retval;
+}
+
+static CharBuf*
+S_unescape(QueryParser *self, CharBuf *orig, CharBuf *target) {
+ ZombieCharBuf *source = ZCB_WRAP(orig);
+ uint32_t code_point;
+ UNUSED_VAR(self);
+
+ CB_Set_Size(target, 0);
+ CB_Grow(target, CB_Get_Size(orig) + 4);
+
+ while (0 != (code_point = ZCB_Nip_One(source))) {
+ if (code_point == '\\') {
+ uint32_t next_code_point = ZCB_Nip_One(source);
+ if (next_code_point == ':'
+ || next_code_point == '"'
+ || next_code_point == '\\'
+ ) {
+ CB_Cat_Char(target, next_code_point);
+ }
+ else {
+ CB_Cat_Char(target, code_point);
+ if (next_code_point) { CB_Cat_Char(target, next_code_point); }
+ }
+ }
+ else {
+ CB_Cat_Char(target, code_point);
+ }
+ }
+
+ return target;
+}
+
+Query*
+QParser_expand_leaf(QueryParser *self, Query *query) {
+ LeafQuery *leaf_query = (LeafQuery*)query;
+ Schema *schema = self->schema;
+ ZombieCharBuf *source_text = ZCB_BLANK();
+ bool_t is_phrase = false;
+ bool_t ambiguous = false;
+
+ // Determine whether we can actually process the input.
+ if (!Query_Is_A(query, LEAFQUERY)) { return NULL; }
+ if (!CB_Get_Size(LeafQuery_Get_Text(leaf_query))) { return NULL; }
+ ZCB_Assign(source_text, LeafQuery_Get_Text(leaf_query));
+
+ // If quoted, always generate PhraseQuery.
+ ZCB_Trim(source_text);
+ if (ZCB_Code_Point_At(source_text, 0) == '"') {
+ is_phrase = true;
+ ZCB_Nip(source_text, 1);
+ if (ZCB_Code_Point_From(source_text, 1) == '"'
+ && ZCB_Code_Point_From(source_text, 2) != '\\'
+ ) {
+ ZCB_Chop(source_text, 1);
+ }
+ }
+
+ // Either use LeafQuery's field or default to Parser's list.
+ VArray *fields;
+ if (LeafQuery_Get_Field(leaf_query)) {
+ fields = VA_new(1);
+ VA_Push(fields, INCREF(LeafQuery_Get_Field(leaf_query)));
+ }
+ else {
+ fields = (VArray*)INCREF(self->fields);
+ }
+
+ CharBuf *unescaped = CB_new(ZCB_Get_Size(source_text));
+ VArray *queries = VA_new(VA_Get_Size(fields));
+ for (uint32_t i = 0, max = VA_Get_Size(fields); i < max; i++) {
+ CharBuf *field = (CharBuf*)VA_Fetch(fields, i);
+ Analyzer *analyzer = self->analyzer
+ ? self->analyzer
+ : Schema_Fetch_Analyzer(schema, field);
+
+ if (!analyzer) {
+ VA_Push(queries,
+ (Obj*)QParser_Make_Term_Query(self, field,
+ (Obj*)source_text));
+ }
+ else {
+ // Extract token texts.
+ CharBuf *split_source
+ = S_unescape(self, (CharBuf*)source_text, unescaped);
+ VArray *maybe_texts = Analyzer_Split(analyzer, split_source);
+ uint32_t num_maybe_texts = VA_Get_Size(maybe_texts);
+ VArray *token_texts = VA_new(num_maybe_texts);
+
+ // Filter out zero-length token texts.
+ for (uint32_t j = 0; j < num_maybe_texts; j++) {
+ CharBuf *token_text = (CharBuf*)VA_Fetch(maybe_texts, j);
+ if (CB_Get_Size(token_text)) {
+ VA_Push(token_texts, INCREF(token_text));
+ }
+ }
+
+ if (VA_Get_Size(token_texts) == 0) {
+ /* Query might include stop words. Who knows? */
+ ambiguous = true;
+ }
+
+ // Add either a TermQuery or a PhraseQuery.
+ if (is_phrase || VA_Get_Size(token_texts) > 1) {
+ VA_Push(queries, (Obj*)
+ QParser_Make_Phrase_Query(self, field, token_texts));
+ }
+ else if (VA_Get_Size(token_texts) == 1) {
+ VA_Push(queries,
+ (Obj*)QParser_Make_Term_Query(self, field, VA_Fetch(token_texts, 0)));
+ }
+
+ DECREF(token_texts);
+ DECREF(maybe_texts);
+ }
+ }
+
+ Query *retval;
+ if (VA_Get_Size(queries) == 0) {
+ retval = (Query*)NoMatchQuery_new();
+ if (ambiguous) {
+ NoMatchQuery_Set_Fails_To_Match((NoMatchQuery*)retval, false);
+ }
+ }
+ else if (VA_Get_Size(queries) == 1) {
+ retval = (Query*)INCREF(VA_Fetch(queries, 0));
+ }
+ else {
+ retval = QParser_Make_OR_Query(self, queries);
+ }
+
+ // Clean up.
+ DECREF(unescaped);
+ DECREF(queries);
+ DECREF(fields);
+
+ return retval;
+}
+
+static CharBuf*
+S_extract_something(QueryParser *self, const CharBuf *query_string,
+ CharBuf *label, Hash *extractions, match_t match) {
+ CharBuf *retval = CB_Clone(query_string);
+ size_t qstring_size = CB_Get_Size(query_string);
+ size_t orig_label_size = CB_Get_Size(label);
+ char *begin_match;
+ char *end_match;
+
+ while (match(retval, &begin_match, &end_match)) {
+ size_t len = end_match - begin_match;
+ size_t retval_size = CB_Get_Size(retval);
+ char *retval_buf = (char*)CB_Get_Ptr8(retval);
+ char *retval_end = retval_buf + retval_size;
+ size_t before_match = begin_match - retval_buf;
+ size_t after_match = retval_end - end_match;
+ CharBuf *new_retval = CB_new(qstring_size);
+
+ // Store inner text.
+ CB_catf(label, "%u32", self->label_inc++);
+ Hash_Store(extractions, (Obj*)label,
+ (Obj*)CB_new_from_utf8(begin_match, len));
+
+ // Splice the label into the query string.
+ CB_Cat_Str(new_retval, retval_buf, before_match);
+ CB_Cat(new_retval, label);
+ CB_Cat_Str(new_retval, " ", 1); // Extra space for safety.
+ CB_Cat_Str(new_retval, end_match, after_match);
+ DECREF(retval);
+ retval = new_retval;
+ CB_Set_Size(label, orig_label_size);
+ }
+
+ return retval;
+}
+
+static CharBuf*
+S_extract_phrases(QueryParser *self, const CharBuf *query_string,
+ Hash *extractions) {
+ return S_extract_something(self, query_string, self->phrase_label,
+ extractions, S_match_phrase);
+}
+
+static bool_t
+S_match_phrase(CharBuf *input, char**begin_match, char **end_match) {
+ ZombieCharBuf *iterator = ZCB_WRAP(input);
+ uint32_t code_point;
+
+ while (0 != (code_point = ZCB_Code_Point_At(iterator, 0))) {
+ if (code_point == '\\') {
+ ZCB_Nip(iterator, 2);
+ continue;
+ }
+ if (code_point == '"') {
+ *begin_match = (char*)ZCB_Get_Ptr8(iterator);
+ *end_match = *begin_match + ZCB_Get_Size(iterator);
+ ZCB_Nip_One(iterator);
+ while (0 != (code_point = ZCB_Nip_One(iterator))) {
+ if (code_point == '\\') {
+ ZCB_Nip_One(iterator);
+ continue;
+ }
+ else if (code_point == '"') {
+ *end_match = (char*)ZCB_Get_Ptr8(iterator);
+ return true;
+ }
+ }
+ return true;
+ }
+ ZCB_Nip_One(iterator);
+ }
+ return false;
+}
+
+static CharBuf*
+S_extract_paren_groups(QueryParser *self, const CharBuf *query_string,
+ Hash *extractions) {
+ return S_extract_something(self, query_string, self->bool_group_label,
+ extractions, S_match_bool_group);
+}
+
+static bool_t
+S_match_bool_group(CharBuf *input, char**begin_match, char **end_match) {
+ ZombieCharBuf *iterator = ZCB_WRAP(input);
+ uint32_t code_point;
+
+ while (0 != (code_point = ZCB_Code_Point_At(iterator, 0))) {
+ if (code_point == '(') {
+FOUND_OPEN_PAREN:
+ *begin_match = (char*)ZCB_Get_Ptr8(iterator);
+ *end_match = *begin_match + ZCB_Get_Size(iterator);
+ ZCB_Nip_One(iterator);
+ while (0 != (code_point = ZCB_Code_Point_At(iterator, 0))) {
+ if (code_point == '(') { goto FOUND_OPEN_PAREN; }
+ ZCB_Nip_One(iterator);
+ if (code_point == ')') {
+ *end_match = (char*)ZCB_Get_Ptr8(iterator);
+ return true;
+ }
+ }
+ return true;
+ }
+ ZCB_Nip_One(iterator);
+ }
+ return false;
+}
+
+Query*
+QParser_make_term_query(QueryParser *self, const CharBuf *field, Obj *term) {
+ UNUSED_VAR(self);
+ return (Query*)TermQuery_new(field, term);
+}
+
+Query*
+QParser_make_phrase_query(QueryParser *self, const CharBuf *field,
+ VArray *terms) {
+ UNUSED_VAR(self);
+ return (Query*)PhraseQuery_new(field, terms);
+}
+
+Query*
+QParser_make_or_query(QueryParser *self, VArray *children) {
+ UNUSED_VAR(self);
+ return (Query*)ORQuery_new(children);
+}
+
+Query*
+QParser_make_and_query(QueryParser *self, VArray *children) {
+ UNUSED_VAR(self);
+ return (Query*)ANDQuery_new(children);
+}
+
+Query*
+QParser_make_not_query(QueryParser *self, Query *negated_query) {
+ UNUSED_VAR(self);
+ return (Query*)NOTQuery_new(negated_query);
+}
+
+Query*
+QParser_make_req_opt_query(QueryParser *self, Query *required_query,
+ Query *optional_query) {
+ UNUSED_VAR(self);
+ return (Query*)ReqOptQuery_new(required_query, optional_query);
+}
+
+/********************************************************************/
+
+ParserClause*
+ParserClause_new(Query *query, uint32_t occur) {
+ ParserClause *self = (ParserClause*)VTable_Make_Obj(PARSERCLAUSE);
+ return ParserClause_init(self, query, occur);
+}
+
+ParserClause*
+ParserClause_init(ParserClause *self, Query *query, uint32_t occur) {
+ self->query = (Query*)INCREF(query);
+ self->occur = occur;
+ return self;
+}
+
+void
+ParserClause_destroy(ParserClause *self) {
+ DECREF(self->query);
+ SUPER_DESTROY(self, PARSERCLAUSE);
+}
+
+/********************************************************************/
+
+ParserToken*
+ParserToken_new(uint32_t type, const char *text, size_t len) {
+ ParserToken *self = (ParserToken*)VTable_Make_Obj(PARSERTOKEN);
+ return ParserToken_init(self, type, text, len);
+}
+
+ParserToken*
+ParserToken_init(ParserToken *self, uint32_t type, const char *text,
+ size_t len) {
+ self->type = type;
+ self->text = text ? CB_new_from_utf8(text, len) : NULL;
+ return self;
+}
+
+void
+ParserToken_destroy(ParserToken *self) {
+ DECREF(self->text);
+ SUPER_DESTROY(self, PARSERTOKEN);
+}
+
+
diff --git a/core/Lucy/Search/QueryParser.cfh b/core/Lucy/Search/QueryParser.cfh
new file mode 100644
index 0000000..c2c377e
--- /dev/null
+++ b/core/Lucy/Search/QueryParser.cfh
@@ -0,0 +1,265 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Transform a string into a Query object.
+ *
+ * QueryParser accepts search strings as input and produces
+ * L<Lucy::Search::Query> objects, suitable for feeding into
+ * L<IndexSearcher|Lucy::Search::IndexSearcher> and other
+ * L<Searcher|Lucy::Search::Searcher> subclasses.
+ *
+ * The following syntactical constructs are recognized by QueryParser:
+ *
+ * * Boolean operators 'AND', 'OR', and 'AND NOT'.
+ * * Prepented +plus and -minus, indicating that the labeled entity
+ * should be either required or forbidden -- be it a single word, a
+ * phrase, or a parenthetical group.
+ * * Logical groups, delimited by parentheses.
+ * * Phrases, delimited by double quotes.
+ *
+ * Additionally, the following syntax can be enabled via Set_Heed_Colons():
+ *
+ * * Field-specific constructs, in the form of 'fieldname:termtext' or
+ * 'fieldname:(foo bar)'. (The field specified by 'fieldname:' will be
+ * used instead of the QueryParser's default fields).
+ *
+ *
+ */
+class Lucy::Search::QueryParser cnick QParser
+ inherits Lucy::Object::Obj {
+
+ Schema *schema;
+ Analyzer *analyzer;
+ CharBuf *default_boolop;
+ VArray *fields;
+ CharBuf *phrase_label;
+ CharBuf *bool_group_label;
+ bool_t heed_colons;
+ uint32_t label_inc;
+
+ inert incremented QueryParser*
+ new(Schema *schema, Analyzer *analyzer = NULL,
+ const CharBuf *default_boolop = NULL, VArray *fields = NULL);
+
+ /** Constructor.
+ *
+ * @param schema A L<Schema|Lucy::Plan::Schema>.
+ * @param analyzer An L<Analyzer|Lucy::Analysis::Analyzer>.
+ * Ordinarily, the analyzers specified by each field's definition will be
+ * used, but if C<analyzer> is supplied, it will override and be used for
+ * all fields. This can lead to mismatches between what is in the index
+ * and what is being searched for, so use caution.
+ * @param fields The names of the fields which will be searched against.
+ * Defaults to those fields which are defined as indexed in the supplied
+ * Schema.
+ * @param default_boolop Two possible values: 'AND' and 'OR'. The default
+ * is 'OR', which means: return documents which match any of the query
+ * terms. If you want only documents which match all of the query terms,
+ * set this to 'AND'.
+ */
+ public inert QueryParser*
+ init(QueryParser *self, Schema *schema, Analyzer *analyzer = NULL,
+ const CharBuf *default_boolop = NULL, VArray *fields = NULL);
+
+ /** Build a Query object from the contents of a query string. At present,
+ * implemented internally by calling Tree(), Expand(), and Prune().
+ *
+ * @param query_string The string to be parsed. May be NULL.
+ * @return a Query.
+ */
+ public incremented Query*
+ Parse(QueryParser *self, const CharBuf *query_string = NULL);
+
+ /** Parse the logical structure of a query string, building a tree
+ * comprised of Query objects. Leaf nodes in the tree will most often be
+ * LeafQuery objects but might be MatchAllQuery or NoMatchQuery objects as
+ * well. Internal nodes will be objects which subclass PolyQuery:
+ * ANDQuery, ORQuery, NOTQuery, and RequiredOptionalQuery.
+ *
+ * The output of Tree() is an intermediate form which must be passed
+ * through Expand() before being used to feed a search.
+ *
+ * @param query_string The string to be parsed.
+ * @return a Query.
+ */
+ public incremented Query*
+ Tree(QueryParser *self, const CharBuf *query_string);
+
+ /** Walk the hierarchy of a Query tree, descending through all PolyQuery
+ * nodes and calling Expand_Leaf() on any LeafQuery nodes encountered.
+ *
+ * @param query A Query object.
+ * @return A Query -- usually the same one that was supplied after
+ * in-place modification, but possibly another.
+ */
+ public incremented Query*
+ Expand(QueryParser *self, Query *query);
+
+ /** Convert a LeafQuery into either a TermQuery, a PhraseQuery, or an
+ * ORQuery joining multiple TermQueries/PhraseQueries to accommodate
+ * multiple fields. LeafQuery text will be passed through the relevant
+ * Analyzer for each field. Quoted text will be transformed into
+ * PhraseQuery objects. Unquoted text will be converted to either a
+ * TermQuery or a PhraseQuery depending on how many tokens are generated.
+ *
+ * @param query A Query. Only LeafQuery objects will be processed; others
+ * will be passed through.
+ * @return A Query.
+ */
+ public incremented Query*
+ Expand_Leaf(QueryParser *self, Query *query);
+
+ /** Prevent certain Query structures from returning too many results.
+ * Query objects built via Tree() and Expand() can generate "return the
+ * world" result sets, such as in the case of
+ * <code>NOT a_term_not_in_the_index</code>; Prune() walks the hierarchy
+ * and eliminates such branches.
+ *
+ * 'NOT foo' => [NOMATCH]
+ * 'foo OR NOT bar' => 'foo'
+ * 'foo OR (-bar AND -baz) => 'foo'
+ *
+ * Prune() also eliminates some double-negative constructs -- even though
+ * such constructs may not actually return the world:
+ *
+ * 'foo AND -(-bar)' => 'foo'
+ *
+ * In this example, safety is taking precedence over logical consistency.
+ * If you want logical consistency instead, call Tree() then Expand(),
+ * skipping Prune().
+ *
+ *
+ * @param query A Query.
+ * @return a Query; in most cases, the supplied Query after in-place
+ * modification.
+ */
+ public incremented Query*
+ Prune(QueryParser *self, Query *query = NULL);
+
+ /** Factory method creating a TermQuery.
+ *
+ * @param field Field name.
+ * @param term Term text.
+ * @return A Query.
+ */
+ public incremented Query*
+ Make_Term_Query(QueryParser *self, const CharBuf *field, Obj *term);
+
+ /** Factory method creating a PhraseQuery.
+ *
+ * @param field Field that the phrase must occur in.
+ * @param terms Ordered array of terms that must match.
+ * @return A Query.
+ */
+ public incremented Query*
+ Make_Phrase_Query(QueryParser *self, const CharBuf *field, VArray *terms);
+
+ /** Factory method creating an ORQuery.
+ *
+ * @param children Array of child Queries.
+ * @return A Query.
+ */
+ public incremented Query*
+ Make_OR_Query(QueryParser *self, VArray *children = NULL);
+
+ /** Factory method creating an ANDQuery.
+ *
+ * @param children Array of child Queries.
+ * @return A Query.
+ */
+ public incremented Query*
+ Make_AND_Query(QueryParser *self, VArray *children = NULL);
+
+ /** Factory method creating a NOTQuery.
+ *
+ * @param negated_query Query to be inverted.
+ * @return A Query.
+ */
+ public incremented Query*
+ Make_NOT_Query(QueryParser *self, Query *negated_query);
+
+ /** Factory method creating a RequiredOptionalQuery.
+ *
+ * @param required_query Query must must match.
+ * @param optional_query Query which should match.
+ * @return A Query.
+ */
+ public incremented Query*
+ Make_Req_Opt_Query(QueryParser *self, Query *required_query,
+ Query *optional_query);
+
+ nullable Analyzer*
+ Get_Analyzer(QueryParser *self);
+
+ Schema*
+ Get_Schema(QueryParser *self);
+
+ CharBuf*
+ Get_Default_BoolOp(QueryParser *self);
+
+ VArray*
+ Get_Fields(QueryParser *self);
+
+ bool_t
+ Heed_Colons(QueryParser *self);
+
+ /** Enable/disable parsing of <code>fieldname:foo</code> constructs.
+ */
+ public void
+ Set_Heed_Colons(QueryParser *self, bool_t heed_colons);
+
+ public void
+ Destroy(QueryParser *self);
+}
+
+/** Private utility class.
+ */
+class Lucy::QueryParser::ParserClause inherits Lucy::Object::Obj {
+
+ uint32_t occur;
+ Query *query;
+
+ inert incremented ParserClause*
+ new(Query *query, uint32_t occur);
+
+ inert ParserClause*
+ init(ParserClause *self, Query *query, uint32_t occur);
+
+ public void
+ Destroy(ParserClause *self);
+}
+
+/** Private utility class.
+ */
+class Lucy::QueryParser::ParserToken inherits Lucy::Object::Obj {
+
+ uint32_t type;
+ CharBuf *text;
+
+ inert incremented ParserToken*
+ new(uint32_t type, const char *text = NULL, size_t len = 0);
+
+ inert ParserToken*
+ init(ParserToken *self, uint32_t type, const char *text = NULL,
+ size_t len = 0);
+
+ public void
+ Destroy(ParserToken *self);
+}
+
+
diff --git a/core/Lucy/Search/RangeMatcher.c b/core/Lucy/Search/RangeMatcher.c
new file mode 100644
index 0000000..811a19b
--- /dev/null
+++ b/core/Lucy/Search/RangeMatcher.c
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RANGEMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/RangeMatcher.h"
+#include "Lucy/Index/SortCache.h"
+
+RangeMatcher*
+RangeMatcher_new(int32_t lower_bound, int32_t upper_bound, SortCache *sort_cache,
+ int32_t doc_max) {
+ RangeMatcher *self = (RangeMatcher*)VTable_Make_Obj(RANGEMATCHER);
+ return RangeMatcher_init(self, lower_bound, upper_bound, sort_cache,
+ doc_max);
+}
+
+RangeMatcher*
+RangeMatcher_init(RangeMatcher *self, int32_t lower_bound, int32_t upper_bound,
+ SortCache *sort_cache, int32_t doc_max) {
+ Matcher_init((Matcher*)self);
+
+ // Init.
+ self->doc_id = 0;
+
+ // Assign.
+ self->lower_bound = lower_bound;
+ self->upper_bound = upper_bound;
+ self->sort_cache = (SortCache*)INCREF(sort_cache);
+ self->doc_max = doc_max;
+
+ // Derive.
+
+ return self;
+}
+
+void
+RangeMatcher_destroy(RangeMatcher *self) {
+ DECREF(self->sort_cache);
+ SUPER_DESTROY(self, RANGEMATCHER);
+}
+
+int32_t
+RangeMatcher_next(RangeMatcher* self) {
+ while (1) {
+ if (++self->doc_id > self->doc_max) {
+ self->doc_id--;
+ return 0;
+ }
+ else {
+ // Check if ord for this document is within the specied range.
+ // TODO: Unroll? i.e. use SortCache_Get_Ords at constructor time
+ // and save ourselves some method call overhead.
+ const int32_t ord
+ = SortCache_Ordinal(self->sort_cache, self->doc_id);
+ if (ord >= self->lower_bound && ord <= self->upper_bound) {
+ break;
+ }
+ }
+ }
+ return self->doc_id;
+}
+
+int32_t
+RangeMatcher_advance(RangeMatcher* self, int32_t target) {
+ self->doc_id = target - 1;
+ return RangeMatcher_next(self);
+}
+
+float
+RangeMatcher_score(RangeMatcher* self) {
+ UNUSED_VAR(self);
+ return 0.0f;
+}
+
+int32_t
+RangeMatcher_get_doc_id(RangeMatcher* self) {
+ return self->doc_id;
+}
+
+
diff --git a/core/Lucy/Search/RangeMatcher.cfh b/core/Lucy/Search/RangeMatcher.cfh
new file mode 100644
index 0000000..4f8367c
--- /dev/null
+++ b/core/Lucy/Search/RangeMatcher.cfh
@@ -0,0 +1,51 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Search::RangeMatcher inherits Lucy::Search::Matcher {
+
+ int32_t doc_id;
+ int32_t doc_max;
+ int32_t lower_bound;
+ int32_t upper_bound;
+ SortCache *sort_cache;
+
+ inert incremented RangeMatcher*
+ new(int32_t lower_bound, int32_t upper_bound, SortCache *sort_cache,
+ int32_t doc_max);
+
+ inert RangeMatcher*
+ init(RangeMatcher *self, int32_t lower_bound, int32_t upper_bound,
+ SortCache *sort_cache, int32_t doc_max);
+
+ public int32_t
+ Next(RangeMatcher *self);
+
+ public int32_t
+ Advance(RangeMatcher *self, int32_t target);
+
+ public float
+ Score(RangeMatcher* self);
+
+ public int32_t
+ Get_Doc_ID(RangeMatcher* self);
+
+ public void
+ Destroy(RangeMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/RangeQuery.c b/core/Lucy/Search/RangeQuery.c
new file mode 100644
index 0000000..f0857ef
--- /dev/null
+++ b/core/Lucy/Search/RangeQuery.c
@@ -0,0 +1,276 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RANGEQUERY
+#define C_LUCY_RANGECOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/RangeQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/SortReader.h"
+#include "Lucy/Index/SortCache.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/RangeMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+// Determine the lowest ordinal that should match.
+static int32_t
+S_find_lower_bound(RangeCompiler *self, SortCache *sort_cache);
+
+// Determine the highest ordinal that should match.
+static int32_t
+S_find_upper_bound(RangeCompiler *self, SortCache *sort_cache);
+
+RangeQuery*
+RangeQuery_new(const CharBuf *field, Obj *lower_term, Obj *upper_term,
+ bool_t include_lower, bool_t include_upper) {
+ RangeQuery *self = (RangeQuery*)VTable_Make_Obj(RANGEQUERY);
+ return RangeQuery_init(self, field, lower_term, upper_term,
+ include_lower, include_upper);
+}
+
+RangeQuery*
+RangeQuery_init(RangeQuery *self, const CharBuf *field, Obj *lower_term,
+ Obj *upper_term, bool_t include_lower, bool_t include_upper) {
+ Query_init((Query*)self, 0.0f);
+ self->field = CB_Clone(field);
+ self->lower_term = lower_term ? Obj_Clone(lower_term) : NULL;
+ self->upper_term = upper_term ? Obj_Clone(upper_term) : NULL;
+ self->include_lower = include_lower;
+ self->include_upper = include_upper;
+ if (!upper_term && !lower_term) {
+ DECREF(self);
+ self = NULL;
+ THROW(ERR, "Must supply at least one of 'upper_term' and 'lower_term'");
+ }
+ return self;
+}
+
+void
+RangeQuery_destroy(RangeQuery *self) {
+ DECREF(self->field);
+ DECREF(self->lower_term);
+ DECREF(self->upper_term);
+ SUPER_DESTROY(self, RANGEQUERY);
+}
+
+bool_t
+RangeQuery_equals(RangeQuery *self, Obj *other) {
+ RangeQuery *twin = (RangeQuery*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, RANGEQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!CB_Equals(self->field, (Obj*)twin->field)) { return false; }
+ if (self->lower_term && !twin->lower_term) { return false; }
+ if (self->upper_term && !twin->upper_term) { return false; }
+ if (!self->lower_term && twin->lower_term) { return false; }
+ if (!self->upper_term && twin->upper_term) { return false; }
+ if (self->lower_term
+ && !Obj_Equals(self->lower_term, twin->lower_term)) { return false; }
+ if (self->upper_term
+ && !Obj_Equals(self->upper_term, twin->upper_term)) { return false; }
+ if (self->include_lower != twin->include_lower) { return false; }
+ if (self->include_upper != twin->include_upper) { return false; }
+ return true;
+}
+
+CharBuf*
+RangeQuery_to_string(RangeQuery *self) {
+ CharBuf *lower_term_str = self->lower_term
+ ? Obj_To_String(self->lower_term)
+ : CB_new_from_trusted_utf8("*", 1);
+ CharBuf *upper_term_str = self->upper_term
+ ? Obj_To_String(self->upper_term)
+ : CB_new_from_trusted_utf8("*", 1);
+ CharBuf *retval = CB_newf("%o:%s%o TO %o%s", self->field,
+ self->include_lower ? "[" : "{",
+ lower_term_str,
+ upper_term_str,
+ self->include_upper ? "]" : "}"
+ );
+ DECREF(upper_term_str);
+ DECREF(lower_term_str);
+ return retval;
+}
+
+void
+RangeQuery_serialize(RangeQuery *self, OutStream *outstream) {
+ OutStream_Write_F32(outstream, self->boost);
+ CB_Serialize(self->field, outstream);
+ if (self->lower_term) {
+ OutStream_Write_U8(outstream, true);
+ FREEZE(self->lower_term, outstream);
+ }
+ else {
+ OutStream_Write_U8(outstream, false);
+ }
+ if (self->upper_term) {
+ OutStream_Write_U8(outstream, true);
+ FREEZE(self->upper_term, outstream);
+ }
+ else {
+ OutStream_Write_U8(outstream, false);
+ }
+ OutStream_Write_U8(outstream, self->include_lower);
+ OutStream_Write_U8(outstream, self->include_upper);
+}
+
+RangeQuery*
+RangeQuery_deserialize(RangeQuery *self, InStream *instream) {
+ // Deserialize components.
+ float boost = InStream_Read_F32(instream);
+ CharBuf *field = CB_deserialize(NULL, instream);
+ Obj *lower_term = InStream_Read_U8(instream) ? THAW(instream) : NULL;
+ Obj *upper_term = InStream_Read_U8(instream) ? THAW(instream) : NULL;
+ bool_t include_lower = InStream_Read_U8(instream);
+ bool_t include_upper = InStream_Read_U8(instream);
+
+ // Init object.
+ self = self ? self : (RangeQuery*)VTable_Make_Obj(RANGEQUERY);
+ RangeQuery_init(self, field, lower_term, upper_term, include_lower,
+ include_upper);
+ RangeQuery_Set_Boost(self, boost);
+
+ DECREF(upper_term);
+ DECREF(lower_term);
+ DECREF(field);
+ return self;
+}
+
+RangeCompiler*
+RangeQuery_make_compiler(RangeQuery *self, Searcher *searcher,
+ float boost) {
+ return RangeCompiler_new(self, searcher, boost);
+}
+
+/**********************************************************************/
+
+RangeCompiler*
+RangeCompiler_new(RangeQuery *parent, Searcher *searcher, float boost) {
+ RangeCompiler *self
+ = (RangeCompiler*)VTable_Make_Obj(RANGECOMPILER);
+ return RangeCompiler_init(self, parent, searcher, boost);
+}
+
+RangeCompiler*
+RangeCompiler_init(RangeCompiler *self, RangeQuery *parent,
+ Searcher *searcher, float boost) {
+ return (RangeCompiler*)Compiler_init((Compiler*)self, (Query*)parent,
+ searcher, NULL, boost);
+}
+
+RangeCompiler*
+RangeCompiler_deserialize(RangeCompiler *self, InStream *instream) {
+ self = self ? self : (RangeCompiler*)VTable_Make_Obj(RANGECOMPILER);
+ return (RangeCompiler*)Compiler_deserialize((Compiler*)self, instream);
+}
+
+Matcher*
+RangeCompiler_make_matcher(RangeCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ RangeQuery *parent = (RangeQuery*)self->parent;
+ SortReader *sort_reader
+ = (SortReader*)SegReader_Fetch(reader, VTable_Get_Name(SORTREADER));
+ SortCache *sort_cache = sort_reader
+ ? SortReader_Fetch_Sort_Cache(sort_reader, parent->field)
+ : NULL;
+ UNUSED_VAR(need_score);
+
+ if (!sort_cache) {
+ return NULL;
+ }
+ else {
+ int32_t lower = S_find_lower_bound(self, sort_cache);
+ int32_t upper = S_find_upper_bound(self, sort_cache);
+ int32_t max_ord = SortCache_Get_Cardinality(sort_cache) + 1;
+ if (lower > max_ord || upper < 0) {
+ return NULL;
+ }
+ else {
+ int32_t doc_max = SegReader_Doc_Max(reader);
+ return (Matcher*)RangeMatcher_new(lower, upper, sort_cache,
+ doc_max);
+ }
+ }
+}
+
+static int32_t
+S_find_lower_bound(RangeCompiler *self, SortCache *sort_cache) {
+ RangeQuery *parent = (RangeQuery*)self->parent;
+ Obj *lower_term = parent->lower_term;
+ int32_t lower_bound = 0;
+
+ if (lower_term) {
+ int32_t low_ord = SortCache_Find(sort_cache, lower_term);
+ if (low_ord < 0) {
+ // The supplied term is lower than all terms in the field.
+ lower_bound = 0;
+ }
+ else {
+ Obj *value = SortCache_Make_Blank(sort_cache);
+ Obj *low_found = SortCache_Value(sort_cache, low_ord, value);
+ bool_t exact_match = low_found == NULL
+ ? false
+ : Obj_Equals(lower_term, low_found);
+
+ lower_bound = low_ord;
+ if (!exact_match || !parent->include_lower) {
+ lower_bound++;
+ }
+ DECREF(value);
+ }
+ }
+
+ return lower_bound;
+}
+
+static int32_t
+S_find_upper_bound(RangeCompiler *self, SortCache *sort_cache) {
+ RangeQuery *parent = (RangeQuery*)self->parent;
+ Obj *upper_term = parent->upper_term;
+ int32_t retval = I32_MAX;
+
+ if (upper_term) {
+ int32_t hi_ord = SortCache_Find(sort_cache, upper_term);
+ if (hi_ord < 0) {
+ // The supplied term is lower than all terms in the field.
+ retval = -1;
+ }
+ else {
+ Obj *value = SortCache_Make_Blank(sort_cache);
+ Obj *hi_found = SortCache_Value(sort_cache, hi_ord, value);
+ bool_t exact_match = hi_found == NULL
+ ? false
+ : Obj_Equals(upper_term, (Obj*)hi_found);
+
+ retval = hi_ord;
+ if (exact_match && !parent->include_upper) {
+ retval--;
+ }
+ DECREF(value);
+ }
+ }
+
+ return retval;
+}
+
+
diff --git a/core/Lucy/Search/RangeQuery.cfh b/core/Lucy/Search/RangeQuery.cfh
new file mode 100644
index 0000000..88f227d
--- /dev/null
+++ b/core/Lucy/Search/RangeQuery.cfh
@@ -0,0 +1,92 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Match a range of values.
+ *
+ * RangeQuery matches documents where the value for a particular field falls
+ * within a given range.
+ */
+
+class Lucy::Search::RangeQuery inherits Lucy::Search::Query
+ : dumpable {
+
+ CharBuf *field;
+ Obj *lower_term;
+ Obj *upper_term;
+ bool_t include_lower;
+ bool_t include_upper;
+
+ inert incremented RangeQuery*
+ new(const CharBuf *field, Obj *lower_term = NULL, Obj *upper_term = NULL,
+ bool_t include_lower = true, bool_t include_upper = true);
+
+ /** Takes 5 parameters; <code>field</code> is required, as
+ * is at least one of either <code>lower_term</code> or
+ * <code>upper_term</code>.
+ *
+ * @param field The name of a <code>sortable</code> field.
+ * @param lower_term Lower delimiter. If not supplied, all values
+ * less than <code>upper_term</code> will pass.
+ * @param upper_term Upper delimiter. If not supplied, all values greater
+ * than <code>lower_term</code> will pass.
+ * @param include_lower Indicates whether docs which match
+ * <code>lower_term</code> should be included in the results.
+ * @param include_upper Indicates whether docs which match
+ * <code>upper_term</code> should be included in the results.
+ */
+ public inert RangeQuery*
+ init(RangeQuery *self, const CharBuf *field,
+ Obj *lower_term = NULL, Obj *upper_term = NULL,
+ bool_t include_lower = true, bool_t include_upper = true);
+
+ public bool_t
+ Equals(RangeQuery *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(RangeQuery *self);
+
+ public incremented RangeCompiler*
+ Make_Compiler(RangeQuery *self, Searcher *searcher, float boost);
+
+ public void
+ Serialize(RangeQuery *self, OutStream *outstream);
+
+ public incremented RangeQuery*
+ Deserialize(RangeQuery *self, InStream *instream);
+
+ public void
+ Destroy(RangeQuery *self);
+}
+
+class Lucy::Search::RangeCompiler inherits Lucy::Search::Compiler {
+
+ inert incremented RangeCompiler*
+ new(RangeQuery *parent, Searcher *searcher, float boost);
+
+ inert RangeCompiler*
+ init(RangeCompiler *self, RangeQuery *parent, Searcher *searcher,
+ float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(RangeCompiler *self, SegReader *reader, bool_t need_score);
+
+ public incremented RangeCompiler*
+ Deserialize(RangeCompiler *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/RequiredOptionalMatcher.c b/core/Lucy/Search/RequiredOptionalMatcher.c
new file mode 100644
index 0000000..0ebde69
--- /dev/null
+++ b/core/Lucy/Search/RequiredOptionalMatcher.c
@@ -0,0 +1,112 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_REQUIREDOPTIONALMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/RequiredOptionalMatcher.h"
+#include "Lucy/Index/Similarity.h"
+
+RequiredOptionalMatcher*
+ReqOptMatcher_new(Similarity *similarity, Matcher *required_matcher,
+ Matcher *optional_matcher) {
+ RequiredOptionalMatcher *self
+ = (RequiredOptionalMatcher*)VTable_Make_Obj(REQUIREDOPTIONALMATCHER);
+ return ReqOptMatcher_init(self, similarity, required_matcher,
+ optional_matcher);
+}
+
+RequiredOptionalMatcher*
+ReqOptMatcher_init(RequiredOptionalMatcher *self, Similarity *similarity,
+ Matcher *required_matcher, Matcher *optional_matcher) {
+ VArray *children = VA_new(2);
+ VA_Push(children, INCREF(required_matcher));
+ VA_Push(children, INCREF(optional_matcher));
+ PolyMatcher_init((PolyMatcher*)self, children, similarity);
+
+ // Assign.
+ self->req_matcher = (Matcher*)INCREF(required_matcher);
+ self->opt_matcher = (Matcher*)INCREF(optional_matcher);
+
+ // Init.
+ self->opt_matcher_first_time = true;
+
+ DECREF(children);
+ return self;
+}
+
+void
+ReqOptMatcher_destroy(RequiredOptionalMatcher *self) {
+ DECREF(self->req_matcher);
+ DECREF(self->opt_matcher);
+ SUPER_DESTROY(self, REQUIREDOPTIONALMATCHER);
+}
+
+int32_t
+ReqOptMatcher_next(RequiredOptionalMatcher *self) {
+ return Matcher_Next(self->req_matcher);
+}
+
+int32_t
+ReqOptMatcher_advance(RequiredOptionalMatcher *self, int32_t target) {
+ return Matcher_Advance(self->req_matcher, target);
+}
+
+int32_t
+ReqOptMatcher_get_doc_id(RequiredOptionalMatcher *self) {
+ return Matcher_Get_Doc_ID(self->req_matcher);
+}
+
+float
+ReqOptMatcher_score(RequiredOptionalMatcher *self) {
+ int32_t const current_doc = Matcher_Get_Doc_ID(self->req_matcher);
+
+ if (self->opt_matcher_first_time) {
+ self->opt_matcher_first_time = false;
+ if (!Matcher_Advance(self->opt_matcher, current_doc)) {
+ DECREF(self->opt_matcher);
+ self->opt_matcher = NULL;
+ }
+ }
+
+ if (self->opt_matcher == NULL) {
+ return Matcher_Score(self->req_matcher);
+ }
+ else {
+ int32_t opt_matcher_doc = Matcher_Get_Doc_ID(self->opt_matcher);
+
+ if (opt_matcher_doc < current_doc) {
+ opt_matcher_doc = Matcher_Advance(self->opt_matcher, current_doc);
+ if (!opt_matcher_doc) {
+ DECREF(self->opt_matcher);
+ self->opt_matcher = NULL;
+ return Matcher_Score(self->req_matcher);
+ }
+ }
+
+ if (opt_matcher_doc == current_doc) {
+ float score = Matcher_Score(self->req_matcher)
+ + Matcher_Score(self->opt_matcher);
+ score *= self->coord_factors[2];
+ return score;
+ }
+ else {
+ return Matcher_Score(self->req_matcher);
+ }
+ }
+}
+
+
diff --git a/core/Lucy/Search/RequiredOptionalMatcher.cfh b/core/Lucy/Search/RequiredOptionalMatcher.cfh
new file mode 100644
index 0000000..6294ffb
--- /dev/null
+++ b/core/Lucy/Search/RequiredOptionalMatcher.cfh
@@ -0,0 +1,53 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Intersect required and optional Matchers.
+ */
+
+class Lucy::Search::RequiredOptionalMatcher cnick ReqOptMatcher
+ inherits Lucy::Search::PolyMatcher {
+
+ Matcher *req_matcher;
+ Matcher *opt_matcher;
+ bool_t opt_matcher_first_time;
+
+ inert incremented RequiredOptionalMatcher*
+ new(Similarity *similarity, Matcher *required_matcher,
+ Matcher *optional_matcher);
+
+ inert RequiredOptionalMatcher*
+ init(RequiredOptionalMatcher *self, Similarity *similarity,
+ Matcher *required_matcher, Matcher *optional_matcher);
+
+ public void
+ Destroy(RequiredOptionalMatcher *self);
+
+ public int32_t
+ Next(RequiredOptionalMatcher *self);
+
+ public int32_t
+ Advance(RequiredOptionalMatcher *self, int32_t target);
+
+ public float
+ Score(RequiredOptionalMatcher *self);
+
+ public int32_t
+ Get_Doc_ID(RequiredOptionalMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/RequiredOptionalQuery.c b/core/Lucy/Search/RequiredOptionalQuery.c
new file mode 100644
index 0000000..3b14774
--- /dev/null
+++ b/core/Lucy/Search/RequiredOptionalQuery.c
@@ -0,0 +1,139 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_REQUIREDOPTIONALQUERY
+#define C_LUCY_REQUIREDOPTIONALCOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/RequiredOptionalQuery.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/RequiredOptionalMatcher.h"
+#include "Lucy/Search/Searcher.h"
+
+RequiredOptionalQuery*
+ReqOptQuery_new(Query *required_query, Query *optional_query) {
+ RequiredOptionalQuery *self
+ = (RequiredOptionalQuery*)VTable_Make_Obj(REQUIREDOPTIONALQUERY);
+ return ReqOptQuery_init(self, required_query, optional_query);
+}
+
+RequiredOptionalQuery*
+ReqOptQuery_init(RequiredOptionalQuery *self, Query *required_query,
+ Query *optional_query) {
+ PolyQuery_init((PolyQuery*)self, NULL);
+ VA_Push(self->children, INCREF(required_query));
+ VA_Push(self->children, INCREF(optional_query));
+ return self;
+}
+
+Query*
+ReqOptQuery_get_required_query(RequiredOptionalQuery *self) {
+ return (Query*)VA_Fetch(self->children, 0);
+}
+
+void
+ReqOptQuery_set_required_query(RequiredOptionalQuery *self,
+ Query *required_query) {
+ VA_Store(self->children, 0, INCREF(required_query));
+}
+
+Query*
+ReqOptQuery_get_optional_query(RequiredOptionalQuery *self) {
+ return (Query*)VA_Fetch(self->children, 1);
+}
+
+void
+ReqOptQuery_set_optional_query(RequiredOptionalQuery *self,
+ Query *optional_query) {
+ VA_Store(self->children, 1, INCREF(optional_query));
+}
+
+CharBuf*
+ReqOptQuery_to_string(RequiredOptionalQuery *self) {
+ CharBuf *req_string = Obj_To_String(VA_Fetch(self->children, 0));
+ CharBuf *opt_string = Obj_To_String(VA_Fetch(self->children, 1));
+ CharBuf *retval = CB_newf("(+%o %o)", req_string, opt_string);
+ DECREF(opt_string);
+ DECREF(req_string);
+ return retval;
+}
+
+bool_t
+ReqOptQuery_equals(RequiredOptionalQuery *self, Obj *other) {
+ if ((RequiredOptionalQuery*)other == self) { return true; }
+ if (!Obj_Is_A(other, REQUIREDOPTIONALQUERY)) { return false; }
+ return PolyQuery_equals((PolyQuery*)self, other);
+}
+
+Compiler*
+ReqOptQuery_make_compiler(RequiredOptionalQuery *self, Searcher *searcher,
+ float boost) {
+ return (Compiler*)ReqOptCompiler_new(self, searcher, boost);
+}
+
+/**********************************************************************/
+
+RequiredOptionalCompiler*
+ReqOptCompiler_new(RequiredOptionalQuery *parent, Searcher *searcher,
+ float boost) {
+ RequiredOptionalCompiler *self
+ = (RequiredOptionalCompiler*)VTable_Make_Obj(
+ REQUIREDOPTIONALCOMPILER);
+ return ReqOptCompiler_init(self, parent, searcher, boost);
+}
+
+RequiredOptionalCompiler*
+ReqOptCompiler_init(RequiredOptionalCompiler *self,
+ RequiredOptionalQuery *parent,
+ Searcher *searcher, float boost) {
+ PolyCompiler_init((PolyCompiler*)self, (PolyQuery*)parent, searcher,
+ boost);
+ ReqOptCompiler_Normalize(self);
+ return self;
+}
+
+Matcher*
+ReqOptCompiler_make_matcher(RequiredOptionalCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ Schema *schema = SegReader_Get_Schema(reader);
+ Similarity *sim = Schema_Get_Similarity(schema);
+ Compiler *req_compiler = (Compiler*)VA_Fetch(self->children, 0);
+ Compiler *opt_compiler = (Compiler*)VA_Fetch(self->children, 1);
+ Matcher *req_matcher
+ = Compiler_Make_Matcher(req_compiler, reader, need_score);
+ Matcher *opt_matcher
+ = Compiler_Make_Matcher(opt_compiler, reader, need_score);
+
+ if (req_matcher == NULL) {
+ // No required matcher, ergo no matches possible.
+ DECREF(opt_matcher);
+ return NULL;
+ }
+ else if (opt_matcher == NULL) {
+ return req_matcher;
+ }
+ else {
+ Matcher *retval
+ = (Matcher*)ReqOptMatcher_new(sim, req_matcher, opt_matcher);
+ DECREF(opt_matcher);
+ DECREF(req_matcher);
+ return retval;
+ }
+}
+
+
diff --git a/core/Lucy/Search/RequiredOptionalQuery.cfh b/core/Lucy/Search/RequiredOptionalQuery.cfh
new file mode 100644
index 0000000..51c201c
--- /dev/null
+++ b/core/Lucy/Search/RequiredOptionalQuery.cfh
@@ -0,0 +1,82 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Join results for two Queries, one required, one optional.
+ *
+ * RequiredOptionalQuery joins the result sets of one Query which MUST match,
+ * and one Query which SHOULD match. When only the required Query matches,
+ * its score is passed along; when both match, the scores are summed.
+ */
+
+class Lucy::Search::RequiredOptionalQuery cnick ReqOptQuery
+ inherits Lucy::Search::PolyQuery {
+
+ inert incremented RequiredOptionalQuery*
+ new(Query *required_query, Query *optional_query);
+
+ /**
+ * @param required_query Query must must match.
+ * @param optional_query Query which should match.
+ */
+ public inert RequiredOptionalQuery*
+ init(RequiredOptionalQuery *self, Query *required_query,
+ Query *optional_query);
+
+ /** Getter for the required Query. */
+ public Query*
+ Get_Required_Query(RequiredOptionalQuery *self);
+
+ /** Setter for the required Query. */
+ public void
+ Set_Required_Query(RequiredOptionalQuery *self, Query *required_query);
+
+ /** Getter for the optional Query. */
+ public Query*
+ Get_Optional_Query(RequiredOptionalQuery *self);
+
+ /** Setter for the optional Query. */
+ public void
+ Set_Optional_Query(RequiredOptionalQuery *self, Query *optional_query);
+
+ public incremented Compiler*
+ Make_Compiler(RequiredOptionalQuery *self, Searcher *searcher,
+ float boost);
+
+ public incremented CharBuf*
+ To_String(RequiredOptionalQuery *self);
+
+ public bool_t
+ Equals(RequiredOptionalQuery *self, Obj *other);
+}
+
+class Lucy::Search::RequiredOptionalCompiler cnick ReqOptCompiler
+ inherits Lucy::Search::PolyCompiler {
+
+ inert incremented RequiredOptionalCompiler*
+ new(RequiredOptionalQuery *parent, Searcher *searcher, float boost);
+
+ inert RequiredOptionalCompiler*
+ init(RequiredOptionalCompiler *self, RequiredOptionalQuery *parent,
+ Searcher *searcher, float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(RequiredOptionalCompiler *self, SegReader *reader,
+ bool_t need_score);
+}
+
+
diff --git a/core/Lucy/Search/Searcher.c b/core/Lucy/Search/Searcher.c
new file mode 100644
index 0000000..11089b6
--- /dev/null
+++ b/core/Lucy/Search/Searcher.c
@@ -0,0 +1,98 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SEARCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Searcher.h"
+
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Collector.h"
+#include "Lucy/Search/Hits.h"
+#include "Lucy/Search/NoMatchQuery.h"
+#include "Lucy/Search/Query.h"
+#include "Lucy/Search/QueryParser.h"
+#include "Lucy/Search/SortSpec.h"
+#include "Lucy/Search/TopDocs.h"
+#include "Lucy/Search/Compiler.h"
+
+Searcher*
+Searcher_init(Searcher *self, Schema *schema) {
+ self->schema = (Schema*)INCREF(schema);
+ self->qparser = NULL;
+ ABSTRACT_CLASS_CHECK(self, SEARCHER);
+ return self;
+}
+
+void
+Searcher_destroy(Searcher *self) {
+ DECREF(self->schema);
+ DECREF(self->qparser);
+ SUPER_DESTROY(self, SEARCHER);
+}
+
+Hits*
+Searcher_hits(Searcher *self, Obj *query, uint32_t offset, uint32_t num_wanted,
+ SortSpec *sort_spec) {
+ Query *real_query = Searcher_Glean_Query(self, query);
+ uint32_t doc_max = Searcher_Doc_Max(self);
+ uint32_t wanted = offset + num_wanted > doc_max
+ ? doc_max
+ : offset + num_wanted;
+ TopDocs *top_docs = Searcher_Top_Docs(self, real_query, wanted,
+ sort_spec);
+ Hits *hits = Hits_new(self, top_docs, offset);
+ DECREF(top_docs);
+ DECREF(real_query);
+ return hits;
+}
+
+Query*
+Searcher_glean_query(Searcher *self, Obj *query) {
+ Query *real_query = NULL;
+
+ if (!query) {
+ real_query = (Query*)NoMatchQuery_new();
+ }
+ else if (Obj_Is_A(query, QUERY)) {
+ real_query = (Query*)INCREF(query);
+ }
+ else if (Obj_Is_A(query, CHARBUF)) {
+ if (!self->qparser) {
+ self->qparser = QParser_new(self->schema, NULL, NULL, NULL);
+ }
+ real_query = QParser_Parse(self->qparser, (CharBuf*)query);
+ }
+ else {
+ THROW(ERR, "Invalid type for 'query' param: %o",
+ Obj_Get_Class_Name(query));
+ }
+
+ return real_query;
+}
+
+Schema*
+Searcher_get_schema(Searcher *self) {
+ return self->schema;
+}
+
+void
+Searcher_close(Searcher *self) {
+ UNUSED_VAR(self);
+}
+
+
diff --git a/core/Lucy/Search/Searcher.cfh b/core/Lucy/Search/Searcher.cfh
new file mode 100644
index 0000000..2f39f5d
--- /dev/null
+++ b/core/Lucy/Search/Searcher.cfh
@@ -0,0 +1,120 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Base class for searching collections of documents.
+ *
+ * Abstract base class for objects which search. Core subclasses include
+ * L<Lucy::Search::IndexSearcher> and
+ * L<Lucy::Search::PolySearcher>.
+ */
+
+class Lucy::Search::Searcher inherits Lucy::Object::Obj {
+
+ Schema *schema;
+ QueryParser *qparser;
+
+ /** Abstract constructor.
+ *
+ * @param schema A Schema.
+ */
+ public inert Searcher*
+ init(Searcher *self, Schema *schema);
+
+ public void
+ Destroy(Searcher *self);
+
+ /** Return the maximum number of docs in the collection represented by the
+ * Searcher, which is also the highest possible internal doc id.
+ * Documents which have been marked as deleted but not yet purged are
+ * included in this count.
+ */
+ public abstract int32_t
+ Doc_Max(Searcher *self);
+
+ /** Return the number of documents which contain the term in the given
+ * field.
+ *
+ * @param field Field name.
+ * @param term The term to look up.
+ */
+ public abstract uint32_t
+ Doc_Freq(Searcher *self, const CharBuf *field, Obj *term);
+
+ /** If the supplied object is a Query, return it; if it's a query string,
+ * create a QueryParser and parse it to produce a query against all
+ * indexed fields.
+ */
+ public incremented Query*
+ Glean_Query(Searcher *self, Obj *query = NULL);
+
+ /** Return a Hits object containing the top results.
+ *
+ * @param query Either a Query object or a query string.
+ * @param offset The number of most-relevant hits to discard, typically
+ * used when "paging" through hits N at a time. Setting
+ * <code>offset</code> to 20 and <code>num_wanted</code> to 10 retrieves
+ * hits 21-30, assuming that 30 hits can be found.
+ * @param num_wanted The number of hits you would like to see after
+ * <code>offset</code> is taken into account.
+ * @param sort_spec A L<Lucy::Search::SortSpec>, which will affect
+ * how results are ranked and returned.
+ */
+ public incremented Hits*
+ Hits(Searcher *self, Obj *query, uint32_t offset = 0,
+ uint32_t num_wanted = 10, SortSpec *sort_spec = NULL);
+
+ /** Iterate over hits, feeding them into a
+ * L<Collector|Lucy::Search::Collector>.
+ *
+ * @param query A Query.
+ * @param collector A Collector.
+ */
+ public abstract void
+ Collect(Searcher *self, Query *query, Collector *collector);
+
+ /** Return a TopDocs object with up to num_wanted hits.
+ */
+ abstract incremented TopDocs*
+ Top_Docs(Searcher *self, Query *query, uint32_t num_wanted,
+ SortSpec *sort_spec = NULL);
+
+ /** Retrieve a document. Throws an error if the doc id is out of range.
+ *
+ * @param doc_id A document id.
+ */
+ public abstract incremented HitDoc*
+ Fetch_Doc(Searcher *self, int32_t doc_id);
+
+ /** Return the DocVector identified by the supplied doc id. Throws an
+ * error if the doc id is out of range.
+ */
+ abstract incremented DocVector*
+ Fetch_Doc_Vec(Searcher *self, int32_t doc_id);
+
+ /** Accessor for the object's <code>schema</code> member.
+ */
+ public Schema*
+ Get_Schema(Searcher *self);
+
+ /** Release external resources.
+ */
+ void
+ Close(Searcher *self);
+}
+
+
diff --git a/core/Lucy/Search/SeriesMatcher.c b/core/Lucy/Search/SeriesMatcher.c
new file mode 100644
index 0000000..bd40a35
--- /dev/null
+++ b/core/Lucy/Search/SeriesMatcher.c
@@ -0,0 +1,111 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SERIESMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/SeriesMatcher.h"
+
+SeriesMatcher*
+SeriesMatcher_new(VArray *matchers, I32Array *offsets) {
+ SeriesMatcher *self = (SeriesMatcher*)VTable_Make_Obj(SERIESMATCHER);
+ return SeriesMatcher_init(self, matchers, offsets);
+}
+
+SeriesMatcher*
+SeriesMatcher_init(SeriesMatcher *self, VArray *matchers, I32Array *offsets) {
+ Matcher_init((Matcher*)self);
+
+ // Init.
+ self->current_matcher = NULL;
+ self->current_offset = 0;
+ self->next_offset = 0;
+ self->doc_id = 0;
+ self->tick = 0;
+
+ // Assign.
+ self->matchers = (VArray*)INCREF(matchers);
+ self->offsets = (I32Array*)INCREF(offsets);
+
+ // Derive.
+ self->num_matchers = (int32_t)I32Arr_Get_Size(offsets);
+
+ return self;
+}
+
+void
+SeriesMatcher_destroy(SeriesMatcher *self) {
+ DECREF(self->matchers);
+ DECREF(self->offsets);
+ SUPER_DESTROY(self, SERIESMATCHER);
+}
+
+int32_t
+SeriesMatcher_next(SeriesMatcher *self) {
+ return SeriesMatcher_advance(self, self->doc_id + 1);
+}
+
+int32_t
+SeriesMatcher_advance(SeriesMatcher *self, int32_t target) {
+ if (target >= self->next_offset) {
+ // Proceed to next matcher or bail.
+ if (self->tick < self->num_matchers) {
+ while (1) {
+ uint32_t next_offset
+ = self->tick + 1 == self->num_matchers
+ ? I32_MAX
+ : I32Arr_Get(self->offsets, self->tick + 1);
+ self->current_matcher = (Matcher*)VA_Fetch(self->matchers,
+ self->tick);
+ self->current_offset = self->next_offset;
+ self->next_offset = next_offset;
+ self->doc_id = next_offset - 1;
+ self->tick++;
+ if (self->current_matcher != NULL
+ || self->tick >= self->num_matchers
+ ) {
+ break;
+ }
+ }
+ return SeriesMatcher_advance(self, target); // Recurse.
+ }
+ else {
+ // We're done.
+ self->doc_id = 0;
+ return 0;
+ }
+ }
+ else {
+ int32_t target_minus_offset = target - self->current_offset;
+ int32_t found
+ = Matcher_Advance(self->current_matcher, target_minus_offset);
+ if (found) {
+ self->doc_id = found + self->current_offset;
+ return self->doc_id;
+ }
+ else {
+ // Recurse.
+ return SeriesMatcher_advance(self, self->next_offset);
+ }
+ }
+}
+
+int32_t
+SeriesMatcher_get_doc_id(SeriesMatcher *self) {
+ return self->doc_id;
+}
+
+
diff --git a/core/Lucy/Search/SeriesMatcher.cfh b/core/Lucy/Search/SeriesMatcher.cfh
new file mode 100644
index 0000000..55f1a61
--- /dev/null
+++ b/core/Lucy/Search/SeriesMatcher.cfh
@@ -0,0 +1,51 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Concatenates multiple Matcher iterators.
+ */
+class Lucy::Search::SeriesMatcher inherits Lucy::Search::Matcher {
+
+ I32Array *offsets;
+ VArray *matchers;
+ Matcher *current_matcher;
+ int32_t doc_id;
+ int32_t tick;
+ int32_t num_matchers;
+ int32_t current_offset;
+ int32_t next_offset;
+
+ public inert incremented SeriesMatcher*
+ new(VArray *matchers, I32Array *offsets);
+
+ public inert SeriesMatcher*
+ init(SeriesMatcher *self, VArray *matchers, I32Array *offsets);
+
+ public int32_t
+ Next(SeriesMatcher *self);
+
+ public int32_t
+ Advance(SeriesMatcher *self, int32_t target);
+
+ public int32_t
+ Get_Doc_ID(SeriesMatcher *self);
+
+ public void
+ Destroy(SeriesMatcher *self);
+}
+
+
diff --git a/core/Lucy/Search/SortRule.c b/core/Lucy/Search/SortRule.c
new file mode 100644
index 0000000..1bc77ae
--- /dev/null
+++ b/core/Lucy/Search/SortRule.c
@@ -0,0 +1,95 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTRULE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/SortRule.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+int32_t SortRule_FIELD = 0;
+int32_t SortRule_SCORE = 1;
+int32_t SortRule_DOC_ID = 2;
+
+SortRule*
+SortRule_new(int32_t type, const CharBuf *field, bool_t reverse) {
+ SortRule *self = (SortRule*)VTable_Make_Obj(SORTRULE);
+ return SortRule_init(self, type, field, reverse);
+}
+
+SortRule*
+SortRule_init(SortRule *self, int32_t type, const CharBuf *field,
+ bool_t reverse) {
+ self->field = field ? CB_Clone(field) : NULL;
+ self->type = type;
+ self->reverse = reverse;
+
+ // Validate.
+ if (type == SortRule_FIELD) {
+ if (!field) {
+ THROW(ERR, "When sorting by field, param 'field' is required");
+ }
+ }
+ else if (type == SortRule_SCORE) { }
+ else if (type == SortRule_DOC_ID) { }
+ else { THROW(ERR, "Unknown type: %i32", type); }
+
+ return self;
+}
+
+void
+SortRule_destroy(SortRule *self) {
+ DECREF(self->field);
+ SUPER_DESTROY(self, SORTRULE);
+}
+
+SortRule*
+SortRule_deserialize(SortRule *self, InStream *instream) {
+ self = self ? self : (SortRule*)VTable_Make_Obj(SORTRULE);
+ self->type = InStream_Read_C32(instream);
+ if (self->type == SortRule_FIELD) {
+ self->field = CB_deserialize(NULL, instream);
+ }
+ self->reverse = InStream_Read_C32(instream);
+ return self;
+}
+
+void
+SortRule_serialize(SortRule *self, OutStream *target) {
+ OutStream_Write_C32(target, self->type);
+ if (self->type == SortRule_FIELD) {
+ CB_Serialize(self->field, target);
+ }
+ OutStream_Write_C32(target, !!self->reverse);
+}
+
+CharBuf*
+SortRule_get_field(SortRule *self) {
+ return self->field;
+}
+
+int32_t
+SortRule_get_type(SortRule *self) {
+ return self->type;
+}
+
+bool_t
+SortRule_get_reverse(SortRule *self) {
+ return self->reverse;
+}
+
+
diff --git a/core/Lucy/Search/SortRule.cfh b/core/Lucy/Search/SortRule.cfh
new file mode 100644
index 0000000..e099074
--- /dev/null
+++ b/core/Lucy/Search/SortRule.cfh
@@ -0,0 +1,75 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Element of a SortSpec.
+ *
+ * SortRules are the building blocks used to assemble
+ * L<SortSpecs|Lucy::Search::SortSpec>; each SortRule defines a single
+ * level of sorting. For example, sorting first by "category" then by score
+ * requires a SortSpec with two SortRule elements.
+ */
+class Lucy::Search::SortRule inherits Lucy::Object::Obj {
+
+ int32_t type;
+ CharBuf *field;
+ bool_t reverse;
+
+ inert int32_t FIELD;
+ inert int32_t SCORE;
+ inert int32_t DOC_ID;
+
+ public inert incremented SortRule*
+ new(int32_t type = 0, const CharBuf *field = NULL,
+ bool_t reverse = false);
+
+ /**
+ * @param type Indicate whether to sort by score, field, etc. (The
+ * default is to sort by a field.)
+ * @param field The name of a <code>sortable</code> field.
+ * @param reverse If true, reverse the order of the sort for this rule.
+ */
+ public inert incremented SortRule*
+ init(SortRule *self, int32_t type = 0, const CharBuf *field = NULL,
+ bool_t reverse = false);
+
+ /** Accessor for "field" member.
+ */
+ public nullable CharBuf*
+ Get_Field(SortRule *self);
+
+ /** Accessor for "type" member.
+ */
+ public int32_t
+ Get_Type(SortRule *self);
+
+ /** Accessor for "reverse" member.
+ */
+ public bool_t
+ Get_Reverse(SortRule *self);
+
+ public incremented SortRule*
+ Deserialize(SortRule *self, InStream *instream);
+
+ public void
+ Serialize(SortRule *self, OutStream *outstream);
+
+ public void
+ Destroy(SortRule *self);
+}
+
+
diff --git a/core/Lucy/Search/SortSpec.c b/core/Lucy/Search/SortSpec.c
new file mode 100644
index 0000000..d01094f
--- /dev/null
+++ b/core/Lucy/Search/SortSpec.c
@@ -0,0 +1,88 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTSPEC
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/SortSpec.h"
+#include "Lucy/Index/IndexReader.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/SortRule.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/SortUtils.h"
+
+SortSpec*
+SortSpec_new(VArray *rules) {
+ SortSpec *self = (SortSpec*)VTable_Make_Obj(SORTSPEC);
+ return SortSpec_init(self, rules);
+}
+
+SortSpec*
+SortSpec_init(SortSpec *self, VArray *rules) {
+ int32_t i, max;
+ self->rules = VA_Shallow_Copy(rules);
+ for (i = 0, max = VA_Get_Size(rules); i < max; i++) {
+ SortRule *rule = (SortRule*)VA_Fetch(rules, i);
+ CERTIFY(rule, SORTRULE);
+ }
+ return self;
+}
+
+void
+SortSpec_destroy(SortSpec *self) {
+ DECREF(self->rules);
+ SUPER_DESTROY(self, SORTSPEC);
+}
+
+SortSpec*
+SortSpec_deserialize(SortSpec *self, InStream *instream) {
+ uint32_t num_rules = InStream_Read_C32(instream);
+ VArray *rules = VA_new(num_rules);
+ uint32_t i;
+
+ // Create base object.
+ self = self ? self : (SortSpec*)VTable_Make_Obj(SORTSPEC);
+
+ // Add rules.
+ for (i = 0; i < num_rules; i++) {
+ VA_Push(rules, (Obj*)SortRule_deserialize(NULL, instream));
+ }
+ SortSpec_init(self, rules);
+ DECREF(rules);
+
+ return self;
+}
+
+VArray*
+SortSpec_get_rules(SortSpec *self) {
+ return self->rules;
+}
+
+void
+SortSpec_serialize(SortSpec *self, OutStream *target) {
+ uint32_t num_rules = VA_Get_Size(self->rules);
+ uint32_t i;
+ OutStream_Write_C32(target, num_rules);
+ for (i = 0; i < num_rules; i++) {
+ SortRule *rule = (SortRule*)VA_Fetch(self->rules, i);
+ SortRule_Serialize(rule, target);
+ }
+}
+
+
diff --git a/core/Lucy/Search/SortSpec.cfh b/core/Lucy/Search/SortSpec.cfh
new file mode 100644
index 0000000..0738b5d
--- /dev/null
+++ b/core/Lucy/Search/SortSpec.cfh
@@ -0,0 +1,57 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Specify a custom sort order for search results.
+ *
+ * By default, searches return results in order of relevance; SortSpec allows
+ * you to indicate an alternate order via an array of
+ * L<SortRules|Lucy::Search::SortRule>.
+ *
+ * Fields you wish to sort against must be <code>sortable</code>.
+ *
+ * For a stable sort (important when paging through results), add a
+ * sort-by-doc rule as the last SortRule.
+ */
+
+class Lucy::Search::SortSpec inherits Lucy::Object::Obj {
+
+ VArray *rules;
+
+ public inert SortSpec*
+ new(VArray *rules);
+
+ /**
+ * @param rules An array of SortRules.
+ */
+ public inert SortSpec*
+ init(SortSpec *self, VArray *rules);
+
+ public incremented SortSpec*
+ Deserialize(SortSpec *self, InStream *instream);
+
+ public void
+ Serialize(SortSpec *self, OutStream *outstream);
+
+ VArray*
+ Get_Rules(SortSpec *self);
+
+ public void
+ Destroy(SortSpec *self);
+}
+
+
diff --git a/core/Lucy/Search/Span.c b/core/Lucy/Search/Span.c
new file mode 100644
index 0000000..cfd6ae8
--- /dev/null
+++ b/core/Lucy/Search/Span.c
@@ -0,0 +1,75 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SPAN
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/Span.h"
+
+Span*
+Span_new(int32_t offset, int32_t length, float weight) {
+ Span *self = (Span*)VTable_Make_Obj(SPAN);
+ return Span_init(self, offset, length, weight);
+}
+
+Span*
+Span_init(Span *self, int32_t offset, int32_t length,
+ float weight) {
+ self->offset = offset;
+ self->length = length;
+ self->weight = weight;
+ return self;
+}
+
+int32_t
+Span_get_offset(Span *self) {
+ return self->offset;
+}
+
+int32_t
+Span_get_length(Span *self) {
+ return self->length;
+}
+
+float
+Span_get_weight(Span *self) {
+ return self->weight;
+}
+
+void
+Span_set_offset(Span *self, int32_t offset) {
+ self->offset = offset;
+}
+
+void
+Span_set_length(Span *self, int32_t length) {
+ self->length = length;
+}
+
+void
+Span_set_weight(Span *self, float weight) {
+ self->weight = weight;
+}
+
+int32_t
+Span_compare_to(Span *self, Obj *other) {
+ Span *competitor = (Span*)CERTIFY(other, SPAN);
+ int32_t comparison = self->offset - competitor->offset;
+ if (comparison == 0) { comparison = self->length - competitor->length; }
+ return comparison;
+}
+
+
diff --git a/core/Lucy/Search/Span.cfh b/core/Lucy/Search/Span.cfh
new file mode 100644
index 0000000..0404801
--- /dev/null
+++ b/core/Lucy/Search/Span.cfh
@@ -0,0 +1,81 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** An offset, a length, and a weight.
+ *
+ * Span objects store information about a span across an array of...
+ * something. The unit is context-dependent.
+ *
+ * Text is one possibility, in which case offset and length might be measured
+ * in Unicode code points. However, the Span could also refer to a span
+ * within an array of tokens, for example -- in which case the start and
+ * offset might be measured in token positions.
+ */
+class Lucy::Search::Span inherits Lucy::Object::Obj {
+
+ int32_t offset;
+ int32_t length;
+ float weight;
+
+ inert incremented Span*
+ new(int32_t offset, int32_t length, float weight = 0.0);
+
+ /**
+ * @param offset Integer offset, unit is context-dependent.
+ * @param length Integer length, unit is context-dependent.
+ * @param weight A floating point weight.
+ */
+ public inert Span*
+ init(Span *self, int32_t offset, int32_t length,
+ float weight = 0.0);
+
+ /** Accessor for <code>offset</code> attribute.
+ */
+ public int32_t
+ Get_Offset(Span *self);
+
+ /** Setter for <code>offset</code> attribute.
+ */
+ public void
+ Set_Offset(Span *self, int32_t offset);
+
+ /** Accessor for <code>length</code> attribute.
+ */
+ public int32_t
+ Get_Length(Span *self);
+
+ /** Setter for <code>length</code> attribute.
+ */
+ public void
+ Set_Length(Span *self, int32_t length);
+
+ /** Accessor for <code>weight</code> attribute.
+ */
+ public float
+ Get_Weight(Span *self);
+
+ /** Setter for <code>weight</code> attribute.
+ */
+ public void
+ Set_Weight(Span *self, float weight);
+
+ public int32_t
+ Compare_To(Span *self, Obj *other);
+}
+
+
diff --git a/core/Lucy/Search/TermMatcher.c b/core/Lucy/Search/TermMatcher.c
new file mode 100644
index 0000000..e5fc865
--- /dev/null
+++ b/core/Lucy/Search/TermMatcher.c
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TERMMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/TermMatcher.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Search/Compiler.h"
+
+TermMatcher*
+TermMatcher_init(TermMatcher *self, Similarity *similarity, PostingList *plist,
+ Compiler *compiler) {
+ Matcher_init((Matcher*)self);
+
+ // Assign.
+ self->sim = (Similarity*)INCREF(similarity);
+ self->plist = (PostingList*)INCREF(plist);
+ self->compiler = (Compiler*)INCREF(compiler);
+ self->weight = Compiler_Get_Weight(compiler);
+
+ // Init.
+ self->posting = NULL;
+
+ return self;
+}
+
+void
+TermMatcher_destroy(TermMatcher *self) {
+ DECREF(self->sim);
+ DECREF(self->plist);
+ DECREF(self->compiler);
+ SUPER_DESTROY(self, TERMMATCHER);
+}
+
+int32_t
+TermMatcher_next(TermMatcher* self) {
+ PostingList *const plist = self->plist;
+ if (plist) {
+ int32_t doc_id = PList_Next(plist);
+ if (doc_id) {
+ self->posting = PList_Get_Posting(plist);
+ return doc_id;
+ }
+ else {
+ // Reclaim resources a little early.
+ DECREF(plist);
+ self->plist = NULL;
+ return 0;
+ }
+ }
+ return 0;
+}
+
+int32_t
+TermMatcher_advance(TermMatcher *self, int32_t target) {
+ PostingList *const plist = self->plist;
+ if (plist) {
+ int32_t doc_id = PList_Advance(plist, target);
+ if (doc_id) {
+ self->posting = PList_Get_Posting(plist);
+ return doc_id;
+ }
+ else {
+ // Reclaim resources a little early.
+ DECREF(plist);
+ self->plist = NULL;
+ return 0;
+ }
+ }
+ return 0;
+}
+
+int32_t
+TermMatcher_get_doc_id(TermMatcher* self) {
+ return Post_Get_Doc_ID(self->posting);
+}
+
+
diff --git a/core/Lucy/Search/TermMatcher.cfh b/core/Lucy/Search/TermMatcher.cfh
new file mode 100644
index 0000000..0398f37
--- /dev/null
+++ b/core/Lucy/Search/TermMatcher.cfh
@@ -0,0 +1,57 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Base class for TermMatchers.
+ *
+ * Each subclass of Posting is associated with a corresponding subclass of
+ * TermMatcher.
+ */
+class Lucy::Search::TermMatcher inherits Lucy::Search::Matcher {
+
+ float weight;
+ Compiler *compiler;
+ Similarity *sim;
+ PostingList *plist;
+ Posting *posting;
+
+ inert TermMatcher*
+ init(TermMatcher *self, Similarity *similarity, PostingList *posting_list,
+ Compiler *compiler);
+
+ public void
+ Destroy(TermMatcher *self);
+
+ public int32_t
+ Next(TermMatcher* self);
+
+ public int32_t
+ Advance(TermMatcher* self, int32_t target);
+
+ public int32_t
+ Get_Doc_ID(TermMatcher* self);
+}
+
+__C__
+#define LUCY_TERMMATCHER_SCORE_CACHE_SIZE 32
+#ifdef LUCY_USE_SHORT_NAMES
+ #define TERMMATCHER_SCORE_CACHE_SIZE LUCY_TERMMATCHER_SCORE_CACHE_SIZE
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Search/TermQuery.c b/core/Lucy/Search/TermQuery.c
new file mode 100644
index 0000000..8d9bdb1
--- /dev/null
+++ b/core/Lucy/Search/TermQuery.c
@@ -0,0 +1,264 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TERMQUERY
+#define C_LUCY_TERMCOMPILER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/TermVector.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Search/Compiler.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Search/TermMatcher.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+TermQuery*
+TermQuery_new(const CharBuf *field, const Obj *term) {
+ TermQuery *self = (TermQuery*)VTable_Make_Obj(TERMQUERY);
+ return TermQuery_init(self, field, term);
+}
+
+TermQuery*
+TermQuery_init(TermQuery *self, const CharBuf *field, const Obj *term) {
+ Query_init((Query*)self, 1.0f);
+ self->field = CB_Clone(field);
+ self->term = Obj_Clone(term);
+ return self;
+}
+
+void
+TermQuery_destroy(TermQuery *self) {
+ DECREF(self->field);
+ DECREF(self->term);
+ SUPER_DESTROY(self, TERMQUERY);
+}
+
+void
+TermQuery_serialize(TermQuery *self, OutStream *outstream) {
+ CB_Serialize(self->field, outstream);
+ FREEZE(self->term, outstream);
+ OutStream_Write_F32(outstream, self->boost);
+}
+
+TermQuery*
+TermQuery_deserialize(TermQuery *self, InStream *instream) {
+ self = self ? self : (TermQuery*)VTable_Make_Obj(TERMQUERY);
+ self->field = CB_deserialize(NULL, instream);
+ self->term = (Obj*)THAW(instream);
+ self->boost = InStream_Read_F32(instream);
+ return self;
+}
+
+CharBuf*
+TermQuery_get_field(TermQuery *self) {
+ return self->field;
+}
+
+Obj*
+TermQuery_get_term(TermQuery *self) {
+ return self->term;
+}
+
+bool_t
+TermQuery_equals(TermQuery *self, Obj *other) {
+ TermQuery *twin = (TermQuery*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, TERMQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (!CB_Equals(self->field, (Obj*)twin->field)) { return false; }
+ if (!Obj_Equals(self->term, twin->term)) { return false; }
+ return true;
+}
+
+CharBuf*
+TermQuery_to_string(TermQuery *self) {
+ CharBuf *term_str = Obj_To_String(self->term);
+ CharBuf *retval = CB_newf("%o:%o", self->field, term_str);
+ DECREF(term_str);
+ return retval;
+}
+
+Compiler*
+TermQuery_make_compiler(TermQuery *self, Searcher *searcher, float boost) {
+ return (Compiler*)TermCompiler_new((Query*)self, searcher, boost);
+}
+
+/******************************************************************/
+
+TermCompiler*
+TermCompiler_new(Query *parent, Searcher *searcher, float boost) {
+ TermCompiler *self = (TermCompiler*)VTable_Make_Obj(TERMCOMPILER);
+ return TermCompiler_init(self, parent, searcher, boost);
+}
+
+TermCompiler*
+TermCompiler_init(TermCompiler *self, Query *parent, Searcher *searcher,
+ float boost) {
+ Schema *schema = Searcher_Get_Schema(searcher);
+ TermQuery *tparent = (TermQuery*)parent;
+ Similarity *sim = Schema_Fetch_Sim(schema, tparent->field);
+
+ // Try harder to get a Similarity if necessary.
+ if (!sim) { sim = Schema_Get_Similarity(schema); }
+
+ // Init.
+ Compiler_init((Compiler*)self, parent, searcher, sim, boost);
+ self->normalized_weight = 0.0f;
+ self->query_norm_factor = 0.0f;
+
+ // Derive.
+ int32_t doc_max = Searcher_Doc_Max(searcher);
+ int32_t doc_freq = Searcher_Doc_Freq(searcher, tparent->field,
+ tparent->term);
+ self->idf = Sim_IDF(sim, doc_freq, doc_max);
+
+ /* The score of any document is approximately equal to:
+ *
+ * (tf_d * idf_t / norm_d) * (tf_q * idf_t / norm_q)
+ *
+ * Here we add in the first IDF, plus user-supplied boost.
+ *
+ * The second clause is factored in by the call to Normalize().
+ *
+ * tf_d and norm_d can only be added by the Matcher, since they are
+ * per-document.
+ */
+ self->raw_weight = self->idf * self->boost;
+
+ // Make final preparations.
+ TermCompiler_Normalize(self);
+
+ return self;
+}
+
+bool_t
+TermCompiler_equals(TermCompiler *self, Obj *other) {
+ TermCompiler *twin = (TermCompiler*)other;
+ if (!Compiler_equals((Compiler*)self, other)) { return false; }
+ if (!Obj_Is_A(other, TERMCOMPILER)) { return false; }
+ if (self->idf != twin->idf) { return false; }
+ if (self->raw_weight != twin->raw_weight) { return false; }
+ if (self->query_norm_factor != twin->query_norm_factor) { return false; }
+ if (self->normalized_weight != twin->normalized_weight) { return false; }
+ return true;
+}
+
+void
+TermCompiler_serialize(TermCompiler *self, OutStream *outstream) {
+ Compiler_serialize((Compiler*)self, outstream);
+ OutStream_Write_F32(outstream, self->idf);
+ OutStream_Write_F32(outstream, self->raw_weight);
+ OutStream_Write_F32(outstream, self->query_norm_factor);
+ OutStream_Write_F32(outstream, self->normalized_weight);
+}
+
+TermCompiler*
+TermCompiler_deserialize(TermCompiler *self, InStream *instream) {
+ self = self ? self : (TermCompiler*)VTable_Make_Obj(TERMCOMPILER);
+ Compiler_deserialize((Compiler*)self, instream);
+ self->idf = InStream_Read_F32(instream);
+ self->raw_weight = InStream_Read_F32(instream);
+ self->query_norm_factor = InStream_Read_F32(instream);
+ self->normalized_weight = InStream_Read_F32(instream);
+ return self;
+}
+
+float
+TermCompiler_sum_of_squared_weights(TermCompiler *self) {
+ return self->raw_weight * self->raw_weight;
+}
+
+void
+TermCompiler_apply_norm_factor(TermCompiler *self, float query_norm_factor) {
+ self->query_norm_factor = query_norm_factor;
+
+ /* Multiply raw weight by the idf and norm_q factors in this:
+ *
+ * (tf_q * idf_q / norm_q)
+ *
+ * Note: factoring in IDF a second time is correct. See formula.
+ */
+ self->normalized_weight
+ = self->raw_weight * self->idf * query_norm_factor;
+}
+
+float
+TermCompiler_get_weight(TermCompiler *self) {
+ return self->normalized_weight;
+}
+
+Matcher*
+TermCompiler_make_matcher(TermCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ TermQuery *tparent = (TermQuery*)self->parent;
+ PostingListReader *plist_reader
+ = (PostingListReader*)SegReader_Fetch(
+ reader, VTable_Get_Name(POSTINGLISTREADER));
+ PostingList *plist = plist_reader
+ ? PListReader_Posting_List(plist_reader, tparent->field, tparent->term)
+ : NULL;
+
+ if (plist == NULL || PList_Get_Doc_Freq(plist) == 0) {
+ DECREF(plist);
+ return NULL;
+ }
+ else {
+ Matcher *retval = PList_Make_Matcher(plist, self->sim,
+ (Compiler*)self, need_score);
+ DECREF(plist);
+ return retval;
+ }
+}
+
+VArray*
+TermCompiler_highlight_spans(TermCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field) {
+ TermQuery *const parent = (TermQuery*)self->parent;
+ VArray *spans = VA_new(0);
+ TermVector *term_vector;
+ I32Array *starts, *ends;
+ uint32_t i, max;
+ UNUSED_VAR(searcher);
+
+ if (!CB_Equals(parent->field, (Obj*)field)) { return spans; }
+
+ // Add all starts and ends.
+ term_vector = DocVec_Term_Vector(doc_vec, field, (CharBuf*)parent->term);
+ if (!term_vector) { return spans; }
+
+ starts = TV_Get_Start_Offsets(term_vector);
+ ends = TV_Get_End_Offsets(term_vector);
+ for (i = 0, max = I32Arr_Get_Size(starts); i < max; i++) {
+ int32_t start = I32Arr_Get(starts, i);
+ int32_t length = I32Arr_Get(ends, i) - start;
+ VA_Push(spans,
+ (Obj*)Span_new(start, length, TermCompiler_Get_Weight(self)));
+ }
+
+ DECREF(term_vector);
+ return spans;
+}
+
+
diff --git a/core/Lucy/Search/TermQuery.cfh b/core/Lucy/Search/TermQuery.cfh
new file mode 100644
index 0000000..c5fbd6c
--- /dev/null
+++ b/core/Lucy/Search/TermQuery.cfh
@@ -0,0 +1,110 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+
+/** Query which matches individual terms.
+ *
+ * TermQuery is a subclass of L<Lucy::Search::Query> for matching
+ * individual terms in a specific field.
+ */
+
+class Lucy::Search::TermQuery inherits Lucy::Search::Query
+ : dumpable {
+
+ CharBuf *field;
+ Obj *term;
+
+ inert incremented TermQuery*
+ new(const CharBuf *field, const Obj *term);
+
+ /**
+ * @param field Field name.
+ * @param term Term text.
+ */
+ public inert TermQuery*
+ init(TermQuery *self, const CharBuf *field, const Obj *term);
+
+ /** Accessor for object's <code>field</code> member.
+ */
+ public CharBuf*
+ Get_Field(TermQuery *self);
+
+ /** Accessor for object's <code>term</code> member.
+ */
+ public Obj*
+ Get_Term(TermQuery *self);
+
+ public incremented Compiler*
+ Make_Compiler(TermQuery *self, Searcher *searcher, float boost);
+
+ public incremented CharBuf*
+ To_String(TermQuery *self);
+
+ public void
+ Serialize(TermQuery *self, OutStream *outstream);
+
+ public incremented TermQuery*
+ Deserialize(TermQuery *self, InStream *instream);
+
+ public bool_t
+ Equals(TermQuery *self, Obj *other);
+
+ public void
+ Destroy(TermQuery *self);
+}
+
+class Lucy::Search::TermCompiler inherits Lucy::Search::Compiler {
+ float idf;
+ float raw_weight;
+ float query_norm_factor;
+ float normalized_weight;
+
+ inert incremented TermCompiler*
+ new(Query *parent, Searcher *searcher, float boost);
+
+ inert TermCompiler*
+ init(TermCompiler *self, Query *parent, Searcher *searcher,
+ float boost);
+
+ public incremented nullable Matcher*
+ Make_Matcher(TermCompiler *self, SegReader *reader, bool_t need_score);
+
+ public float
+ Get_Weight(TermCompiler *self);
+
+ public float
+ Sum_Of_Squared_Weights(TermCompiler *self);
+
+ public void
+ Apply_Norm_Factor(TermCompiler *self, float factor);
+
+ public incremented VArray*
+ Highlight_Spans(TermCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field);
+
+ public bool_t
+ Equals(TermCompiler *self, Obj *other);
+
+ public void
+ Serialize(TermCompiler *self, OutStream *outstream);
+
+ public incremented TermCompiler*
+ Deserialize(TermCompiler *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Search/TopDocs.c b/core/Lucy/Search/TopDocs.c
new file mode 100644
index 0000000..9ca4c83
--- /dev/null
+++ b/core/Lucy/Search/TopDocs.c
@@ -0,0 +1,81 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TOPDOCS
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Search/TopDocs.h"
+#include "Lucy/Index/IndexReader.h"
+#include "Lucy/Index/Lexicon.h"
+#include "Lucy/Search/SortRule.h"
+#include "Lucy/Search/SortSpec.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+TopDocs*
+TopDocs_new(VArray *match_docs, uint32_t total_hits) {
+ TopDocs *self = (TopDocs*)VTable_Make_Obj(TOPDOCS);
+ return TopDocs_init(self, match_docs, total_hits);
+}
+
+TopDocs*
+TopDocs_init(TopDocs *self, VArray *match_docs, uint32_t total_hits) {
+ self->match_docs = (VArray*)INCREF(match_docs);
+ self->total_hits = total_hits;
+ return self;
+}
+
+void
+TopDocs_destroy(TopDocs *self) {
+ DECREF(self->match_docs);
+ SUPER_DESTROY(self, TOPDOCS);
+}
+
+void
+TopDocs_serialize(TopDocs *self, OutStream *outstream) {
+ VA_Serialize(self->match_docs, outstream);
+ OutStream_Write_C32(outstream, self->total_hits);
+}
+
+TopDocs*
+TopDocs_deserialize(TopDocs *self, InStream *instream) {
+ self = self ? self : (TopDocs*)VTable_Make_Obj(TOPDOCS);
+ self->match_docs = VA_deserialize(NULL, instream);
+ self->total_hits = InStream_Read_C32(instream);
+ return self;
+}
+
+VArray*
+TopDocs_get_match_docs(TopDocs *self) {
+ return self->match_docs;
+}
+
+uint32_t
+TopDocs_get_total_hits(TopDocs *self) {
+ return self->total_hits;
+}
+
+void
+TopDocs_set_match_docs(TopDocs *self, VArray *match_docs) {
+ DECREF(self->match_docs);
+ self->match_docs = (VArray*)INCREF(match_docs);
+}
+void
+TopDocs_set_total_hits(TopDocs *self, uint32_t total_hits) {
+ self->total_hits = total_hits;
+}
+
+
diff --git a/core/Lucy/Search/TopDocs.cfh b/core/Lucy/Search/TopDocs.cfh
new file mode 100644
index 0000000..74df3b7
--- /dev/null
+++ b/core/Lucy/Search/TopDocs.cfh
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Top-scoring documents.
+ *
+ * A TopDocs object encapsulates the highest-scoring N documents and their
+ * associated scores.
+ */
+class Lucy::Search::TopDocs inherits Lucy::Object::Obj {
+
+ VArray *match_docs;
+ uint32_t total_hits;
+
+ inert incremented TopDocs*
+ new(VArray *match_docs, uint32_t total_hits);
+
+ inert TopDocs*
+ init(TopDocs *self, VArray *match_docs, uint32_t total_hits);
+
+ /** Accessor for <code>match_docs</code> member.
+ */
+ VArray*
+ Get_Match_Docs(TopDocs *self);
+
+ /** Setter for <code>match_docs</code> member.
+ */
+ void
+ Set_Match_Docs(TopDocs *self, VArray *match_docs);
+
+ /** Accessor for <code>total_hits</code> member.
+ */
+ uint32_t
+ Get_Total_Hits(TopDocs *self);
+
+ /** Setter for <code>total_hits</code> member.
+ */
+ void
+ Set_Total_Hits(TopDocs *self, uint32_t total_hits);
+
+ public void
+ Serialize(TopDocs *self, OutStream *outstream);
+
+ public incremented TopDocs*
+ Deserialize(TopDocs *self, InStream *instream);
+
+ public void
+ Destroy(TopDocs *self);
+}
+
+
diff --git a/core/Lucy/Store/CompoundFileReader.c b/core/Lucy/Store/CompoundFileReader.c
new file mode 100644
index 0000000..437359f
--- /dev/null
+++ b/core/Lucy/Store/CompoundFileReader.c
@@ -0,0 +1,320 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_COMPOUNDFILEREADER
+#define C_LUCY_CFREADERDIRHANDLE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/StringHelper.h"
+
+CompoundFileReader*
+CFReader_open(Folder *folder) {
+ CompoundFileReader *self
+ = (CompoundFileReader*)VTable_Make_Obj(COMPOUNDFILEREADER);
+ return CFReader_do_open(self, folder);
+}
+
+CompoundFileReader*
+CFReader_do_open(CompoundFileReader *self, Folder *folder) {
+ CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
+ Hash *metadata = (Hash*)Json_slurp_json((Folder*)folder, cfmeta_file);
+ Err *error = NULL;
+
+ Folder_init((Folder*)self, Folder_Get_Path(folder));
+
+ // Parse metadata file.
+ if (!metadata || !Hash_Is_A(metadata, HASH)) {
+ error = Err_new(CB_newf("Can't read '%o' in '%o'", cfmeta_file,
+ Folder_Get_Path(folder)));
+ }
+ else {
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+ self->format = format ? (int32_t)Obj_To_I64(format) : 0;
+ self->records = (Hash*)INCREF(Hash_Fetch_Str(metadata, "files", 5));
+ if (self->format < 1) {
+ error = Err_new(CB_newf("Corrupt %o file: Missing or invalid 'format'",
+ cfmeta_file));
+ }
+ else if (self->format > CFWriter_current_file_format) {
+ error = Err_new(CB_newf("Unsupported compound file format: %i32 "
+ "(current = %i32", self->format,
+ CFWriter_current_file_format));
+ }
+ else if (!self->records) {
+ error = Err_new(CB_newf("Corrupt %o file: missing 'files' key",
+ cfmeta_file));
+ }
+ }
+ DECREF(metadata);
+ if (error) {
+ Err_set_error(error);
+ DECREF(self);
+ return NULL;
+ }
+
+ // Open an instream which we'll clone over and over.
+ CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
+ self->instream = Folder_Open_In(folder, cf_file);
+ if (!self->instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ // Assign.
+ self->real_folder = (Folder*)INCREF(folder);
+
+ // Strip directory name from filepaths for old format.
+ if (self->format == 1) {
+ VArray *files = Hash_Keys(self->records);
+ ZombieCharBuf *filename = ZCB_BLANK();
+ ZombieCharBuf *folder_name
+ = IxFileNames_local_part(Folder_Get_Path(folder), ZCB_BLANK());
+ size_t folder_name_len = ZCB_Length(folder_name);
+
+ for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *orig = (CharBuf*)VA_Fetch(files, i);
+ if (CB_Starts_With(orig, (CharBuf*)folder_name)) {
+ Obj *record = Hash_Delete(self->records, (Obj*)orig);
+ ZCB_Assign(filename, orig);
+ ZCB_Nip(filename, folder_name_len + sizeof(DIR_SEP) - 1);
+ Hash_Store(self->records, (Obj*)filename, (Obj*)record);
+ }
+ }
+
+ DECREF(files);
+ }
+
+ return self;
+}
+
+void
+CFReader_destroy(CompoundFileReader *self) {
+ DECREF(self->real_folder);
+ DECREF(self->instream);
+ DECREF(self->records);
+ SUPER_DESTROY(self, COMPOUNDFILEREADER);
+}
+
+Folder*
+CFReader_get_real_folder(CompoundFileReader *self) {
+ return self->real_folder;
+}
+
+void
+CFReader_set_path(CompoundFileReader *self, const CharBuf *path) {
+ Folder_Set_Path(self->real_folder, path);
+ Folder_set_path((Folder*)self, path);
+}
+
+FileHandle*
+CFReader_local_open_filehandle(CompoundFileReader *self,
+ const CharBuf *name, uint32_t flags) {
+ Hash *entry = (Hash*)Hash_Fetch(self->records, (Obj*)name);
+ FileHandle *fh = NULL;
+
+ if (entry) {
+ Err_set_error(Err_new(CB_newf("Can't open FileHandle for virtual file %o in '%o'",
+ name, self->path)));
+ }
+ else {
+ fh = Folder_Local_Open_FileHandle(self->real_folder, name, flags);
+ if (!fh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+
+ return fh;
+}
+
+bool_t
+CFReader_local_delete(CompoundFileReader *self, const CharBuf *name) {
+ Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name);
+ DECREF(record);
+
+ if (record == NULL) {
+ return Folder_Local_Delete(self->real_folder, name);
+ }
+ else {
+ // Once the number of virtual files falls to 0, remove the compound
+ // files.
+ if (Hash_Get_Size(self->records) == 0) {
+ CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
+ if (!Folder_Delete(self->real_folder, cf_file)) {
+ return false;
+ }
+ CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
+ if (!Folder_Delete(self->real_folder, cfmeta_file)) {
+ return false;
+
+ }
+ }
+ return true;
+ }
+}
+
+InStream*
+CFReader_local_open_in(CompoundFileReader *self, const CharBuf *name) {
+ Hash *entry = (Hash*)Hash_Fetch(self->records, (Obj*)name);
+
+ if (!entry) {
+ InStream *instream = Folder_Local_Open_In(self->real_folder, name);
+ if (!instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return instream;
+ }
+ else {
+ Obj *len = Hash_Fetch_Str(entry, "length", 6);
+ Obj *offset = Hash_Fetch_Str(entry, "offset", 6);
+ if (!len || !offset) {
+ Err_set_error(Err_new(CB_newf("Malformed entry for '%o' in '%o'",
+ name, Folder_Get_Path(self->real_folder))));
+ return NULL;
+ }
+ else if (CB_Get_Size(self->path)) {
+ CharBuf *fullpath = CB_newf("%o/%o", self->path, name);
+ InStream *instream = InStream_Reopen(self->instream, fullpath,
+ Obj_To_I64(offset), Obj_To_I64(len));
+ DECREF(fullpath);
+ return instream;
+ }
+ else {
+ return InStream_Reopen(self->instream, name, Obj_To_I64(offset),
+ Obj_To_I64(len));
+ }
+ }
+}
+
+bool_t
+CFReader_local_exists(CompoundFileReader *self, const CharBuf *name) {
+ if (Hash_Fetch(self->records, (Obj*)name)) { return true; }
+ if (Folder_Local_Exists(self->real_folder, name)) { return true; }
+ return false;
+}
+
+bool_t
+CFReader_local_is_directory(CompoundFileReader *self, const CharBuf *name) {
+ if (Hash_Fetch(self->records, (Obj*)name)) { return false; }
+ if (Folder_Local_Is_Directory(self->real_folder, name)) { return true; }
+ return false;
+}
+
+void
+CFReader_close(CompoundFileReader *self) {
+ InStream_Close(self->instream);
+}
+
+bool_t
+CFReader_local_mkdir(CompoundFileReader *self, const CharBuf *name) {
+ if (Hash_Fetch(self->records, (Obj*)name)) {
+ Err_set_error(Err_new(CB_newf("Can't MkDir: '%o' exists", name)));
+ return false;
+ }
+ else {
+ bool_t result = Folder_Local_MkDir(self->real_folder, name);
+ if (!result) { ERR_ADD_FRAME(Err_get_error()); }
+ return result;
+ }
+}
+
+Folder*
+CFReader_local_find_folder(CompoundFileReader *self, const CharBuf *name) {
+ if (Hash_Fetch(self->records, (Obj*)name)) { return false; }
+ return Folder_Local_Find_Folder(self->real_folder, name);
+}
+
+DirHandle*
+CFReader_local_open_dir(CompoundFileReader *self) {
+ return (DirHandle*)CFReaderDH_new(self);
+}
+
+/****************************************************************************/
+
+CFReaderDirHandle*
+CFReaderDH_new(CompoundFileReader *cf_reader) {
+ CFReaderDirHandle *self
+ = (CFReaderDirHandle*)VTable_Make_Obj(CFREADERDIRHANDLE);
+ return CFReaderDH_init(self, cf_reader);
+}
+
+CFReaderDirHandle*
+CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader) {
+ DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
+ self->cf_reader = (CompoundFileReader*)INCREF(cf_reader);
+ self->elems = Hash_Keys(self->cf_reader->records);
+ self->tick = -1;
+ {
+ // Accumulate entries from real Folder.
+ DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder);
+ CharBuf *entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ VA_Push(self->elems, (Obj*)CB_Clone(entry));
+ }
+ DECREF(dh);
+ }
+ return self;
+}
+
+bool_t
+CFReaderDH_close(CFReaderDirHandle *self) {
+ if (self->elems) {
+ VA_Dec_RefCount(self->elems);
+ self->elems = NULL;
+ }
+ if (self->cf_reader) {
+ CFReader_Dec_RefCount(self->cf_reader);
+ self->cf_reader = NULL;
+ }
+ return true;
+}
+
+bool_t
+CFReaderDH_next(CFReaderDirHandle *self) {
+ if (self->elems) {
+ self->tick++;
+ if (self->tick < (int32_t)VA_Get_Size(self->elems)) {
+ CharBuf *path = (CharBuf*)CERTIFY(
+ VA_Fetch(self->elems, self->tick), CHARBUF);
+ CB_Mimic(self->entry, (Obj*)path);
+ return true;
+ }
+ else {
+ self->tick--;
+ return false;
+ }
+ }
+ return false;
+}
+
+bool_t
+CFReaderDH_entry_is_dir(CFReaderDirHandle *self) {
+ if (self->elems) {
+ CharBuf *name = (CharBuf*)VA_Fetch(self->elems, self->tick);
+ if (name) {
+ return CFReader_Local_Is_Directory(self->cf_reader, name);
+ }
+ }
+ return false;
+}
+
+
diff --git a/core/Lucy/Store/CompoundFileReader.cfh b/core/Lucy/Store/CompoundFileReader.cfh
new file mode 100644
index 0000000..697df15
--- /dev/null
+++ b/core/Lucy/Store/CompoundFileReader.cfh
@@ -0,0 +1,114 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read from a compound file.
+ *
+ * A CompoundFileReader provides access to the files contained within the
+ * compound file format written by CompoundFileWriter. The InStream objects
+ * it spits out behave largely like InStreams opened against discrete files --
+ * e.g. Seek(0) seeks to the beginning of the sub-file, not the beginning of
+ * the compound file.
+ *
+ * Each of the InStreams spawned maintains its own memory buffer; however,
+ * they all share a single filehandle. This allows Lucy to get around
+ * the limitations that many operating systems place on the number of
+ * available filehandles.
+ */
+
+class Lucy::Store::CompoundFileReader cnick CFReader
+ inherits Lucy::Store::Folder {
+
+ Folder *real_folder;
+ Hash *records;
+ InStream *instream;
+ int32_t format;
+
+ inert incremented nullable CompoundFileReader*
+ open(Folder *folder);
+
+ /** Return a new CompoundFileReader or set Err_error and return NULL.
+ *
+ * @param folder A folder containing compound files.
+ */
+ inert nullable CompoundFileReader*
+ do_open(CompoundFileReader *self, Folder *folder);
+
+ Folder*
+ Get_Real_Folder(CompoundFileReader *self);
+
+ void
+ Set_Path(CompoundFileReader *self, const CharBuf *path);
+
+ public void
+ Close(CompoundFileReader *self);
+
+ public void
+ Destroy(CompoundFileReader *self);
+
+ bool_t
+ Local_Delete(CompoundFileReader *self, const CharBuf *name);
+
+ bool_t
+ Local_Exists(CompoundFileReader *self, const CharBuf *name);
+
+ bool_t
+ Local_Is_Directory(CompoundFileReader *self, const CharBuf *name);
+
+ incremented nullable FileHandle*
+ Local_Open_FileHandle(CompoundFileReader *self, const CharBuf *name,
+ uint32_t flags);
+
+ incremented nullable InStream*
+ Local_Open_In(CompoundFileReader *self, const CharBuf *name);
+
+ bool_t
+ Local_MkDir(CompoundFileReader *self, const CharBuf *name);
+
+ nullable Folder*
+ Local_Find_Folder(CompoundFileReader *self, const CharBuf *name);
+
+ incremented nullable DirHandle*
+ Local_Open_Dir(CompoundFileReader *self);
+}
+
+/** DirHandle for CompoundFileReader.
+ */
+class Lucy::Store::CFReaderDirHandle cnick CFReaderDH
+ inherits Lucy::Store::DirHandle {
+
+ CompoundFileReader *cf_reader;
+ VArray *elems;
+ int32_t tick;
+
+ inert incremented CFReaderDirHandle*
+ new(CompoundFileReader *cf_reader);
+
+ inert CFReaderDirHandle*
+ init(CFReaderDirHandle *self, CompoundFileReader *cf_reader);
+
+ bool_t
+ Next(CFReaderDirHandle *self);
+
+ bool_t
+ Entry_Is_Dir(CFReaderDirHandle *self);
+
+ bool_t
+ Close(CFReaderDirHandle *self);
+}
+
+
diff --git a/core/Lucy/Store/CompoundFileWriter.c b/core/Lucy/Store/CompoundFileWriter.c
new file mode 100644
index 0000000..c9d0da2
--- /dev/null
+++ b/core/Lucy/Store/CompoundFileWriter.c
@@ -0,0 +1,182 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_COMPOUNDFILEWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Util/Json.h"
+
+int32_t CFWriter_current_file_format = 2;
+
+// Helper which does the heavy lifting for CFWriter_consolidate.
+static void
+S_do_consolidate(CompoundFileWriter *self);
+
+// Clean up files which may be left over from previous merge attempts.
+static void
+S_clean_up_old_temp_files(CompoundFileWriter *self);
+
+CompoundFileWriter*
+CFWriter_new(Folder *folder) {
+ CompoundFileWriter *self
+ = (CompoundFileWriter*)VTable_Make_Obj(COMPOUNDFILEWRITER);
+ return CFWriter_init(self, folder);
+}
+
+CompoundFileWriter*
+CFWriter_init(CompoundFileWriter *self, Folder *folder) {
+ self->folder = (Folder*)INCREF(folder);
+ return self;
+}
+
+void
+CFWriter_destroy(CompoundFileWriter *self) {
+ DECREF(self->folder);
+ SUPER_DESTROY(self, COMPOUNDFILEWRITER);
+}
+
+void
+CFWriter_consolidate(CompoundFileWriter *self) {
+ CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
+ if (Folder_Exists(self->folder, cfmeta_file)) {
+ THROW(ERR, "Merge already performed for %o",
+ Folder_Get_Path(self->folder));
+ }
+ else {
+ S_clean_up_old_temp_files(self);
+ S_do_consolidate(self);
+ }
+}
+
+static void
+S_clean_up_old_temp_files(CompoundFileWriter *self) {
+ Folder *folder = self->folder;
+ CharBuf *cfmeta_temp = (CharBuf*)ZCB_WRAP_STR("cfmeta.json.temp", 16);
+ CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
+
+ if (Folder_Exists(folder, cf_file)) {
+ if (!Folder_Delete(folder, cf_file)) {
+ THROW(ERR, "Can't delete '%o'", cf_file);
+ }
+ }
+ if (Folder_Exists(folder, cfmeta_temp)) {
+ if (!Folder_Delete(folder, cfmeta_temp)) {
+ THROW(ERR, "Can't delete '%o'", cfmeta_temp);
+ }
+ }
+}
+
+static void
+S_do_consolidate(CompoundFileWriter *self) {
+ Folder *folder = self->folder;
+ Hash *metadata = Hash_new(0);
+ Hash *sub_files = Hash_new(0);
+ VArray *files = Folder_List(folder, NULL);
+ VArray *merged = VA_new(VA_Get_Size(files));
+ CharBuf *cf_file = (CharBuf*)ZCB_WRAP_STR("cf.dat", 6);
+ OutStream *outstream = Folder_Open_Out(folder, (CharBuf*)cf_file);
+ uint32_t i, max;
+ bool_t rename_success;
+
+ if (!outstream) { RETHROW(INCREF(Err_get_error())); }
+
+ // Start metadata.
+ Hash_Store_Str(metadata, "files", 5, INCREF(sub_files));
+ Hash_Store_Str(metadata, "format", 6,
+ (Obj*)CB_newf("%i32", CFWriter_current_file_format));
+
+ CharBuf *infilepath = CB_new(30);
+ size_t base_len = 0;
+ VA_Sort(files, NULL, NULL);
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *infilename = (CharBuf*)VA_Fetch(files, i);
+
+ if (!CB_Ends_With_Str(infilename, ".json", 5)) {
+ InStream *instream = Folder_Open_In(folder, infilename);
+ Hash *file_data = Hash_new(2);
+ int64_t offset, len;
+
+ if (!instream) { RETHROW(INCREF(Err_get_error())); }
+
+ // Absorb the file.
+ offset = OutStream_Tell(outstream);
+ OutStream_Absorb(outstream, instream);
+ len = OutStream_Tell(outstream) - offset;
+
+ // Record offset and length.
+ Hash_Store_Str(file_data, "offset", 6,
+ (Obj*)CB_newf("%i64", offset));
+ Hash_Store_Str(file_data, "length", 6,
+ (Obj*)CB_newf("%i64", len));
+ CB_Set_Size(infilepath, base_len);
+ CB_Cat(infilepath, infilename);
+ Hash_Store(sub_files, (Obj*)infilepath, (Obj*)file_data);
+ VA_Push(merged, INCREF(infilename));
+
+ // Add filler NULL bytes so that every sub-file begins on a file
+ // position multiple of 8.
+ OutStream_Align(outstream, 8);
+
+ InStream_Close(instream);
+ DECREF(instream);
+ }
+ }
+ DECREF(infilepath);
+
+ // Write metadata to cfmeta file.
+ CharBuf *cfmeta_temp = (CharBuf*)ZCB_WRAP_STR("cfmeta.json.temp", 16);
+ CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
+ Json_spew_json((Obj*)metadata, (Folder*)self->folder, cfmeta_temp);
+ rename_success = Folder_Rename(self->folder, cfmeta_temp, cfmeta_file);
+ if (!rename_success) { RETHROW(INCREF(Err_get_error())); }
+
+ // Clean up.
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ DECREF(files);
+ DECREF(metadata);
+ {
+ /*
+ CharBuf *merged_file;
+ Obj *ignore;
+ Hash_Iterate(sub_files);
+ while (Hash_Next(sub_files, (Obj**)&merged_file, &ignore)) {
+ if (!Folder_Delete(folder, merged_file)) {
+ CharBuf *mess = MAKE_MESS("Can't delete '%o'", merged_file);
+ DECREF(sub_files);
+ Err_throw_mess(ERR, mess);
+ }
+ }
+ */
+ }
+ DECREF(sub_files);
+ for (uint32_t i = 0, max = VA_Get_Size(merged); i < max; i++) {
+ CharBuf *merged_file = (CharBuf*)VA_Fetch(merged, i);
+ if (!Folder_Delete(folder, merged_file)) {
+ CharBuf *mess = MAKE_MESS("Can't delete '%o'", merged_file);
+ DECREF(merged);
+ Err_throw_mess(ERR, mess);
+ }
+ }
+ DECREF(merged);
+}
+
+
diff --git a/core/Lucy/Store/CompoundFileWriter.cfh b/core/Lucy/Store/CompoundFileWriter.cfh
new file mode 100644
index 0000000..764c229
--- /dev/null
+++ b/core/Lucy/Store/CompoundFileWriter.cfh
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Consolidate segment files.
+ *
+ * CompoundFileWriter combines all the data files in a directory into a single
+ * "compound" file named "cf.dat". Metadata describing filename and
+ * filepointer information is stored in a "cfmeta.json" file.
+ *
+ * Nested subdirectories and files ending in ".json" are excluded from
+ * consolidation.
+ *
+ * Any given directory may only be consolidated once.
+ */
+
+class Lucy::Store::CompoundFileWriter cnick CFWriter
+ inherits Lucy::Object::Obj {
+
+ Folder *folder;
+
+ inert int32_t current_file_format;
+
+ inert incremented CompoundFileWriter*
+ new(Folder *folder);
+
+ inert CompoundFileWriter*
+ init(CompoundFileWriter *self, Folder *folder);
+
+ /** Perform the consolidation operation, building the cf.dat and
+ * cfmeta.json files.
+ *
+ * The commit point is a rename op, where a temp file gets renamed to the
+ * cfmeta file. After the commit completes, the source files are deleted.
+ */
+ void
+ Consolidate(CompoundFileWriter *self);
+
+ public void
+ Destroy(CompoundFileWriter *self);
+}
+
+
diff --git a/core/Lucy/Store/DirHandle.c b/core/Lucy/Store/DirHandle.c
new file mode 100644
index 0000000..f60f86c
--- /dev/null
+++ b/core/Lucy/Store/DirHandle.c
@@ -0,0 +1,48 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DIRHANDLE
+#include <stdarg.h>
+#include "Lucy/Util/ToolSet.h"
+#include "Lucy/Store/DirHandle.h"
+
+DirHandle*
+DH_init(DirHandle *self, const CharBuf *dir) {
+ self->dir = CB_Clone(dir);
+ self->entry = CB_new(32);
+ ABSTRACT_CLASS_CHECK(self, DIRHANDLE);
+ return self;
+}
+
+void
+DH_destroy(DirHandle *self) {
+ DH_Close(self);
+ DECREF(self->dir);
+ DECREF(self->entry);
+ SUPER_DESTROY(self, DIRHANDLE);
+}
+
+CharBuf*
+DH_get_dir(DirHandle *self) {
+ return self->dir;
+}
+
+CharBuf*
+DH_get_entry(DirHandle *self) {
+ return self->entry;
+}
+
+
diff --git a/core/Lucy/Store/DirHandle.cfh b/core/Lucy/Store/DirHandle.cfh
new file mode 100644
index 0000000..ae4acf5
--- /dev/null
+++ b/core/Lucy/Store/DirHandle.cfh
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Iterate over the files in a directory.
+ */
+abstract class Lucy::Store::DirHandle cnick DH
+ inherits Lucy::Object::Obj {
+
+ CharBuf *dir;
+ CharBuf *entry;
+
+ /** Abstract constructor.
+ *
+ * @param dir The path to the directory.
+ */
+ inert DirHandle*
+ init(DirHandle *self, const CharBuf *dir);
+
+ /** Proceed to the next entry in the directory.
+ *
+ * @return true on success, false when finished.
+ */
+ abstract bool_t
+ Next(DirHandle *self);
+
+ /** Attempt to close the DirHandle. Returns true on success, sets
+ * Err_error and returns false on failure.
+ */
+ abstract bool_t
+ Close(DirHandle *self);
+
+ /** Return the object's <code>dir</code> attribute.
+ */
+ CharBuf*
+ Get_Dir(DirHandle *self);
+
+ /** Return the path of the current entry. The value is updated by each
+ * call to Next(), and is only valid when Next() has returned
+ * successfully.
+ */
+ CharBuf*
+ Get_Entry(DirHandle *self);
+
+ /** Returns true if the current entry is a directory, false otherwise.
+ */
+ abstract bool_t
+ Entry_Is_Dir(DirHandle *self);
+
+ /** Returns true if the current entry is a symbolic link (or a Windows
+ * junction), false otherwise.
+ */
+ abstract bool_t
+ Entry_Is_Symlink(DirHandle *self);
+
+ /** Invokes Close(), but ignores any errors.
+ */
+ public void
+ Destroy(DirHandle *self);
+}
+
+
diff --git a/core/Lucy/Store/FSDirHandle.c b/core/Lucy/Store/FSDirHandle.c
new file mode 100644
index 0000000..fb65203
--- /dev/null
+++ b/core/Lucy/Store/FSDirHandle.c
@@ -0,0 +1,319 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FSDIRHANDLE
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <sys/stat.h>
+
+#include "Lucy/Util/ToolSet.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/VArray.h"
+#include "Lucy/Store/FSDirHandle.h"
+
+#ifdef CHY_HAS_SYS_TYPES_H
+ #include <sys/types.h>
+#endif
+
+FSDirHandle*
+FSDH_open(const CharBuf *dir) {
+ FSDirHandle *self = (FSDirHandle*)VTable_Make_Obj(FSDIRHANDLE);
+ return FSDH_do_open(self, dir);
+}
+
+void
+FSDH_destroy(FSDirHandle *self) {
+ // Throw away saved error -- it's too late to call Close() now.
+ DECREF(self->saved_error);
+ self->saved_error = NULL;
+ SUPER_DESTROY(self, FSDIRHANDLE);
+}
+
+static INLINE bool_t
+SI_is_updir(const char *name, size_t len) {
+ if (len == 2 && strncmp(name, "..", 2) == 0) {
+ return true;
+ }
+ else if (len == 1 && name[0] == '.') {
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+/********************************** Windows ********************************/
+#if (defined(CHY_HAS_WINDOWS_H) && !defined(__CYGWIN__))
+
+#include <windows.h>
+
+FSDirHandle*
+FSDH_do_open(FSDirHandle *self, const CharBuf *dir) {
+ size_t dir_path_size = CB_Get_Size(dir);
+ char *dir_path_ptr = (char*)CB_Get_Ptr8(dir);
+ char search_string[MAX_PATH + 1];
+ char *path_ptr = search_string;
+
+ DH_init((DirHandle*)self, dir);
+ self->sys_dir_entry = MALLOCATE(sizeof(WIN32_FIND_DATA));
+ self->sys_dirhandle = INVALID_HANDLE_VALUE;
+ self->saved_error = NULL;
+
+ if (dir_path_size >= MAX_PATH - 2) {
+ // Deal with Windows ceiling on file path lengths.
+ Err_set_error(Err_new(CB_newf("Directory path is too long: %o",
+ dir)));
+ LUCY_DECREF(self);
+ return NULL;
+ }
+
+ // Append trailing wildcard so Windows lists dir contents rather than just
+ // the dir name itself.
+ memcpy(path_ptr, dir_path_ptr, dir_path_size);
+ memcpy(path_ptr + dir_path_size, "\\*\0", 3);
+
+ self->sys_dirhandle
+ = FindFirstFile(search_string, (WIN32_FIND_DATA*)self->sys_dir_entry);
+ if (INVALID_HANDLE_VALUE == self->sys_dirhandle) {
+ // Directory inaccessible or doesn't exist.
+ Err_set_error(Err_new(CB_newf("Failed to open dir '%o'", dir)));
+ LUCY_DECREF(self);
+ return NULL;
+ }
+ else {
+ // Compensate for the fact that FindFirstFile has already returned the
+ // first entry but DirHandle's API requires that you call Next() to
+ // start the iterator.
+ self->delayed_iter = true;
+ }
+
+ return self;
+}
+
+bool_t
+FSDH_entry_is_dir(FSDirHandle *self) {
+ WIN32_FIND_DATA *find_data = (WIN32_FIND_DATA*)self->sys_dir_entry;
+ if (find_data) {
+ if ((find_data->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+bool_t
+FSDH_entry_is_symlink(FSDirHandle *self) {
+ WIN32_FIND_DATA *find_data = (WIN32_FIND_DATA*)self->sys_dir_entry;
+ if (find_data) {
+ if ((find_data->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+bool_t
+FSDH_close(FSDirHandle *self) {
+ if (self->sys_dirhandle && self->sys_dirhandle != INVALID_HANDLE_VALUE) {
+ HANDLE dirhandle = (HANDLE)self->sys_dirhandle;
+ self->sys_dirhandle = NULL;
+ if (dirhandle != INVALID_HANDLE_VALUE && !FindClose(dirhandle)) {
+ if (!self->saved_error) {
+ char *win_error = Err_win_error();
+ self->saved_error
+ = Err_new(CB_newf("Error while closing directory: %s",
+ win_error));
+ FREEMEM(win_error);
+ }
+ }
+ }
+ if (self->sys_dir_entry) {
+ FREEMEM(self->sys_dir_entry);
+ self->sys_dir_entry = NULL;
+ }
+
+ // If we encountered an error condition previously, report it now.
+ if (self->saved_error) {
+ Err_set_error((Err*)LUCY_INCREF(self->saved_error));
+ return false;
+ }
+ else {
+ return true;
+ }
+}
+
+bool_t
+FSDH_next(FSDirHandle *self) {
+ HANDLE dirhandle = (HANDLE)self->sys_dirhandle;
+ WIN32_FIND_DATA *find_data = (WIN32_FIND_DATA*)self->sys_dir_entry;
+
+ // Attempt to move forward or absorb cached iter.
+ if (!dirhandle || dirhandle == INVALID_HANDLE_VALUE) {
+ return false;
+ }
+ else if (self->delayed_iter) {
+ self->delayed_iter = false;
+ }
+ else if ((FindNextFile(dirhandle, find_data) == 0)) {
+ // Iterator exhausted. Verify that no errors were encountered.
+ CB_Set_Size(self->entry, 0);
+ if (GetLastError() != ERROR_NO_MORE_FILES) {
+ char *win_error = Err_win_error();
+ self->saved_error
+ = Err_new(CB_newf("Error while traversing directory: %s",
+ win_error));
+ FREEMEM(win_error);
+ }
+ return false;
+ }
+
+ // Process the results of the iteration.
+ {
+ size_t len = strlen(find_data->cFileName);
+ if (SI_is_updir(find_data->cFileName, len)) {
+ return FSDH_Next(self);
+ }
+ else {
+ CB_Mimic_Str(self->entry, find_data->cFileName, len);
+ return true;
+ }
+ }
+}
+
+/********************************** UNIXEN *********************************/
+#elif defined(CHY_HAS_DIRENT_H)
+
+#include <dirent.h>
+
+FSDirHandle*
+FSDH_do_open(FSDirHandle *self, const CharBuf *dir) {
+ char *dir_path_ptr = (char*)CB_Get_Ptr8(dir);
+
+ DH_init((DirHandle*)self, dir);
+ self->sys_dir_entry = NULL;
+ self->fullpath = NULL;
+
+ self->sys_dirhandle = opendir(dir_path_ptr);
+ if (!self->sys_dirhandle) {
+ Err_set_error(Err_new(CB_newf("Failed to opendir '%o'", dir)));
+ DECREF(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+bool_t
+FSDH_next(FSDirHandle *self) {
+ self->sys_dir_entry = (struct dirent*)readdir((DIR*)self->sys_dirhandle);
+ if (!self->sys_dir_entry) {
+ CB_Set_Size(self->entry, 0);
+ return false;
+ }
+ else {
+ struct dirent *sys_dir_entry = (struct dirent*)self->sys_dir_entry;
+ #ifdef CHY_HAS_DIRENT_D_NAMLEN
+ size_t len = sys_dir_entry->d_namlen;
+ #else
+ size_t len = strlen(sys_dir_entry->d_name);
+ #endif
+ if (SI_is_updir(sys_dir_entry->d_name, len)) {
+ return FSDH_Next(self);
+ }
+ else {
+ CB_Mimic_Str(self->entry, sys_dir_entry->d_name, len);
+ return true;
+ }
+ }
+}
+
+bool_t
+FSDH_entry_is_dir(FSDirHandle *self) {
+ struct dirent *sys_dir_entry = (struct dirent*)self->sys_dir_entry;
+ if (!sys_dir_entry) { return false; }
+
+ // If d_type is available, try to avoid a stat() call. If it's not, or if
+ // the type comes back as unknown, fall back to stat().
+ #ifdef CHY_HAS_DIRENT_D_TYPE
+ if (sys_dir_entry->d_type == DT_DIR) {
+ return true;
+ }
+ else if (sys_dir_entry->d_type != DT_UNKNOWN) {
+ return false;
+ }
+ #endif
+
+ struct stat stat_buf;
+ if (!self->fullpath) {
+ self->fullpath = CB_new(CB_Get_Size(self->dir) + 20);
+ }
+ CB_setf(self->fullpath, "%o%s%o", self->dir, CHY_DIR_SEP,
+ self->entry);
+ if (stat((char*)CB_Get_Ptr8(self->fullpath), &stat_buf) != -1) {
+ if (stat_buf.st_mode & S_IFDIR) { return true; }
+ }
+ return false;
+}
+
+bool_t
+FSDH_entry_is_symlink(FSDirHandle *self) {
+ struct dirent *sys_dir_entry = (struct dirent*)self->sys_dir_entry;
+ if (!sys_dir_entry) { return false; }
+
+ #ifdef CHY_HAS_DIRENT_D_TYPE
+ return sys_dir_entry->d_type == DT_LNK ? true : false;
+ #else
+ {
+ struct stat stat_buf;
+ if (!self->fullpath) {
+ self->fullpath = CB_new(CB_Get_Size(self->dir) + 20);
+ }
+ CB_setf(self->fullpath, "%o%s%o", self->dir, CHY_DIR_SEP,
+ self->entry);
+ if (stat((char*)CB_Get_Ptr8(self->fullpath), &stat_buf) != -1) {
+ if (stat_buf.st_mode & S_IFLNK) { return true; }
+ }
+ return false;
+ }
+ #endif // CHY_HAS_DIRENT_D_TYPE
+}
+
+bool_t
+FSDH_close(FSDirHandle *self) {
+ if (self->fullpath) {
+ CB_Dec_RefCount(self->fullpath);
+ self->fullpath = NULL;
+ }
+ if (self->sys_dirhandle) {
+ DIR *sys_dirhandle = (DIR*)self->sys_dirhandle;
+ self->sys_dirhandle = NULL;
+ if (closedir(sys_dirhandle) == -1) {
+ Err_set_error(Err_new(CB_newf("Error closing dirhandle: %s",
+ strerror(errno))));
+ return false;
+ }
+ }
+ return true;
+}
+
+#else
+ #error "Need either dirent.h or windows.h"
+#endif // CHY_HAS_DIRENT_H vs. CHY_HAS_WINDOWS_H
+
+
diff --git a/core/Lucy/Store/FSDirHandle.cfh b/core/Lucy/Store/FSDirHandle.cfh
new file mode 100644
index 0000000..aecb776
--- /dev/null
+++ b/core/Lucy/Store/FSDirHandle.cfh
@@ -0,0 +1,52 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** File system DirHandle.
+ */
+class Lucy::Store::FSDirHandle cnick FSDH
+ inherits Lucy::Store::DirHandle {
+
+ void *sys_dirhandle;
+ void *sys_dir_entry;
+ CharBuf *fullpath;
+ Err *saved_error;
+ bool_t delayed_iter;
+
+ inert incremented nullable FSDirHandle*
+ open(const CharBuf *path);
+
+ inert nullable FSDirHandle*
+ do_open(FSDirHandle *self, const CharBuf *path);
+
+ bool_t
+ Next(FSDirHandle *self);
+
+ bool_t
+ Entry_Is_Dir(FSDirHandle *self);
+
+ bool_t
+ Entry_Is_Symlink(FSDirHandle *self);
+
+ bool_t
+ Close(FSDirHandle *self);
+
+ public void
+ Destroy(FSDirHandle *self);
+}
+
+
diff --git a/core/Lucy/Store/FSFileHandle.c b/core/Lucy/Store/FSFileHandle.c
new file mode 100644
index 0000000..0c65e98
--- /dev/null
+++ b/core/Lucy/Store/FSFileHandle.c
@@ -0,0 +1,596 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FSFILEHANDLE
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <fcntl.h> // open, POSIX flags
+#include <stdarg.h>
+
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h> // close
+#endif
+
+#ifdef CHY_HAS_SYS_MMAN_H
+ #include <sys/mman.h>
+#elif defined(CHY_HAS_WINDOWS_H)
+ #include <windows.h>
+ #include <io.h>
+#else
+ #error "No support for memory mapped files"
+#endif
+
+#include "Lucy/Store/FSFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+
+// Convert FileHandle flags to POSIX flags.
+static INLINE int
+SI_posix_flags(uint32_t fh_flags) {
+ int posix_flags = 0;
+ if (fh_flags & FH_WRITE_ONLY) { posix_flags |= O_WRONLY; }
+ if (fh_flags & FH_READ_ONLY) { posix_flags |= O_RDONLY; }
+ if (fh_flags & FH_CREATE) { posix_flags |= O_CREAT; }
+ if (fh_flags & FH_EXCLUSIVE) { posix_flags |= O_EXCL; }
+#ifdef O_LARGEFILE
+ posix_flags |= O_LARGEFILE;
+#endif
+#ifdef _O_BINARY
+ posix_flags |= _O_BINARY;
+#endif
+ return posix_flags;
+}
+
+#define IS_64_BIT (SIZEOF_PTR == 8 ? true : false)
+
+// Memory map a region of the file with shared (read-only) permissions. If
+// the requested length is 0, return NULL. If an error occurs, return NULL
+// and set Err_error.
+static INLINE void*
+SI_map(FSFileHandle *self, int64_t offset, int64_t len);
+
+// Release a memory mapped region assigned by SI_map.
+static INLINE bool_t
+SI_unmap(FSFileHandle *self, char *ptr, int64_t len);
+
+// 32-bit or 64-bit inlined helpers for FSFH_window.
+static INLINE bool_t
+SI_window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+// Architecture- and OS- specific initialization for a read-only FSFileHandle.
+static INLINE bool_t
+SI_init_read_only(FSFileHandle *self);
+
+// Windows-specific routine needed for closing read-only handles.
+#ifdef CHY_HAS_WINDOWS_H
+static INLINE bool_t
+SI_close_win_handles(FSFileHandle *self);
+#endif
+
+FSFileHandle*
+FSFH_open(const CharBuf *path, uint32_t flags) {
+ FSFileHandle *self = (FSFileHandle*)VTable_Make_Obj(FSFILEHANDLE);
+ return FSFH_do_open(self, path, flags);
+}
+
+FSFileHandle*
+FSFH_do_open(FSFileHandle *self, const CharBuf *path, uint32_t flags) {
+ FH_do_open((FileHandle*)self, path, flags);
+ if (!path || !CB_Get_Size(path)) {
+ Err_set_error(Err_new(CB_newf("Missing required param 'path'")));
+ DECREF(self);
+ return NULL;
+ }
+
+ // Attempt to open file.
+ if (flags & FH_WRITE_ONLY) {
+ self->fd = open((char*)CB_Get_Ptr8(path), SI_posix_flags(flags), 0666);
+ if (self->fd == -1) {
+ self->fd = 0;
+ Err_set_error(Err_new(CB_newf("Attempt to open '%o' failed: %s",
+ path, strerror(errno))));
+ DECREF(self);
+ return NULL;
+ }
+ if (flags & FH_EXCLUSIVE) {
+ self->len = 0;
+ }
+ else {
+ // Derive length.
+ self->len = lseek64(self->fd, I64_C(0), SEEK_END);
+ if (self->len == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
+ self->path, strerror(errno))));
+ DECREF(self);
+ return NULL;
+ }
+ else {
+ int64_t check_val = lseek64(self->fd, I64_C(0), SEEK_SET);
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
+ self->path, strerror(errno))));
+ DECREF(self);
+ return NULL;
+ }
+ }
+ }
+ }
+ else if (flags & FH_READ_ONLY) {
+ if (SI_init_read_only(self)) {
+ // On 64-bit systems, map the whole file up-front.
+ if (IS_64_BIT && self->len) {
+ self->buf = (char*)SI_map(self, 0, self->len);
+ if (!self->buf) {
+ // An error occurred during SI_map, which has set
+ // Err_error for us already.
+ DECREF(self);
+ return NULL;
+ }
+ }
+ }
+ else {
+ DECREF(self);
+ return NULL;
+ }
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Must specify FH_READ_ONLY or FH_WRITE_ONLY to open '%o'",
+ path)));
+ DECREF(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+bool_t
+FSFH_close(FSFileHandle *self) {
+ // On 64-bit systems, cancel the whole-file mapping.
+ if (IS_64_BIT && (self->flags & FH_READ_ONLY) && self->buf != NULL) {
+ if (!SI_unmap(self, self->buf, self->len)) { return false; }
+ self->buf = NULL;
+ }
+
+ // Close system-specific handles.
+ if (self->fd) {
+ if (close(self->fd)) {
+ Err_set_error(Err_new(CB_newf("Failed to close file: %s",
+ strerror(errno))));
+ return false;
+ }
+ self->fd = 0;
+ }
+ #if (defined(CHY_HAS_WINDOWS_H) && !defined(CHY_HAS_SYS_MMAN_H))
+ if (self->win_fhandle) {
+ if (!SI_close_win_handles(self)) { return false; }
+ }
+ #endif
+
+ return true;
+}
+
+bool_t
+FSFH_write(FSFileHandle *self, const void *data, size_t len) {
+ if (len) {
+ // Write data, track file length, check for errors.
+ int64_t check_val = write(self->fd, data, len);
+ self->len += check_val;
+ if ((size_t)check_val != len) {
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf("Error when writing %u64 bytes: %s",
+ (uint64_t)len, strerror(errno))));
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Attempted to write %u64 bytes, but wrote %i64",
+ (uint64_t)len, check_val)));
+ }
+ return false;
+ }
+ }
+
+ return true;
+}
+
+int64_t
+FSFH_length(FSFileHandle *self) {
+ return self->len;
+}
+
+bool_t
+FSFH_window(FSFileHandle *self, FileWindow *window, int64_t offset,
+ int64_t len) {
+ const int64_t end = offset + len;
+ if (!(self->flags & FH_READ_ONLY)) {
+ Err_set_error(Err_new(CB_newf("Can't read from write-only handle")));
+ return false;
+ }
+ else if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from negative offset %i64",
+ offset)));
+ return false;
+ }
+ else if (end > self->len) {
+ Err_set_error(Err_new(CB_newf("Tried to read past EOF: offset %i64 + request %i64 > len %i64",
+ offset, len, self->len)));
+ return false;
+ }
+ else {
+ return SI_window(self, window, offset, len);
+ }
+}
+
+/********************************* 64-bit *********************************/
+
+#if IS_64_BIT
+
+static INLINE bool_t
+SI_window(FSFileHandle *self, FileWindow *window, int64_t offset,
+ int64_t len) {
+ FileWindow_Set_Window(window, self->buf + offset, offset, len);
+ return true;
+}
+
+bool_t
+FSFH_release_window(FSFileHandle *self, FileWindow *window) {
+ UNUSED_VAR(self);
+ FileWindow_Set_Window(window, NULL, 0, 0);
+ return true;
+}
+
+bool_t
+FSFH_read(FSFileHandle *self, char *dest, int64_t offset, size_t len) {
+ const int64_t end = offset + len;
+
+ if (self->flags & FH_WRITE_ONLY) {
+ Err_set_error(Err_new(CB_newf("Can't read from write-only filehandle")));
+ return false;
+ }
+ if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from an offset less than 0 (%i64)",
+ offset)));
+ return false;
+ }
+ else if (end > self->len) {
+ Err_set_error(Err_new(CB_newf("Tried to read past EOF: offset %i64 + request %u64 > len %i64",
+ offset, (uint64_t)len, self->len)));
+ return false;
+ }
+ memcpy(dest, self->buf + offset, len);
+ return true;
+}
+
+/********************************* 32-bit *********************************/
+
+#else
+
+static INLINE bool_t
+SI_window(FSFileHandle *self, FileWindow *window, int64_t offset,
+ int64_t len) {
+ // Release the previously mmap'd region, if any.
+ FSFH_release_window(self, window);
+
+ {
+ // Start map on a page boundary. Ensure that the window is at
+ // least wide enough to view all the data spec'd in the original
+ // request.
+ const int64_t remainder = offset % self->page_size;
+ const int64_t adjusted_offset = offset - remainder;
+ const int64_t adjusted_len = len + remainder;
+ char *const buf
+ = (char*)SI_map(self, adjusted_offset, adjusted_len);
+ if (len && buf == NULL) {
+ return false;
+ }
+ else {
+ FileWindow_Set_Window(window, buf, adjusted_offset,
+ adjusted_len);
+ }
+ }
+
+ return true;
+}
+
+bool_t
+FSFH_release_window(FSFileHandle *self, FileWindow *window) {
+ if (!SI_unmap(self, window->buf, window->len)) { return false; }
+ FileWindow_Set_Window(window, NULL, 0, 0);
+ return true;
+}
+
+#endif // IS_64_BIT vs. 32-bit
+
+/********************************* UNIXEN *********************************/
+
+#ifdef CHY_HAS_SYS_MMAN_H
+
+static INLINE bool_t
+SI_init_read_only(FSFileHandle *self) {
+ // Open.
+ self->fd = open((char*)CB_Get_Ptr8(self->path),
+ SI_posix_flags(self->flags), 0666);
+ if (self->fd == -1) {
+ self->fd = 0;
+ Err_set_error(Err_new(CB_newf("Can't open '%o': %s", self->path,
+ strerror(errno))));
+ return false;
+ }
+
+ // Derive len.
+ self->len = lseek64(self->fd, I64_C(0), SEEK_END);
+ if (self->len == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s", self->path,
+ strerror(errno))));
+ return false;
+ }
+ else {
+ int64_t check_val = lseek64(self->fd, I64_C(0), SEEK_SET);
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf("lseek64 on %o failed: %s",
+ self->path, strerror(errno))));
+ return false;
+ }
+ }
+
+ // Get system page size.
+#if defined(_SC_PAGESIZE)
+ self->page_size = sysconf(_SC_PAGESIZE);
+#elif defined(_SC_PAGE_SIZE)
+ self->page_size = sysconf(_SC_PAGE_SIZE);
+#else
+ #error "Can't determine system memory page size"
+#endif
+
+ return true;
+}
+
+static INLINE void*
+SI_map(FSFileHandle *self, int64_t offset, int64_t len) {
+ void *buf = NULL;
+
+ if (len) {
+ // Read-only memory mapping.
+ buf = mmap(NULL, len, PROT_READ, MAP_SHARED, self->fd, offset);
+ if (buf == (void*)(-1)) {
+ Err_set_error(Err_new(CB_newf("mmap of offset %i64 and length %i64 (page size %i64) "
+ "against '%o' failed: %s",
+ offset, len, self->page_size,
+ self->path, strerror(errno))));
+ return NULL;
+ }
+ }
+
+ return buf;
+}
+
+static INLINE bool_t
+SI_unmap(FSFileHandle *self, char *buf, int64_t len) {
+ if (buf != NULL) {
+ if (munmap(buf, len)) {
+ Err_set_error(Err_new(CB_newf("Failed to munmap '%o': %s",
+ self->path, strerror(errno))));
+ return false;
+ }
+ }
+ return true;
+}
+
+#if !IS_64_BIT
+bool_t
+FSFH_read(FSFileHandle *self, char *dest, int64_t offset, size_t len) {
+ int64_t check_val;
+
+ // Sanity check.
+ if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from an offset less than 0 (%i64)",
+ offset)));
+ return false;
+ }
+
+ // Read.
+ check_val = pread64(self->fd, dest, len, offset);
+ if (check_val != (int64_t)len) {
+ if (check_val == -1) {
+ Err_set_error(Err_new(CB_newf("Tried to read %u64 bytes, got %i64: %s",
+ (uint64_t)len, check_val, strerror(errno))));
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Tried to read %u64 bytes, got %i64",
+ (uint64_t)len, check_val)));
+ }
+ return false;
+ }
+
+ return true;
+}
+#endif // !IS_64_BIT
+
+/********************************* WINDOWS **********************************/
+
+#elif defined(CHY_HAS_WINDOWS_H)
+
+static INLINE bool_t
+SI_init_read_only(FSFileHandle *self) {
+ LARGE_INTEGER large_int;
+ char *filepath = (char*)CB_Get_Ptr8(self->path);
+ SYSTEM_INFO sys_info;
+
+ // Get system page size.
+ GetSystemInfo(&sys_info);
+ self->page_size = sys_info.dwAllocationGranularity;
+
+ // Open.
+ self->win_fhandle = CreateFile(
+ filepath,
+ GENERIC_READ,
+ FILE_SHARE_READ,
+ NULL,
+ OPEN_EXISTING,
+ FILE_ATTRIBUTE_READONLY | FILE_FLAG_OVERLAPPED,
+ NULL
+ );
+ if (self->win_fhandle == INVALID_HANDLE_VALUE) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("CreateFile for %o failed: %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+
+ // Derive len.
+ GetFileSizeEx(self->win_fhandle, &large_int);
+ self->len = large_int.QuadPart;
+ if (self->len < 0) {
+ Err_set_error(Err_new(CB_newf("GetFileSizeEx for %o returned a negative length: '%i64'",
+ self->path, self->len)));
+ return false;
+ }
+
+ // Init mapping handle.
+ self->buf = NULL;
+ if (self->len) {
+ self->win_maphandle = CreateFileMapping(self->win_fhandle, NULL,
+ PAGE_READONLY, 0, 0, NULL);
+ if (self->win_maphandle == NULL) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("CreateFileMapping for %o failed: %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static INLINE void*
+SI_map(FSFileHandle *self, int64_t offset, int64_t len) {
+ void *buf = NULL;
+
+ if (len) {
+ // Read-only memory map.
+ uint64_t offs = (uint64_t)offset;
+ DWORD file_offset_hi = offs >> 32;
+ DWORD file_offset_lo = offs & 0xFFFFFFFF;
+ size_t amount = (size_t)len;
+ buf = MapViewOfFile(self->win_maphandle, FILE_MAP_READ,
+ file_offset_hi, file_offset_lo, amount);
+ if (buf == NULL) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("MapViewOfFile for %o failed: %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ }
+ }
+
+ return buf;
+}
+
+static INLINE bool_t
+SI_unmap(FSFileHandle *self, char *buf, int64_t len) {
+ if (buf != NULL) {
+ if (!UnmapViewOfFile(buf)) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to unmap '%o': %s",
+ self->path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ }
+ return true;
+}
+
+static INLINE bool_t
+SI_close_win_handles(FSFileHandle *self) {
+ // Close both standard handle and mapping handle.
+ if (self->win_maphandle) {
+ if (!CloseHandle(self->win_maphandle)) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to close file mapping handle: %s",
+ win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ self->win_maphandle = NULL;
+ }
+ if (self->win_fhandle) {
+ if (!CloseHandle(self->win_fhandle)) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to close file handle: %s",
+ win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+ self->win_fhandle = NULL;
+ }
+
+ return true;
+}
+
+#if !IS_64_BIT
+bool_t
+FSFH_read(FSFileHandle *self, char *dest, int64_t offset, size_t len) {
+ BOOL check_val;
+ DWORD got;
+ OVERLAPPED read_op_state;
+ uint64_t offs = (uint64_t)offset;
+
+ read_op_state.hEvent = NULL;
+ read_op_state.OffsetHigh = offs >> 32;
+ read_op_state.Offset = offs & 0xFFFFFFFF;
+
+ // Sanity check.
+ if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from an offset less than 0 (%i64)",
+ offset)));
+ return false;
+ }
+
+ // ReadFile() takes a DWORD (unsigned 32-bit integer) as a length
+ // argument, so throw a sensible error rather than wrap around.
+ if (len > U32_MAX) {
+ Err_set_error(Err_new(CB_newf("Can't read more than 4 GB (%u64)",
+ (uint64_t)len)));
+ return false;
+ }
+
+ // Read.
+ check_val = ReadFile(self->win_fhandle, dest, len, &got, &read_op_state);
+ if (!check_val && GetLastError() == ERROR_IO_PENDING) {
+ // Read has been queued by the OS and will soon complete. Wait for
+ // it, since this is a blocking IO call from the point of the rest of
+ // the library.
+ check_val = GetOverlappedResult(self->win_fhandle, &read_op_state,
+ &got, TRUE);
+ }
+
+ // Verify that the read has succeeded by now.
+ if (!check_val) {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("Failed to read %u64 bytes: %s",
+ (uint64_t)len, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+
+ return true;
+}
+#endif // !IS_64_BIT
+
+#endif // CHY_HAS_SYS_MMAN_H vs. CHY_HAS_WINDOWS_H
+
+
diff --git a/core/Lucy/Store/FSFileHandle.cfh b/core/Lucy/Store/FSFileHandle.cfh
new file mode 100644
index 0000000..1f0019b
--- /dev/null
+++ b/core/Lucy/Store/FSFileHandle.cfh
@@ -0,0 +1,62 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** File system FileHandle.
+ */
+class Lucy::Store::FSFileHandle cnick FSFH
+ inherits Lucy::Store::FileHandle {
+
+ int fd;
+ void *win_fhandle;
+ void *win_maphandle;
+ int64_t len;
+ int64_t page_size;
+ char *buf;
+
+ /** Return a new FSFileHandle, or set Err_error and return NULL if
+ * something goes wrong.
+ *
+ * @param path Filepath.
+ * @param flags FileHandle constructor flags.
+ */
+ inert incremented nullable FSFileHandle*
+ open(const CharBuf *path = NULL, uint32_t flags);
+
+ inert nullable FSFileHandle*
+ do_open(FSFileHandle *self, const CharBuf *path = NULL, uint32_t flags);
+
+ bool_t
+ Window(FSFileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+ bool_t
+ Release_Window(FSFileHandle *self, FileWindow *window);
+
+ bool_t
+ Read(FSFileHandle *self, char *dest, int64_t offset, size_t len);
+
+ bool_t
+ Write(FSFileHandle *self, const void *data, size_t len);
+
+ int64_t
+ Length(FSFileHandle *self);
+
+ bool_t
+ Close(FSFileHandle *self);
+}
+
+
diff --git a/core/Lucy/Store/FSFolder.c b/core/Lucy/Store/FSFolder.c
new file mode 100644
index 0000000..244aaca
--- /dev/null
+++ b/core/Lucy/Store/FSFolder.c
@@ -0,0 +1,337 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FSFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include <ctype.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/stat.h>
+
+#ifdef CHY_HAS_SYS_TYPES_H
+ #include <sys/types.h>
+#endif
+
+// For rmdir, (hard) link.
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+
+// For mkdir, rmdir.
+#ifdef CHY_HAS_DIRECT_H
+ #include <direct.h>
+#endif
+
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/FSDirHandle.h"
+#include "Lucy/Store/FSFileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+// Return a CharBuf containing a platform-specific absolute filepath.
+static CharBuf*
+S_fullpath(FSFolder *self, const CharBuf *path);
+
+// Return true if the supplied path is a directory.
+static bool_t
+S_dir_ok(const CharBuf *path);
+
+// Create a directory, or set Err_error and return false.
+bool_t
+S_create_dir(const CharBuf *path);
+
+// Return true unless the supplied path contains a slash.
+bool_t
+S_is_local_entry(const CharBuf *path);
+
+// Create a hard link.
+bool_t
+S_hard_link(CharBuf *from_path, CharBuf *to_path);
+
+FSFolder*
+FSFolder_new(const CharBuf *path) {
+ FSFolder *self = (FSFolder*)VTable_Make_Obj(FSFOLDER);
+ return FSFolder_init(self, path);
+}
+
+FSFolder*
+FSFolder_init(FSFolder *self, const CharBuf *path) {
+ CharBuf *abs_path = FSFolder_absolutify(path);
+ Folder_init((Folder*)self, abs_path);
+ DECREF(abs_path);
+ return self;
+}
+
+void
+FSFolder_initialize(FSFolder *self) {
+ if (!S_dir_ok(self->path)) {
+ if (!S_create_dir(self->path)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ }
+}
+
+bool_t
+FSFolder_check(FSFolder *self) {
+ return S_dir_ok(self->path);
+}
+
+FileHandle*
+FSFolder_local_open_filehandle(FSFolder *self, const CharBuf *name,
+ uint32_t flags) {
+ CharBuf *fullpath = S_fullpath(self, name);
+ FSFileHandle *fh = FSFH_open(fullpath, flags);
+ if (!fh) { ERR_ADD_FRAME(Err_get_error()); }
+ DECREF(fullpath);
+ return (FileHandle*)fh;
+}
+
+bool_t
+FSFolder_local_mkdir(FSFolder *self, const CharBuf *name) {
+ CharBuf *dir = S_fullpath(self, name);
+ bool_t result = S_create_dir(dir);
+ if (!result) { ERR_ADD_FRAME(Err_get_error()); }
+ DECREF(dir);
+ return result;
+}
+
+DirHandle*
+FSFolder_local_open_dir(FSFolder *self) {
+ DirHandle *dh = (DirHandle*)FSDH_open(self->path);
+ if (!dh) { ERR_ADD_FRAME(Err_get_error()); }
+ return dh;
+}
+
+bool_t
+FSFolder_local_exists(FSFolder *self, const CharBuf *name) {
+ if (Hash_Fetch(self->entries, (Obj*)name)) {
+ return true;
+ }
+ else if (!S_is_local_entry(name)) {
+ return false;
+ }
+ else {
+ struct stat stat_buf;
+ CharBuf *fullpath = S_fullpath(self, name);
+ bool_t retval = false;
+ if (stat((char*)CB_Get_Ptr8(fullpath), &stat_buf) != -1) {
+ retval = true;
+ }
+ DECREF(fullpath);
+ return retval;
+ }
+}
+
+bool_t
+FSFolder_local_is_directory(FSFolder *self, const CharBuf *name) {
+ // Check for a cached object, then fall back to a system call.
+ Obj *elem = Hash_Fetch(self->entries, (Obj*)name);
+ if (elem && Obj_Is_A(elem, FOLDER)) {
+ return true;
+ }
+ else {
+ CharBuf *fullpath = S_fullpath(self, name);
+ bool_t result = S_dir_ok(fullpath);
+ DECREF(fullpath);
+ return result;
+ }
+}
+
+bool_t
+FSFolder_rename(FSFolder *self, const CharBuf* from, const CharBuf *to) {
+ CharBuf *from_path = S_fullpath(self, from);
+ CharBuf *to_path = S_fullpath(self, to);
+ bool_t retval = !rename((char*)CB_Get_Ptr8(from_path),
+ (char*)CB_Get_Ptr8(to_path));
+ if (!retval) {
+ Err_set_error(Err_new(CB_newf("rename from '%o' to '%o' failed: %s",
+ from_path, to_path, strerror(errno))));
+ }
+ DECREF(from_path);
+ DECREF(to_path);
+ return retval;
+}
+
+bool_t
+FSFolder_hard_link(FSFolder *self, const CharBuf *from,
+ const CharBuf *to) {
+ CharBuf *from_path = S_fullpath(self, from);
+ CharBuf *to_path = S_fullpath(self, to);
+ bool_t retval = S_hard_link(from_path, to_path);
+ DECREF(from_path);
+ DECREF(to_path);
+ return retval;
+}
+
+bool_t
+FSFolder_local_delete(FSFolder *self, const CharBuf *name) {
+ CharBuf *fullpath = S_fullpath(self, name);
+ char *path_ptr = (char*)CB_Get_Ptr8(fullpath);
+#ifdef CHY_REMOVE_ZAPS_DIRS
+ bool_t result = !remove(path_ptr);
+#else
+ bool_t result = !rmdir(path_ptr) || !remove(path_ptr);
+#endif
+ DECREF(Hash_Delete(self->entries, (Obj*)name));
+ DECREF(fullpath);
+ return result;
+}
+
+void
+FSFolder_close(FSFolder *self) {
+ Hash_Clear(self->entries);
+}
+
+Folder*
+FSFolder_local_find_folder(FSFolder *self, const CharBuf *name) {
+ Folder *subfolder = NULL;
+ if (!name || !CB_Get_Size(name)) {
+ // No entity can be identified by NULL or empty string.
+ return NULL;
+ }
+ else if (!S_is_local_entry(name)) {
+ return NULL;
+ }
+ else if (CB_Starts_With_Str(name, ".", 1)) {
+ // Don't allow access outside of the main dir.
+ return NULL;
+ }
+ else if (NULL != (subfolder = (Folder*)Hash_Fetch(self->entries, (Obj*)name))) {
+ if (Folder_Is_A(subfolder, FOLDER)) {
+ return subfolder;
+ }
+ else {
+ return NULL;
+ }
+ }
+
+ CharBuf *fullpath = S_fullpath(self, name);
+ if (S_dir_ok(fullpath)) {
+ subfolder = (Folder*)FSFolder_new(fullpath);
+ if (!subfolder) {
+ DECREF(fullpath);
+ THROW(ERR, "Failed to open FSFolder at '%o'", fullpath);
+ }
+ // Try to open a CompoundFileReader. On failure, just use the
+ // existing folder.
+ CharBuf *cfmeta_file = (CharBuf*)ZCB_WRAP_STR("cfmeta.json", 11);
+ if (Folder_Local_Exists(subfolder, cfmeta_file)) {
+ CompoundFileReader *cf_reader = CFReader_open(subfolder);
+ if (cf_reader) {
+ DECREF(subfolder);
+ subfolder = (Folder*)cf_reader;
+ }
+ }
+ Hash_Store(self->entries, (Obj*)name, (Obj*)subfolder);
+ }
+ DECREF(fullpath);
+
+ return subfolder;
+}
+
+static CharBuf*
+S_fullpath(FSFolder *self, const CharBuf *path) {
+ CharBuf *fullpath = CB_newf("%o%s%o", self->path, DIR_SEP, path);
+ if (DIR_SEP[0] != '/') {
+ CB_Swap_Chars(fullpath, '/', DIR_SEP[0]);
+ }
+ return fullpath;
+}
+
+static bool_t
+S_dir_ok(const CharBuf *path) {
+ struct stat stat_buf;
+ if (stat((char*)CB_Get_Ptr8(path), &stat_buf) != -1) {
+ if (stat_buf.st_mode & S_IFDIR) { return true; }
+ }
+ return false;
+}
+
+bool_t
+S_create_dir(const CharBuf *path) {
+ if (-1 == chy_makedir((char*)CB_Get_Ptr8(path), 0777)) {
+ Err_set_error(Err_new(CB_newf("Couldn't create directory '%o': %s",
+ path, strerror(errno))));
+ return false;
+ }
+ return true;
+}
+
+bool_t
+S_is_local_entry(const CharBuf *path) {
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ uint32_t code_point;
+ while (0 != (code_point = ZCB_Nip_One(scratch))) {
+ if (code_point == '/') { return false; }
+ }
+ return true;
+}
+
+/***************************************************************************/
+
+#if (defined(CHY_HAS_WINDOWS_H) && !defined(__CYGWIN__))
+
+// Windows.h defines INCREF and DECREF, so we include it only at the end of
+// this file and undef those symbols.
+#undef INCREF
+#undef DECREF
+
+#include <windows.h>
+
+bool_t
+S_hard_link(CharBuf *from_path, CharBuf *to_path) {
+ char *from8 = (char*)CB_Get_Ptr8(from_path);
+ char *to8 = (char*)CB_Get_Ptr8(to_path);
+
+ if (CreateHardLink(to8, from8, NULL)) {
+ return true;
+ }
+ else {
+ char *win_error = Err_win_error();
+ Err_set_error(Err_new(CB_newf("CreateHardLink for new file '%o' from '%o' failed: %s",
+ to_path, from_path, win_error)));
+ FREEMEM(win_error);
+ return false;
+ }
+}
+
+#elif (defined(CHY_HAS_UNISTD_H))
+
+bool_t
+S_hard_link(CharBuf *from_path, CharBuf *to_path) {
+ char *from8 = (char*)CB_Get_Ptr8(from_path);
+ char *to8 = (char*)CB_Get_Ptr8(to_path);
+
+ if (-1 == link(from8, to8)) {
+ Err_set_error(Err_new(CB_newf("hard link for new file '%o' from '%o' failed: %s",
+ to_path, from_path, strerror(errno))));
+ return false;
+ }
+ else {
+ return true;
+ }
+}
+
+#else
+ #error "Need either windows.h or unistd.h"
+#endif /* CHY_HAS_UNISTD_H vs. CHY_HAS_WINDOWS_H */
+
+
diff --git a/core/Lucy/Store/FSFolder.cfh b/core/Lucy/Store/FSFolder.cfh
new file mode 100644
index 0000000..416601b
--- /dev/null
+++ b/core/Lucy/Store/FSFolder.cfh
@@ -0,0 +1,87 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** File System implementation of Folder.
+ *
+ * Implementation of L<Lucy::Store::Folder> using a single file system
+ * directory and multiple files.
+ */
+
+class Lucy::Store::FSFolder inherits Lucy::Store::Folder {
+
+ inert incremented FSFolder*
+ new(const CharBuf *path);
+
+ /**
+ * @param path Location of the index. If the specified directory does
+ * not exist already, it will NOT be created, in order to prevent
+ * misconfigured read applications from spawning bogus files -- so it may
+ * be necessary to create the directory yourself.
+ */
+ public inert FSFolder*
+ init(FSFolder *self, const CharBuf *path);
+
+ /** Attempt to create the directory specified by <code>path</code>.
+ */
+ public void
+ Initialize(FSFolder *self);
+
+ /** Verify that <code>path</code> is a directory. TODO: check
+ * permissions.
+ */
+ public bool_t
+ Check(FSFolder *self);
+
+ public void
+ Close(FSFolder *self);
+
+ incremented nullable FileHandle*
+ Local_Open_FileHandle(FSFolder *self, const CharBuf *name,
+ uint32_t flags);
+
+ incremented nullable DirHandle*
+ Local_Open_Dir(FSFolder *self);
+
+ bool_t
+ Local_MkDir(FSFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Exists(FSFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Is_Directory(FSFolder *self, const CharBuf *name);
+
+ nullable Folder*
+ Local_Find_Folder(FSFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Delete(FSFolder *self, const CharBuf *name);
+
+ public bool_t
+ Rename(FSFolder *self, const CharBuf* from, const CharBuf *to);
+
+ public bool_t
+ Hard_Link(FSFolder *self, const CharBuf *from, const CharBuf *to);
+
+ /** Transform a relative path into an abolute path.
+ */
+ inert incremented CharBuf*
+ absolutify(const CharBuf *path);
+}
+
+
diff --git a/core/Lucy/Store/FileHandle.c b/core/Lucy/Store/FileHandle.c
new file mode 100644
index 0000000..03af95c
--- /dev/null
+++ b/core/Lucy/Store/FileHandle.c
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdarg.h>
+
+#define C_LUCY_FILEHANDLE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/FileHandle.h"
+
+int32_t FH_object_count = 0;
+
+FileHandle*
+FH_do_open(FileHandle *self, const CharBuf *path, uint32_t flags) {
+ self->path = path ? CB_Clone(path) : CB_new(0);
+ self->flags = flags;
+
+ // Track number of live FileHandles released into the wild.
+ FH_object_count++;
+
+ ABSTRACT_CLASS_CHECK(self, FILEHANDLE);
+ return self;
+}
+
+void
+FH_destroy(FileHandle *self) {
+ FH_Close(self);
+ DECREF(self->path);
+ SUPER_DESTROY(self, FILEHANDLE);
+
+ // Decrement count of FileHandle objects in existence.
+ FH_object_count--;
+}
+
+bool_t
+FH_grow(FileHandle *self, int64_t length) {
+ UNUSED_VAR(self);
+ UNUSED_VAR(length);
+ return true;
+}
+
+void
+FH_set_path(FileHandle *self, const CharBuf *path) {
+ CB_Mimic(self->path, (Obj*)path);
+}
+
+CharBuf*
+FH_get_path(FileHandle *self) {
+ return self->path;
+}
+
+
diff --git a/core/Lucy/Store/FileHandle.cfh b/core/Lucy/Store/FileHandle.cfh
new file mode 100644
index 0000000..3122397
--- /dev/null
+++ b/core/Lucy/Store/FileHandle.cfh
@@ -0,0 +1,153 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Abstract class for reading and writing to files.
+ *
+ * FileHandle abstracts low-level unbuffered read/write and memory mapping
+ * operations. Implementations might reference data in a local file system, a
+ * RAM "file", a custom-constructed compound file, etc.
+ *
+ * Desired behaviors for a FileHandle are expressed via flags supplied to the
+ * constructor:
+ *
+ * * FH_READ_ONLY - Read only.
+ * * FH_WRITE_ONLY - Write only.
+ * * FH_CREATE - Create the file if it does not yet exist.
+ * * FH_EXCLUSIVE - The attempt to open the file should fail if the file
+ * already exists.
+ */
+
+abstract class Lucy::Store::FileHandle cnick FH
+ inherits Lucy::Object::Obj {
+
+ CharBuf *path;
+ uint32_t flags;
+
+ /* Integer which is incremented each time a FileHandle is created and
+ * decremented when a FileHandle is destroyed. Since so many classes use
+ * FileHandle objects, they're the canary in the coal mine for detecting
+ * object-destruction memory leaks.
+ */
+ inert int32_t object_count;
+
+ /** Abstract constructor.
+ *
+ * @param path The path to the file.
+ * @param flags A 32-bit integer with bits set to indicate desired
+ * behaviors.
+ */
+ inert nullable FileHandle*
+ do_open(FileHandle *self, const CharBuf *path = NULL, uint32_t flags);
+
+ /** Ensure that the FileWindow's buffer provides access to file data for
+ * <code>len</code> bytes starting at <code>offset</code>.
+ *
+ * @param window A FileWindow.
+ * @param offset File position to begin at.
+ * @param len Number of bytes to expose via the window.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Window(FileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+ /** Clean up the FileWindow, doing whatever is necessary to free its
+ * buffer and reset its internal variables.
+ *
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Release_Window(FileHandle *self, FileWindow *window);
+
+ /** Copy file content into the supplied buffer.
+ *
+ * @param dest Supplied memory.
+ * @param offset File position to begin at.
+ * @param len Number of bytes to copy.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Read(FileHandle *self, char *dest, int64_t offset, size_t len);
+
+ /** Write supplied content.
+ *
+ * @param data Content to write.
+ * @param len Number of bytes to write.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Write(FileHandle *self, const void *data, size_t len);
+
+ /** Return the current length of the file in bytes, or set Err_error and
+ * return -1 on failure.
+ */
+ abstract int64_t
+ Length(FileHandle *self);
+
+ /** Advisory call alerting the FileHandle that it should prepare to occupy
+ * <code>len</code> bytes. The default implementation is a no-op.
+ *
+ * @return true on success, false on failure (sets Err_error).
+ */
+ bool_t
+ Grow(FileHandle *self, int64_t len);
+
+ /** Close the FileHandle, possibly releasing resources. Implementations
+ * should be be able to handle multiple invocations, returning success
+ * unless something unexpected happens.
+ *
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Close(FileHandle *self);
+
+ /** Set the object's <code>path</code> attribute.
+ */
+ void
+ Set_Path(FileHandle *self, const CharBuf *path);
+
+ /** Return the object's <code>path</code> attribute.
+ */
+ nullable CharBuf*
+ Get_Path(FileHandle *self);
+
+ /** Invokes Close(), but ignores whether it succeeds or fails.
+ */
+ public void
+ Destroy(FileHandle *self);
+}
+
+__C__
+
+#define LUCY_FH_READ_ONLY 0x1
+#define LUCY_FH_WRITE_ONLY 0x2
+#define LUCY_FH_CREATE 0x4
+#define LUCY_FH_EXCLUSIVE 0x8
+
+// Default size for the memory buffer used by both InStream and OutStream.
+#define LUCY_IO_STREAM_BUF_SIZE 1024
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define IO_STREAM_BUF_SIZE LUCY_IO_STREAM_BUF_SIZE
+ #define FH_READ_ONLY LUCY_FH_READ_ONLY
+ #define FH_WRITE_ONLY LUCY_FH_WRITE_ONLY
+ #define FH_CREATE LUCY_FH_CREATE
+ #define FH_EXCLUSIVE LUCY_FH_EXCLUSIVE
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Store/FileWindow.c b/core/Lucy/Store/FileWindow.c
new file mode 100644
index 0000000..51b747e
--- /dev/null
+++ b/core/Lucy/Store/FileWindow.c
@@ -0,0 +1,52 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/FileWindow.h"
+
+FileWindow*
+FileWindow_new() {
+ FileWindow *self = (FileWindow*)VTable_Make_Obj(FILEWINDOW);
+ return FileWindow_init(self);
+}
+
+FileWindow*
+FileWindow_init(FileWindow *self) {
+ return self;
+}
+
+void
+FileWindow_set_offset(FileWindow *self, int64_t offset) {
+ if (self->buf != NULL) {
+ if (offset != self->offset) {
+ THROW(ERR, "Can't set offset to %i64 instead of %i64 unless buf "
+ "is NULL", offset, self->offset);
+ }
+ }
+ self->offset = offset;
+}
+
+void
+FileWindow_set_window(FileWindow *self, char *buf, int64_t offset,
+ int64_t len) {
+ self->buf = buf;
+ self->offset = offset;
+ self->len = len;
+}
+
+
diff --git a/core/Lucy/Store/FileWindow.cfh b/core/Lucy/Store/FileWindow.cfh
new file mode 100644
index 0000000..a7d9107
--- /dev/null
+++ b/core/Lucy/Store/FileWindow.cfh
@@ -0,0 +1,40 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** A portion of a file, viewed as an array of bytes.
+ */
+class Lucy::Store::FileWindow inherits Lucy::Object::Obj {
+
+ char *buf;
+ int64_t offset;
+ int64_t len;
+
+ inert FileWindow*
+ init(FileWindow *self);
+
+ inert incremented FileWindow*
+ new();
+
+ void
+ Set_Offset(FileWindow *self, int64_t offset);
+
+ void
+ Set_Window(FileWindow *self, char *buf, int64_t offset, int64_t len);
+}
+
+
diff --git a/core/Lucy/Store/Folder.c b/core/Lucy/Store/Folder.c
new file mode 100644
index 0000000..71538b0
--- /dev/null
+++ b/core/Lucy/Store/Folder.c
@@ -0,0 +1,478 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FOLDER
+#include "Lucy/Util/ToolSet.h"
+#include <ctype.h>
+#include <limits.h>
+
+#ifndef SIZE_MAX
+ #define SIZE_MAX ((size_t)-1)
+#endif
+
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+Folder*
+Folder_init(Folder *self, const CharBuf *path) {
+ // Init.
+ self->entries = Hash_new(16);
+
+ // Copy.
+ if (path == NULL) {
+ self->path = CB_new_from_trusted_utf8("", 0);
+ }
+ else {
+ // Copy path, strip trailing slash or equivalent.
+ self->path = CB_Clone(path);
+ if (CB_Ends_With_Str(self->path, DIR_SEP, strlen(DIR_SEP))) {
+ CB_Chop(self->path, 1);
+ }
+ }
+
+ ABSTRACT_CLASS_CHECK(self, FOLDER);
+ return self;
+}
+
+void
+Folder_destroy(Folder *self) {
+ DECREF(self->path);
+ DECREF(self->entries);
+ SUPER_DESTROY(self, FOLDER);
+}
+
+InStream*
+Folder_open_in(Folder *self, const CharBuf *path) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ InStream *instream = NULL;
+
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ instream = Folder_Local_Open_In(enclosing_folder, (CharBuf*)name);
+ if (!instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+
+ return instream;
+}
+
+/* This method exists as a hook for CompoundFileReader to override; it is
+ * necessary because calling CFReader_Local_Open_FileHandle() won't find
+ * virtual files. No other class should need to override it. */
+InStream*
+Folder_local_open_in(Folder *self, const CharBuf *name) {
+ FileHandle *fh = Folder_Local_Open_FileHandle(self, name, FH_READ_ONLY);
+ InStream *instream = NULL;
+ if (fh) {
+ instream = InStream_open((Obj*)fh);
+ DECREF(fh);
+ if (!instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return instream;
+}
+
+OutStream*
+Folder_open_out(Folder *self, const CharBuf *path) {
+ const uint32_t flags = FH_WRITE_ONLY | FH_CREATE | FH_EXCLUSIVE;
+ FileHandle *fh = Folder_Open_FileHandle(self, path, flags);
+ OutStream *outstream = NULL;
+ if (fh) {
+ outstream = OutStream_open((Obj*)fh);
+ DECREF(fh);
+ if (!outstream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return outstream;
+}
+
+FileHandle*
+Folder_open_filehandle(Folder *self, const CharBuf *path, uint32_t flags) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ FileHandle *fh = NULL;
+
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ fh = Folder_Local_Open_FileHandle(enclosing_folder,
+ (CharBuf*)name, flags);
+ if (!fh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+
+ return fh;
+}
+
+bool_t
+Folder_delete(Folder *self, const CharBuf *path) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ bool_t result = Folder_Local_Delete(enclosing_folder, (CharBuf*)name);
+ return result;
+ }
+ else {
+ return false;
+ }
+}
+
+bool_t
+Folder_delete_tree(Folder *self, const CharBuf *path) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+
+ // Don't allow Folder to delete itself.
+ if (!path || !CB_Get_Size(path)) { return false; }
+
+ if (enclosing_folder) {
+ ZombieCharBuf *local = IxFileNames_local_part(path, ZCB_BLANK());
+ if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)local)) {
+ Folder *inner_folder
+ = Folder_Local_Find_Folder(enclosing_folder, (CharBuf*)local);
+ DirHandle *dh = Folder_Local_Open_Dir(inner_folder);
+ if (dh) {
+ VArray *files = VA_new(20);
+ VArray *dirs = VA_new(20);
+ CharBuf *entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ VA_Push(files, (Obj*)CB_Clone(entry));
+ if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
+ VA_Push(dirs, (Obj*)CB_Clone(entry));
+ }
+ }
+ for (uint32_t i = 0, max = VA_Get_Size(dirs); i < max; i++) {
+ CharBuf *name = (CharBuf*)VA_Fetch(files, i);
+ bool_t success = Folder_Delete_Tree(inner_folder, name);
+ if (!success && Folder_Local_Exists(inner_folder, name)) {
+ break;
+ }
+ }
+ for (uint32_t i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *name = (CharBuf*)VA_Fetch(files, i);
+ bool_t success = Folder_Local_Delete(inner_folder, name);
+ if (!success && Folder_Local_Exists(inner_folder, name)) {
+ break;
+ }
+ }
+ DECREF(dirs);
+ DECREF(files);
+ DECREF(dh);
+ }
+ }
+ return Folder_Local_Delete(enclosing_folder, (CharBuf*)local);
+ }
+ else {
+ // Return failure if the entry wasn't there in the first place.
+ return false;
+ }
+}
+
+static bool_t
+S_is_updir(CharBuf *path) {
+ if (CB_Equals_Str(path, ".", 1) || CB_Equals_Str(path, "..", 2)) {
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+static void
+S_add_to_file_list(Folder *self, VArray *list, CharBuf *dir, CharBuf *prefix) {
+ size_t orig_prefix_size = CB_Get_Size(prefix);
+ DirHandle *dh = Folder_Open_Dir(self, dir);
+ CharBuf *entry;
+
+ if (!dh) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+
+ entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) { // Updates entry
+ if (!S_is_updir(entry)) {
+ CharBuf *relpath = CB_newf("%o%o", prefix, entry);
+ if (VA_Get_Size(list) == VA_Get_Capacity(list)) {
+ VA_Grow(list, VA_Get_Size(list) * 2);
+ }
+ VA_Push(list, (Obj*)relpath);
+
+ if (DH_Entry_Is_Dir(dh) && !DH_Entry_Is_Symlink(dh)) {
+ CharBuf *subdir = CB_Get_Size(dir)
+ ? CB_newf("%o/%o", dir, entry)
+ : CB_Clone(entry);
+ CB_catf(prefix, "%o/", entry);
+ S_add_to_file_list(self, list, subdir, prefix); // recurse
+ CB_Set_Size(prefix, orig_prefix_size);
+ DECREF(subdir);
+ }
+ }
+ }
+
+ if (!DH_Close(dh)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ DECREF(dh);
+}
+
+DirHandle*
+Folder_open_dir(Folder *self, const CharBuf *path) {
+ DirHandle *dh = NULL;
+ Folder *folder = Folder_Find_Folder(self, path ? path : (CharBuf*)&EMPTY);
+ if (!folder) {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+ else {
+ dh = Folder_Local_Open_Dir(folder);
+ if (!dh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+ return dh;
+}
+
+bool_t
+Folder_mkdir(Folder *self, const CharBuf *path) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ bool_t result = false;
+
+ if (!CB_Get_Size(path)) {
+ Err_set_error(Err_new(CB_newf("Invalid path: '%o'", path)));
+ }
+ else if (!enclosing_folder) {
+ Err_set_error(Err_new(CB_newf("Can't recursively create dir %o",
+ path)));
+ }
+ else {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ result = Folder_Local_MkDir(enclosing_folder, (CharBuf*)name);
+ if (!result) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+
+ return result;
+}
+
+bool_t
+Folder_exists(Folder *self, const CharBuf *path) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ bool_t retval = false;
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ if (Folder_Local_Exists(enclosing_folder, (CharBuf*)name)) {
+ retval = true;
+ }
+ }
+ return retval;
+}
+
+bool_t
+Folder_is_directory(Folder *self, const CharBuf *path) {
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ bool_t retval = false;
+ if (enclosing_folder) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ if (Folder_Local_Is_Directory(enclosing_folder, (CharBuf*)name)) {
+ retval = true;
+ }
+ }
+ return retval;
+}
+
+VArray*
+Folder_list(Folder *self, const CharBuf *path) {
+ Folder *local_folder = Folder_Find_Folder(self, path);
+ VArray *list = NULL;
+ DirHandle *dh = Folder_Local_Open_Dir(local_folder);
+ if (dh) {
+ CharBuf *entry = DH_Get_Entry(dh);
+ list = VA_new(32);
+ while (DH_Next(dh)) { VA_Push(list, (Obj*)CB_Clone(entry)); }
+ DECREF(dh);
+ }
+ else {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return list;
+}
+
+VArray*
+Folder_list_r(Folder *self, const CharBuf *path) {
+ Folder *local_folder = Folder_Find_Folder(self, path);
+ VArray *list = VA_new(0);
+ if (local_folder) {
+ CharBuf *dir = CB_new(20);
+ CharBuf *prefix = CB_new(20);
+ if (path && CB_Get_Size(path)) {
+ CB_setf(prefix, "%o/", path);
+ }
+ S_add_to_file_list(local_folder, list, dir, prefix);
+ DECREF(prefix);
+ DECREF(dir);
+ }
+ return list;
+}
+
+ByteBuf*
+Folder_slurp_file(Folder *self, const CharBuf *path) {
+ InStream *instream = Folder_Open_In(self, path);
+ ByteBuf *retval = NULL;
+
+ if (!instream) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ else {
+ uint64_t length = InStream_Length(instream);
+
+ if (length >= SIZE_MAX) {
+ InStream_Close(instream);
+ DECREF(instream);
+ THROW(ERR, "File %o is too big to slurp (%u64 bytes)", path,
+ length);
+ }
+ else {
+ size_t size = (size_t)length;
+ char *ptr = (char*)MALLOCATE((size_t)size + 1);
+ InStream_Read_Bytes(instream, ptr, size);
+ ptr[size] = '\0';
+ retval = BB_new_steal_bytes(ptr, size, size + 1);
+ InStream_Close(instream);
+ DECREF(instream);
+ }
+ }
+
+ return retval;
+}
+
+CharBuf*
+Folder_get_path(Folder *self) {
+ return self->path;
+}
+
+void
+Folder_set_path(Folder *self, const CharBuf *path) {
+ DECREF(self->path);
+ self->path = CB_Clone(path);
+}
+
+void
+Folder_consolidate(Folder *self, const CharBuf *path) {
+ Folder *folder = Folder_Find_Folder(self, path);
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ if (!folder) {
+ THROW(ERR, "Can't consolidate %o", path);
+ }
+ else if (Folder_Is_A(folder, COMPOUNDFILEREADER)) {
+ THROW(ERR, "Can't consolidate %o twice", path);
+ }
+ else {
+ CompoundFileWriter *cf_writer = CFWriter_new(folder);
+ CFWriter_Consolidate(cf_writer);
+ DECREF(cf_writer);
+ if (CB_Get_Size(path)) {
+ ZombieCharBuf *name = IxFileNames_local_part(path, ZCB_BLANK());
+ CompoundFileReader *cf_reader = CFReader_open(folder);
+ if (!cf_reader) { RETHROW(INCREF(Err_get_error())); }
+ Hash_Store(enclosing_folder->entries, (Obj*)name,
+ (Obj*)cf_reader);
+ }
+ }
+}
+
+static Folder*
+S_enclosing_folder(Folder *self, ZombieCharBuf *path) {
+ size_t path_component_len = 0;
+ uint32_t code_point;
+
+ // Strip trailing slash.
+ if (ZCB_Code_Point_From(path, 0) == '/') { ZCB_Chop(path, 1); }
+
+ // Find first component of the file path.
+ ZombieCharBuf *scratch = ZCB_WRAP((CharBuf*)path);
+ ZombieCharBuf *path_component = ZCB_WRAP((CharBuf*)path);
+ while (0 != (code_point = ZCB_Nip_One(scratch))) {
+ if (code_point == '/') {
+ ZCB_Truncate(path_component, path_component_len);
+ ZCB_Nip(path, path_component_len + 1);
+ break;
+ }
+ path_component_len++;
+ }
+
+ // If we've eaten up the entire filepath, self is enclosing folder.
+ if (ZCB_Get_Size(scratch) == 0) { return self; }
+
+ {
+ Folder *local_folder
+ = Folder_Local_Find_Folder(self, (CharBuf*)path_component);
+ if (!local_folder) {
+ /* This element of the filepath doesn't exist, or it's not a
+ * directory. However, there are filepath characters left over,
+ * implying that this component ought to be a directory -- so the
+ * original file path is invalid. */
+ return NULL;
+ }
+
+ // This file path component is a folder. Recurse into it.
+ return S_enclosing_folder(local_folder, path);
+ }
+}
+
+Folder*
+Folder_enclosing_folder(Folder *self, const CharBuf *path) {
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ return S_enclosing_folder(self, scratch);
+}
+
+Folder*
+Folder_find_folder(Folder *self, const CharBuf *path) {
+ if (!path || !CB_Get_Size(path)) {
+ return self;
+ }
+ else {
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ Folder *enclosing_folder = S_enclosing_folder(self, scratch);
+ if (!enclosing_folder) {
+ return NULL;
+ }
+ else {
+ return Folder_Local_Find_Folder(enclosing_folder,
+ (CharBuf*)scratch);
+ }
+ }
+}
+
+
diff --git a/core/Lucy/Store/Folder.cfh b/core/Lucy/Store/Folder.cfh
new file mode 100644
index 0000000..a11bf06
--- /dev/null
+++ b/core/Lucy/Store/Folder.cfh
@@ -0,0 +1,273 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Abstract class representing a directory.
+ *
+ * A "file" within a Folder might be a real file on disk -- or it might be a
+ * RAM buffer. Similarly, Delete() might delete a file from the file system, or
+ * a key-value pair from a hash, or something else.
+ *
+ * The archetypal implementation of Folder,
+ * L<FSFolder|Lucy::Store::FSFolder>, represents a directory on
+ * the file system holding a collection of files.
+ */
+abstract class Lucy::Store::Folder inherits Lucy::Object::Obj {
+
+ CharBuf *path;
+ Hash *entries;
+
+ public inert nullable Folder*
+ init(Folder *self, const CharBuf *path);
+
+ public void
+ Destroy(Folder *self);
+
+ /** Getter for <code>path</code> member var.
+ */
+ public CharBuf*
+ Get_Path(Folder *self);
+
+ /** Setter for <code>path</code> member var.
+ */
+ void
+ Set_Path(Folder *self, const CharBuf *path);
+
+ /** Open an OutStream, or set Err_error and return NULL on failure.
+ *
+ * @param path A relative filepath.
+ * @return an OutStream.
+ */
+ public incremented nullable OutStream*
+ Open_Out(Folder *self, const CharBuf *path);
+
+ /** Open an InStream, or set Err_error and return NULL on failure.
+ *
+ * @param path A relative filepath.
+ * @return an InStream.
+ */
+ public incremented nullable InStream*
+ Open_In(Folder *self, const CharBuf *path);
+
+ /** Open a FileHandle, or set Err_error and return NULL on failure.
+ *
+ * @param path A relative filepath.
+ * @param flags FileHandle flags.
+ * @return a FileHandle.
+ */
+ public incremented nullable FileHandle*
+ Open_FileHandle(Folder *self, const CharBuf *path, uint32_t flags);
+
+ /** Open a DirHandle or set Err_error and return NULL on failure.
+ *
+ * @param path Path to a subdirectory, relative to the Folder's path. If
+ * empty or NULL, returns a DirHandle for this Folder.
+ * @return a DirHandle.
+ */
+ public incremented nullable DirHandle*
+ Open_Dir(Folder *self, const CharBuf *path = NULL);
+
+ /** Create a subdirectory.
+ *
+ * @param path A relative filepath.
+ * @return true on success, false on failure (sets Err_error).
+ */
+ public bool_t
+ MkDir(Folder *self, const CharBuf *path);
+
+ /** List all local entries within a directory. Set Err_error and return
+ * NULL if something goes wrong.
+ *
+ * @param path A relative filepath optionally specifying a subdirectory.
+ * @return an unsorted array of filenames.
+ */
+ incremented VArray*
+ List(Folder *self, const CharBuf *path = NULL);
+
+ /** Recursively list all files and directories in the Folder.
+ *
+ * @param path A relative filepath optionally specifying a subdirectory.
+ * @return an unsorted array of relative filepaths.
+ */
+ incremented VArray*
+ List_R(Folder *self, const CharBuf *path = NULL);
+
+ /** Indicate whether an entity exists at <code>path</code>.
+ *
+ * @param path A relative filepath.
+ * @return true if <code>path</code> exists.
+ */
+ public bool_t
+ Exists(Folder *self, const CharBuf *path);
+
+ /** Indicate whether a directory exists at <code>path</code>.
+ *
+ * @param path A relative filepath.
+ * @return true if <code>path</code> is a directory.
+ */
+ bool_t
+ Is_Directory(Folder *self, const CharBuf *path);
+
+ /** Delete an entry from the folder.
+ *
+ * @param path A relative filepath.
+ * @return true if the deletion was successful.
+ */
+ public bool_t
+ Delete(Folder *self, const CharBuf *path);
+
+ /** Delete recursively, starting at <code>path</code>
+ *
+ * @param path A relative filepath specifying a file or subdirectory.
+ * @return true if the whole tree is deleted successfully, false if any
+ * part remains.
+ */
+ public bool_t
+ Delete_Tree(Folder *self, const CharBuf *path);
+
+ /** Rename a file or directory, or set Err_error and return false on
+ * failure. If an entry exists at <code>to</code>, the results are
+ * undefined.
+ *
+ * @param from The filepath prior to renaming.
+ * @param to The filepath after renaming.
+ * @return true on success, false on failure.
+ */
+ public abstract bool_t
+ Rename(Folder *self, const CharBuf *from, const CharBuf *to);
+
+ /** Create a hard link at path <code>to</code> pointing at the existing
+ * file <code>from</code>, or set Err_error and return false on failure.
+ *
+ * @return true on success, false on failure.
+ */
+ public abstract bool_t
+ Hard_Link(Folder *self, const CharBuf *from, const CharBuf *to);
+
+ /** Read a file and return its contents.
+ *
+ * @param path A relative filepath.
+ * @param return the file's contents.
+ */
+ public incremented ByteBuf*
+ Slurp_File(Folder *self, const CharBuf *path);
+
+ /** Collapse the contents of the directory into a compound file.
+ */
+ void
+ Consolidate(Folder *self, const CharBuf *path);
+
+ /** Given a filepath, return the Folder representing everything except
+ * the last component. E.g. the 'foo/bar' Folder for '/foo/bar/baz.txt',
+ * the 'foo' Folder for 'foo/bar', etc.
+ *
+ * If <code>path</code> is invalid, because an intermediate directory
+ * either doesn't exist or isn't a directory, return NULL.
+ */
+ nullable Folder*
+ Enclosing_Folder(Folder *self, const CharBuf *path);
+
+ /** Return the Folder at the subdirectory specified by <code>path</code>.
+ * If <code>path</code> is NULL or an empty string, return this Folder.
+ * If the entity at <code>path</code> either doesn't exist or isn't a
+ * subdirectory, return NULL.
+ *
+ * @param path A relative filepath specifying a subdirectory.
+ * @return A Folder.
+ */
+ nullable Folder*
+ Find_Folder(Folder *self, const CharBuf *path);
+
+ /** Perform implementation-specific initialization. For example: FSFolder
+ * creates its own directory.
+ */
+ public abstract void
+ Initialize(Folder *self);
+
+ /** Verify that operations may be performed on this Folder.
+ *
+ * @return true on success.
+ */
+ public abstract bool_t
+ Check(Folder *self);
+
+ /** Close the folder and release implementation-specific resources.
+ */
+ public abstract void
+ Close(Folder *self);
+
+ /** Open a FileHandle for a local file, or set Err_error and return NULL
+ * on failure.
+ */
+ abstract incremented nullable FileHandle*
+ Local_Open_FileHandle(Folder *self, const CharBuf *name, uint32_t flags);
+
+ /** Open an InStream for a local file, or set Err_error and return NULL on
+ * failure.
+ */
+ incremented nullable InStream*
+ Local_Open_In(Folder *self, const CharBuf *name);
+
+ /** Open a DirHandle to iterate over the local entries in this Folder, or
+ * set Err_error and return NULL on failure.
+ */
+ abstract incremented nullable DirHandle*
+ Local_Open_Dir(Folder *self);
+
+ /** Create a local subdirectory.
+ *
+ * @param name The name of the subdirectory.
+ * @return true on success, false on failure (sets Err_error)
+ */
+ abstract bool_t
+ Local_MkDir(Folder *self, const CharBuf *name);
+
+ /** Indicate whether a local entry exists for the supplied
+ * <code>name</code>.
+ *
+ * @param name The name of the local entry.
+ */
+ abstract bool_t
+ Local_Exists(Folder *self, const CharBuf *name);
+
+ /** Indicate whether a local subdirectory exists with the supplied
+ * <code>name</code>.
+ *
+ * @param name The name of the local subdirectory.
+ */
+ abstract bool_t
+ Local_Is_Directory(Folder *self, const CharBuf *name);
+
+ /** Return the Folder object representing the specified directory, if such
+ * a directory exists.
+ *
+ * @param name The name of a local directory.
+ * @return a Folder.
+ */
+ abstract nullable Folder*
+ Local_Find_Folder(Folder *self, const CharBuf *name);
+
+ /** Delete a local entry.
+ *
+ * @param name The name of the entry to be deleted.
+ * @return true if the deletion was successful.
+ */
+ abstract bool_t
+ Local_Delete(Folder *self, const CharBuf *name);
+}
+
+
diff --git a/core/Lucy/Store/InStream.c b/core/Lucy/Store/InStream.c
new file mode 100644
index 0000000..1f481e1
--- /dev/null
+++ b/core/Lucy/Store/InStream.c
@@ -0,0 +1,469 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INSTREAM
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/FSFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/RAMFileHandle.h"
+
+// Inlined version of InStream_Tell.
+static INLINE int64_t
+SI_tell(InStream *self);
+
+// Inlined version of InStream_Read_Bytes.
+static INLINE void
+SI_read_bytes(InStream *self, char* buf, size_t len);
+
+// Inlined version of InStream_Read_U8.
+static INLINE uint8_t
+SI_read_u8(InStream *self);
+
+// Ensure that the buffer contains exactly the specified amount of data.
+static void
+S_fill(InStream *self, int64_t amount);
+
+// Refill the buffer, with either IO_STREAM_BUF_SIZE bytes or all remaining
+// file content -- whichever is smaller. Throw an error if we're at EOF and
+// can't load at least one byte.
+static int64_t
+S_refill(InStream *self);
+
+InStream*
+InStream_open(Obj *file) {
+ InStream *self = (InStream*)VTable_Make_Obj(INSTREAM);
+ return InStream_do_open(self, file);
+}
+
+InStream*
+InStream_do_open(InStream *self, Obj *file) {
+ // Init.
+ self->buf = NULL;
+ self->limit = NULL;
+ self->offset = 0;
+ self->window = FileWindow_new();
+
+ // Obtain a FileHandle.
+ if (Obj_Is_A(file, FILEHANDLE)) {
+ self->file_handle = (FileHandle*)INCREF(file);
+ }
+ else if (Obj_Is_A(file, RAMFILE)) {
+ self->file_handle
+ = (FileHandle*)RAMFH_open(NULL, FH_READ_ONLY, (RAMFile*)file);
+ }
+ else if (Obj_Is_A(file, CHARBUF)) {
+ self->file_handle
+ = (FileHandle*)FSFH_open((CharBuf*)file, FH_READ_ONLY);
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'",
+ Obj_Get_Class_Name(file))));
+ DECREF(self);
+ return NULL;
+ }
+ if (!self->file_handle) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ // Get length and filename from the FileHandle.
+ self->filename = CB_Clone(FH_Get_Path(self->file_handle));
+ self->len = FH_Length(self->file_handle);
+ if (self->len == -1) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ return self;
+}
+
+void
+InStream_close(InStream *self) {
+ if (self->file_handle) {
+ FH_Release_Window(self->file_handle, self->window);
+ // Note that we don't close the FileHandle, because it's probably
+ // shared.
+ DECREF(self->file_handle);
+ self->file_handle = NULL;
+ }
+}
+
+void
+InStream_destroy(InStream *self) {
+ if (self->file_handle) {
+ InStream_Close(self);
+ }
+ DECREF(self->filename);
+ DECREF(self->window);
+ SUPER_DESTROY(self, INSTREAM);
+}
+
+InStream*
+InStream_reopen(InStream *self, const CharBuf *filename, int64_t offset,
+ int64_t len) {
+ if (!self->file_handle) {
+ THROW(ERR, "Can't Reopen() closed InStream %o", self->filename);
+ }
+ if (offset + len > FH_Length(self->file_handle)) {
+ THROW(ERR, "Offset + length too large (%i64 + %i64 > %i64)",
+ offset, len, FH_Length(self->file_handle));
+ }
+
+ InStream *twin = (InStream*)VTable_Make_Obj(self->vtable);
+ InStream_do_open(twin, (Obj*)self->file_handle);
+ if (filename != NULL) { CB_Mimic(twin->filename, (Obj*)filename); }
+ twin->offset = offset;
+ twin->len = len;
+ InStream_Seek(twin, 0);
+
+ return twin;
+}
+
+InStream*
+InStream_clone(InStream *self) {
+ InStream *twin = (InStream*)VTable_Make_Obj(self->vtable);
+ InStream_do_open(twin, (Obj*)self->file_handle);
+ InStream_Seek(twin, SI_tell(self));
+ return twin;
+}
+
+CharBuf*
+InStream_get_filename(InStream *self) {
+ return self->filename;
+}
+
+static int64_t
+S_refill(InStream *self) {
+ // Determine the amount to request.
+ const int64_t sub_file_pos = SI_tell(self);
+ const int64_t remaining = self->len - sub_file_pos;
+ const int64_t amount = remaining < IO_STREAM_BUF_SIZE
+ ? remaining
+ : IO_STREAM_BUF_SIZE;
+ if (!remaining) {
+ THROW(ERR, "Read past EOF of '%o' (offset: %i64 len: %i64)",
+ self->filename, self->offset, self->len);
+ }
+
+ // Make the request.
+ S_fill(self, amount);
+
+ return amount;
+}
+
+void
+InStream_refill(InStream *self) {
+ S_refill(self);
+}
+
+static void
+S_fill(InStream *self, int64_t amount) {
+ FileWindow *const window = self->window;
+ const int64_t virtual_file_pos = SI_tell(self);
+ const int64_t real_file_pos = virtual_file_pos + self->offset;
+ const int64_t remaining = self->len - virtual_file_pos;
+
+ // Throw an error if the requested amount would take us beyond EOF.
+ if (amount > remaining) {
+ THROW(ERR, "Read past EOF of %o (pos: %u64 len: %u64 request: %u64)",
+ self->filename, virtual_file_pos, self->len, amount);
+ }
+
+ // Make the request.
+ if (FH_Window(self->file_handle, window, real_file_pos, amount)) {
+ char *const window_limit = window->buf + window->len;
+ self->buf = window->buf
+ - window->offset // theoretical start of real file
+ + self->offset // top of virtual file
+ + virtual_file_pos; // position within virtual file
+ self->limit = window_limit - self->buf > remaining
+ ? self->buf + remaining
+ : window_limit;
+ }
+ else {
+ Err *error = Err_get_error();
+ CB_catf(Err_Get_Mess(error), " (%o)", self->filename);
+ RETHROW(INCREF(error));
+ }
+}
+
+void
+InStream_fill(InStream *self, int64_t amount) {
+ S_fill(self, amount);
+}
+
+void
+InStream_seek(InStream *self, int64_t target) {
+ FileWindow *const window = self->window;
+ int64_t virtual_window_top = window->offset - self->offset;
+ int64_t virtual_window_end = virtual_window_top + window->len;
+
+ if (target < 0) {
+ THROW(ERR, "Can't Seek '%o' to negative target %i64", self->filename,
+ target);
+ }
+ // Seek within window if possible.
+ else if (target >= virtual_window_top
+ && target <= virtual_window_end
+ ) {
+ self->buf = window->buf - window->offset + self->offset + target;
+ }
+ else if (target > self->len) {
+ THROW(ERR, "Can't Seek '%o' past EOF (%i64 > %i64)", self->filename,
+ target, self->len);
+ }
+ else {
+ // Target is outside window. Set all buffer and limit variables to
+ // NULL to trigger refill on the next read. Store the file position
+ // in the FileWindow's offset.
+ FH_Release_Window(self->file_handle, window);
+ self->buf = NULL;
+ self->limit = NULL;
+ FileWindow_Set_Offset(window, self->offset + target);
+ }
+}
+
+static INLINE int64_t
+SI_tell(InStream *self) {
+ FileWindow *const window = self->window;
+ int64_t pos_in_buf = PTR_TO_I64(self->buf) - PTR_TO_I64(window->buf);
+ return pos_in_buf + window->offset - self->offset;
+}
+
+int64_t
+InStream_tell(InStream *self) {
+ return SI_tell(self);
+}
+
+int64_t
+InStream_length(InStream *self) {
+ return self->len;
+}
+
+char*
+InStream_buf(InStream *self, size_t request) {
+ const int64_t bytes_in_buf = PTR_TO_I64(self->limit) - PTR_TO_I64(self->buf);
+
+ /* It's common for client code to overestimate how much is needed, because
+ * the request has to figure in worst-case for compressed data. However,
+ * if we can still serve them everything they request (e.g. they ask for 5
+ * bytes, they really need 1 byte, and there's 1k in the buffer), we can
+ * skip the following refill block. */
+ if ((int64_t)request > bytes_in_buf) {
+ const int64_t remaining_in_file = self->len - SI_tell(self);
+ int64_t amount = request;
+
+ // Try to bump up small requests.
+ if (amount < IO_STREAM_BUF_SIZE) { amount = IO_STREAM_BUF_SIZE; }
+
+ // Don't read past EOF.
+ if (remaining_in_file < amount) { amount = remaining_in_file; }
+
+ // Only fill if the recalculated, possibly smaller request exceeds the
+ // amount available in the buffer.
+ if (amount > bytes_in_buf) {
+ S_fill(self, amount);
+ }
+ }
+
+ return self->buf;
+}
+
+void
+InStream_advance_buf(InStream *self, char *buf) {
+ if (buf > self->limit) {
+ int64_t overrun = PTR_TO_I64(buf) - PTR_TO_I64(self->limit);
+ THROW(ERR, "Supplied value is %i64 bytes beyond end of buffer",
+ overrun);
+ }
+ else if (buf < self->buf) {
+ int64_t underrun = PTR_TO_I64(self->buf) - PTR_TO_I64(buf);
+ THROW(ERR, "Can't Advance_Buf backwards: (underrun: %i64))", underrun);
+ }
+ else {
+ self->buf = buf;
+ }
+}
+
+void
+InStream_read_bytes(InStream *self, char* buf, size_t len) {
+ SI_read_bytes(self, buf, len);
+}
+
+static INLINE void
+SI_read_bytes(InStream *self, char* buf, size_t len) {
+ const int64_t available = PTR_TO_I64(self->limit) - PTR_TO_I64(self->buf);
+ if (available >= (int64_t)len) {
+ // Request is entirely within buffer, so copy.
+ memcpy(buf, self->buf, len);
+ self->buf += len;
+ }
+ else {
+ // Pass along whatever we've got in the buffer.
+ if (available > 0) {
+ memcpy(buf, self->buf, (size_t)available);
+ buf += available;
+ len -= (size_t)available;
+ self->buf += available;
+ }
+
+ if (len < IO_STREAM_BUF_SIZE) {
+ // Ensure that we have enough mapped, then copy the rest.
+ int64_t got = S_refill(self);
+ if (got < (int64_t)len) {
+ int64_t orig_pos = SI_tell(self) - available;
+ int64_t orig_len = len + available;
+ THROW(ERR, "Read past EOF of %o (pos: %i64 len: %i64 "
+ "request: %i64)", self->filename, orig_pos,
+ self->len, orig_len);
+ }
+ memcpy(buf, self->buf, len);
+ self->buf += len;
+ }
+ else {
+ // Too big to handle via the buffer, so resort to a brute-force
+ // read.
+ const int64_t sub_file_pos = SI_tell(self);
+ const int64_t real_file_pos = sub_file_pos + self->offset;
+ bool_t success
+ = FH_Read(self->file_handle, buf, real_file_pos, len);
+ if (!success) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ InStream_seek(self, sub_file_pos + len);
+ }
+ }
+}
+
+int8_t
+InStream_read_i8(InStream *self) {
+ return (int8_t)SI_read_u8(self);
+}
+
+static INLINE uint8_t
+SI_read_u8(InStream *self) {
+ if (self->buf >= self->limit) { S_refill(self); }
+ return (uint8_t)(*self->buf++);
+}
+
+uint8_t
+InStream_read_u8(InStream *self) {
+ return SI_read_u8(self);
+}
+
+static INLINE uint32_t
+SI_read_u32(InStream *self) {
+ uint32_t retval;
+ SI_read_bytes(self, (char*)&retval, 4);
+#ifdef LITTLE_END
+ retval = NumUtil_decode_bigend_u32((char*)&retval);
+#endif
+ return retval;
+}
+
+uint32_t
+InStream_read_u32(InStream *self) {
+ return SI_read_u32(self);
+}
+
+int32_t
+InStream_read_i32(InStream *self) {
+ return (int32_t)SI_read_u32(self);
+}
+
+static INLINE uint64_t
+SI_read_u64(InStream *self) {
+ uint64_t retval;
+ SI_read_bytes(self, (char*)&retval, 8);
+#ifdef LITTLE_END
+ retval = NumUtil_decode_bigend_u64((char*)&retval);
+#endif
+ return retval;
+}
+
+uint64_t
+InStream_read_u64(InStream *self) {
+ return SI_read_u64(self);
+}
+
+int64_t
+InStream_read_i64(InStream *self) {
+ return (int64_t)SI_read_u64(self);
+}
+
+float
+InStream_read_f32(InStream *self) {
+ union { float f; uint32_t u32; } duo;
+ SI_read_bytes(self, (char*)&duo, sizeof(float));
+#ifdef LITTLE_END
+ duo.u32 = NumUtil_decode_bigend_u32(&duo.u32);
+#endif
+ return duo.f;
+}
+
+double
+InStream_read_f64(InStream *self) {
+ union { double d; uint64_t u64; } duo;
+ SI_read_bytes(self, (char*)&duo, sizeof(double));
+#ifdef LITTLE_END
+ duo.u64 = NumUtil_decode_bigend_u64(&duo.u64);
+#endif
+ return duo.d;
+}
+
+uint32_t
+InStream_read_c32(InStream *self) {
+ uint32_t retval = 0;
+ while (1) {
+ const uint8_t ubyte = SI_read_u8(self);
+ retval = (retval << 7) | (ubyte & 0x7f);
+ if ((ubyte & 0x80) == 0) {
+ break;
+ }
+ }
+ return retval;
+}
+
+uint64_t
+InStream_read_c64(InStream *self) {
+ uint64_t retval = 0;
+ while (1) {
+ const uint8_t ubyte = SI_read_u8(self);
+ retval = (retval << 7) | (ubyte & 0x7f);
+ if ((ubyte & 0x80) == 0) {
+ break;
+ }
+ }
+ return retval;
+}
+
+int
+InStream_read_raw_c64(InStream *self, char *buf) {
+ uint8_t *dest = (uint8_t*)buf;
+ do {
+ *dest = SI_read_u8(self);
+ } while ((*dest++ & 0x80) != 0);
+ return dest - (uint8_t*)buf;
+}
+
+
diff --git a/core/Lucy/Store/InStream.cfh b/core/Lucy/Store/InStream.cfh
new file mode 100644
index 0000000..1b32422
--- /dev/null
+++ b/core/Lucy/Store/InStream.cfh
@@ -0,0 +1,197 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Read index files.
+ *
+ * InStream objects are the primary interface for reading from index files.
+ * They are high-level (relatively speaking), media-agnostic wrappers
+ * around low-level, media-specific FileHandle objects.
+ *
+ * InStreams provide a number of routines for safely decoding common constructs
+ * such as big-endian or compressed integers; for the most part, these
+ * routines throw exceptions rather than require manual checking of return
+ * values for error conditions.
+ *
+ * Multiple InStream objects often share the same underlying FileHandle; this
+ * practice is safe because InStreams do not modify or rely upon the file
+ * position or other state within the FileHandle.
+ */
+class Lucy::Store::InStream inherits Lucy::Object::Obj {
+
+ int64_t offset;
+ int64_t len;
+ char *buf;
+ char *limit;
+ CharBuf *filename;
+ FileHandle *file_handle;
+ FileWindow *window;
+
+ inert incremented nullable InStream*
+ open(Obj *file);
+
+ /** Return a new InStream, or set Err_error and return NULL on failure.
+ *
+ * @param file A FileHandle, a file path, or a RAMFile.
+ */
+ inert nullable InStream*
+ do_open(InStream *self, Obj *file);
+
+ /** Clone the instream, but specify a new offset, length, and possibly
+ * filename. Initial file position will be set to the top of the file
+ * (taking <code>offset</code> into account).
+ *
+ * @param filename An alias filename. If NULL, the filename of the
+ * underlying FileHandle will be used.
+ * @param offset Top of the file as seen by the new InStream, in bytes
+ * from the top of the file as seen by the underlying FileHandle.
+ * @param len Length of the file as seen by the new InStream.
+ */
+ incremented InStream*
+ Reopen(InStream *self, const CharBuf *filename = NULL, int64_t offset,
+ int64_t len);
+
+ /** Clone the InStream. Clones share the same underlying FileHandle and
+ * start at the current file position, but are able to seek and read
+ * independently.
+ */
+ public incremented InStream*
+ Clone(InStream *self);
+
+ /** Decrement the number of streams using the underlying FileHandle. When
+ * the number drops to zero, possibly release system resources.
+ */
+ void
+ Close(InStream *self);
+
+ public void
+ Destroy(InStream *self);
+
+ /** Seek to <code>target</code>.
+ */
+ final void
+ Seek(InStream *self, int64_t target);
+
+ /** Return the current file position.
+ */
+ final int64_t
+ Tell(InStream *self);
+
+ /** Return the length of the "file" in bytes.
+ */
+ final int64_t
+ Length(InStream *self);
+
+ /** Fill the InStream's buffer, letting the FileHandle decide how many bytes
+ * of data to fill it with.
+ */
+ void
+ Refill(InStream *self);
+
+ /** Pour an exact number of bytes into the InStream's buffer.
+ */
+ void
+ Fill(InStream *self, int64_t amount);
+
+ /** Get the InStream's buffer. Check to see whether <code>request</code>
+ * bytes are already in the buffer. If not, fill the buffer with either
+ * <code>request</code> bytes or the number of bytes remaining before EOF,
+ * whichever is smaller.
+ *
+ * @param request Advisory byte size request.
+ * @return Pointer to the InStream's internal buffer.
+ */
+ final char*
+ Buf(InStream *self, size_t request);
+
+ /** Set the buf to a new value, checking for overrun. The idiom is for
+ * the caller to call Buf(), use no more bytes than requested, then use
+ * Advance_Buf() to update the InStream object.
+ */
+ final void
+ Advance_Buf(InStream *self, char *buf);
+
+ /** Read <code>len</code> bytes from the InStream into <code>buf</code>.
+ */
+ final void
+ Read_Bytes(InStream *self, char *buf, size_t len);
+
+ /** Read a signed 8-bit integer.
+ */
+ final int8_t
+ Read_I8(InStream *self);
+
+ /** Read an unsigned 8-bit integer.
+ */
+ final uint8_t
+ Read_U8(InStream *self);
+
+ /** Read a signed 32-bit integer.
+ */
+ final int32_t
+ Read_I32(InStream *self);
+
+ /** Read an unsigned 32-bit integer.
+ */
+ final uint32_t
+ Read_U32(InStream *self);
+
+ /** Read a signed 64-bit integer.
+ */
+ final int64_t
+ Read_I64(InStream *self);
+
+ /** Read an unsigned 64-bit integer.
+ */
+ final uint64_t
+ Read_U64(InStream *self);
+
+ /** Read an IEEE 764 32-bit floating point number.
+ */
+ final float
+ Read_F32(InStream *self);
+
+ /** Read an IEEE 764 64-bit floating point number.
+ */
+ final double
+ Read_F64(InStream *self);
+
+ /** Read in a compressed 32-bit unsigned integer.
+ */
+ uint32_t
+ Read_C32(InStream *self);
+
+ /** Read a 64-bit integer, using the same encoding as a C32 but occupying
+ * as many as 10 bytes.
+ */
+ final uint64_t
+ Read_C64(InStream *self);
+
+ /** Read the bytes for a C32/C64 into <code>buf</code>. Return the number
+ * of bytes read. The caller must ensure that sufficient space exists in
+ * <code>buf</code> (worst case is 10 bytes).
+ */
+ final int
+ Read_Raw_C64(InStream *self, char *buf);
+
+ /** Accessor for filename member.
+ */
+ CharBuf*
+ Get_Filename(InStream *self);
+}
+
+
diff --git a/core/Lucy/Store/Lock.c b/core/Lucy/Store/Lock.c
new file mode 100644
index 0000000..27c0fe4
--- /dev/null
+++ b/core/Lucy/Store/Lock.c
@@ -0,0 +1,303 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LOCK
+#define C_LUCY_LOCKFILELOCK
+#include "Lucy/Util/ToolSet.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/ProcessID.h"
+#include "Lucy/Util/Sleep.h"
+
+Lock*
+Lock_init(Lock *self, Folder *folder, const CharBuf *name,
+ const CharBuf *host, int32_t timeout, int32_t interval) {
+ // Validate.
+ if (interval <= 0) {
+ DECREF(self);
+ THROW(ERR, "Invalid value for 'interval': %i32", interval);
+ }
+ {
+ ZombieCharBuf *scratch = ZCB_WRAP(name);
+ uint32_t code_point;
+ while (0 != (code_point = ZCB_Nip_One(scratch))) {
+ if (isalnum(code_point)
+ || code_point == '.'
+ || code_point == '-'
+ || code_point == '_'
+ ) {
+ continue;
+ }
+ DECREF(self);
+ THROW(ERR, "Lock name contains disallowed characters: '%o'",
+ name);
+ }
+ }
+
+ // Assign.
+ self->folder = (Folder*)INCREF(folder);
+ self->timeout = timeout;
+ self->name = CB_Clone(name);
+ self->host = CB_Clone(host);
+ self->interval = interval;
+
+ // Derive.
+ self->lock_path = CB_newf("locks/%o.lock", name);
+
+ return self;
+}
+
+void
+Lock_destroy(Lock *self) {
+ DECREF(self->folder);
+ DECREF(self->host);
+ DECREF(self->name);
+ DECREF(self->lock_path);
+ SUPER_DESTROY(self, LOCK);
+}
+
+CharBuf*
+Lock_get_name(Lock *self) {
+ return self->name;
+}
+
+CharBuf*
+Lock_get_lock_path(Lock *self) {
+ return self->lock_path;
+}
+
+CharBuf*
+Lock_get_host(Lock *self) {
+ return self->host;
+}
+
+bool_t
+Lock_obtain(Lock *self) {
+ int32_t time_left = self->interval == 0 ? 0 : self->timeout;
+ bool_t locked = Lock_Request(self);
+
+ while (!locked) {
+ time_left -= self->interval;
+ if (time_left <= 0) { break; }
+ Sleep_millisleep(self->interval);
+ locked = Lock_Request(self);
+ }
+
+ if (!locked) { ERR_ADD_FRAME(Err_get_error()); }
+ return locked;
+}
+
+/***************************************************************************/
+
+LockFileLock*
+LFLock_new(Folder *folder, const CharBuf *name, const CharBuf *host,
+ int32_t timeout, int32_t interval) {
+ LockFileLock *self = (LockFileLock*)VTable_Make_Obj(LOCKFILELOCK);
+ return LFLock_init(self, folder, name, host, timeout, interval);
+}
+
+LockFileLock*
+LFLock_init(LockFileLock *self, Folder *folder, const CharBuf *name,
+ const CharBuf *host, int32_t timeout, int32_t interval) {
+ int pid = PID_getpid();
+ Lock_init((Lock*)self, folder, name, host, timeout, interval);
+ self->link_path = CB_newf("%o.%o.%i64", self->lock_path, host, pid);
+ return self;
+}
+
+bool_t
+LFLock_shared(LockFileLock *self) {
+ UNUSED_VAR(self); return false;
+}
+
+bool_t
+LFLock_request(LockFileLock *self) {
+ Hash *file_data;
+ bool_t wrote_json;
+ bool_t success = false;
+ bool_t deletion_failed = false;
+
+ if (Folder_Exists(self->folder, self->lock_path)) {
+ Err_set_error((Err*)LockErr_new(CB_newf("Can't obtain lock: '%o' exists",
+ self->lock_path)));
+ return false;
+ }
+
+ // Create the "locks" subdirectory if necessary.
+ CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
+ if (!Folder_Exists(self->folder, lock_dir_name)) {
+ if (!Folder_MkDir(self->folder, lock_dir_name)) {
+ Err *mkdir_err = (Err*)CERTIFY(Err_get_error(), ERR);
+ LockErr *err = LockErr_new(CB_newf("Can't create 'locks' directory: %o",
+ Err_Get_Mess(mkdir_err)));
+ // Maybe our attempt failed because another process succeeded.
+ if (Folder_Find_Folder(self->folder, lock_dir_name)) {
+ DECREF(err);
+ }
+ else {
+ // Nope, everything failed, so bail out.
+ Err_set_error((Err*)err);
+ return false;
+ }
+ }
+ }
+
+ // Prepare to write pid, lock name, and host to the lock file as JSON.
+ file_data = Hash_new(3);
+ Hash_Store_Str(file_data, "pid", 3,
+ (Obj*)CB_newf("%i32", (int32_t)PID_getpid()));
+ Hash_Store_Str(file_data, "host", 4, INCREF(self->host));
+ Hash_Store_Str(file_data, "name", 4, INCREF(self->name));
+
+ // Write to a temporary file, then use the creation of a hard link to
+ // ensure atomic but non-destructive creation of the lockfile with its
+ // complete contents.
+ wrote_json = Json_spew_json((Obj*)file_data, self->folder, self->link_path);
+ if (wrote_json) {
+ success = Folder_Hard_Link(self->folder, self->link_path,
+ self->lock_path);
+ if (!success) {
+ Err *hard_link_err = (Err*)CERTIFY(Err_get_error(), ERR);
+ Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
+ self->lock_path,
+ Err_Get_Mess(hard_link_err))));
+ }
+ deletion_failed = !Folder_Delete(self->folder, self->link_path);
+ }
+ else {
+ Err *spew_json_err = (Err*)CERTIFY(Err_get_error(), ERR);
+ Err_set_error((Err*)LockErr_new(CB_newf("Failed to obtain lock at '%o': %o",
+ self->lock_path,
+ Err_Get_Mess(spew_json_err))));
+ }
+ DECREF(file_data);
+
+ // Verify that our temporary file got zapped.
+ if (wrote_json && deletion_failed) {
+ CharBuf *mess = MAKE_MESS("Failed to delete '%o'", self->link_path);
+ Err_throw_mess(ERR, mess);
+ }
+
+ return success;
+}
+
+void
+LFLock_release(LockFileLock *self) {
+ if (Folder_Exists(self->folder, self->lock_path)) {
+ LFLock_Maybe_Delete_File(self, self->lock_path, true, false);
+ }
+}
+
+bool_t
+LFLock_is_locked(LockFileLock *self) {
+ return Folder_Exists(self->folder, self->lock_path);
+}
+
+void
+LFLock_clear_stale(LockFileLock *self) {
+ LFLock_Maybe_Delete_File(self, self->lock_path, false, true);
+}
+
+bool_t
+LFLock_maybe_delete_file(LockFileLock *self, const CharBuf *path,
+ bool_t delete_mine, bool_t delete_other) {
+ Folder *folder = self->folder;
+ bool_t success = false;
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+
+ // Only delete locks that start with our lock name.
+ CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
+ if (!ZCB_Starts_With(scratch, lock_dir_name)) {
+ return false;
+ }
+ ZCB_Nip(scratch, CB_Get_Size(lock_dir_name) + 1);
+ if (!ZCB_Starts_With(scratch, self->name)) {
+ return false;
+ }
+
+ // Attempt to delete dead lock file.
+ if (Folder_Exists(folder, path)) {
+ Hash *hash = (Hash*)Json_slurp_json(folder, path);
+ if (hash != NULL && Obj_Is_A((Obj*)hash, HASH)) {
+ CharBuf *pid_buf = (CharBuf*)Hash_Fetch_Str(hash, "pid", 3);
+ CharBuf *host = (CharBuf*)Hash_Fetch_Str(hash, "host", 4);
+ CharBuf *name
+ = (CharBuf*)Hash_Fetch_Str(hash, "name", 4);
+
+ // Match hostname and lock name.
+ if (host != NULL
+ && CB_Equals(host, (Obj*)self->host)
+ && name != NULL
+ && CB_Equals(name, (Obj*)self->name)
+ && pid_buf != NULL
+ ) {
+ // Verify that pid is either mine or dead.
+ int pid = (int)CB_To_I64(pid_buf);
+ if ((delete_mine && pid == PID_getpid()) // This process.
+ || (delete_other && !PID_active(pid)) // Dead pid.
+ ) {
+ if (Folder_Delete(folder, path)) {
+ success = true;
+ }
+ else {
+ CharBuf *mess
+ = MAKE_MESS("Can't delete '%o'", path);
+ DECREF(hash);
+ Err_throw_mess(ERR, mess);
+ }
+ }
+ }
+ }
+ DECREF(hash);
+ }
+
+ return success;
+}
+
+void
+LFLock_destroy(LockFileLock *self) {
+ DECREF(self->link_path);
+ SUPER_DESTROY(self, LOCKFILELOCK);
+}
+
+/***************************************************************************/
+
+LockErr*
+LockErr_new(CharBuf *message) {
+ LockErr *self = (LockErr*)VTable_Make_Obj(LOCKERR);
+ return LockErr_init(self, message);
+}
+
+LockErr*
+LockErr_init(LockErr *self, CharBuf *message) {
+ Err_init((Err*)self, message);
+ return self;
+}
+
+LockErr*
+LockErr_make(LockErr *self) {
+ UNUSED_VAR(self);
+ return LockErr_new(CB_new(0));
+}
+
+
diff --git a/core/Lucy/Store/Lock.cfh b/core/Lucy/Store/Lock.cfh
new file mode 100644
index 0000000..17bdfda
--- /dev/null
+++ b/core/Lucy/Store/Lock.cfh
@@ -0,0 +1,175 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Abstract class representing an interprocess mutex lock.
+ *
+ * The Lock class produces an interprocess mutex lock. The default subclass
+ * uses dot-lock files, but alternative implementations are possible.
+ *
+ * Each lock must have a name which is unique per resource to be locked. Each
+ * lock also has a "host" id which should be unique per machine; it is used to
+ * help clear away stale locks.
+ */
+
+abstract class Lucy::Store::Lock inherits Lucy::Object::Obj {
+
+ Folder *folder;
+ CharBuf *name;
+ CharBuf *lock_path;
+ CharBuf *host;
+ int32_t timeout;
+ int32_t interval;
+
+ /** Abstract constructor.
+ *
+ * @param folder A Folder.
+ * @param name String identifying the resource to be locked, which must
+ * consist solely of characters matching [-_.A-Za-z0-9].
+ * @param host A unique per-machine identifier.
+ * @param timeout Time in milliseconds to keep retrying before abandoning
+ * the attempt to Obtain() a lock.
+ * @param interval Time in milliseconds between retries.
+ */
+ public inert Lock*
+ init(Lock *self, Folder *folder, const CharBuf *name,
+ const CharBuf *host, int32_t timeout = 0, int32_t interval = 100);
+
+ /** Returns true if the Lock is shared, false if the Lock is exclusive.
+ */
+ public abstract bool_t
+ Shared(Lock *self);
+
+ /** Call Request() once per <code>interval</code> until Request() returns
+ * success or the <code>timeout</code> has been reached.
+ *
+ * @return true on success, false on failure (sets Err_error).
+ */
+ public bool_t
+ Obtain(Lock *self);
+
+ /** Make one attempt to acquire the lock.
+ *
+ * The semantics of Request() differ depending on whether Shared() returns
+ * true. If the Lock is Shared(), then Request() should not fail if
+ * another lock is held against the resource identified by
+ * <code>name</code> (though it might fail for other reasons). If it is
+ * not Shared() -- i.e. it's an exclusive (write) lock -- then other locks
+ * should cause Request() to fail.
+ *
+ * @return true on success, false on failure (sets Err_error).
+ */
+ public abstract bool_t
+ Request(Lock *self);
+
+ /** Release the lock.
+ */
+ public abstract void
+ Release(Lock *self);
+
+ /** Indicate whether the resource identified by this lock's name is
+ * currently locked.
+ *
+ * @return true if the resource is locked, false otherwise.
+ */
+ public abstract bool_t
+ Is_Locked(Lock *self);
+
+ /** Release all locks that meet the following three conditions: the lock
+ * name matches, the host id matches, and the process id that the lock
+ * was created under no longer identifies an active process.
+ */
+ public abstract void
+ Clear_Stale(Lock *self);
+
+ CharBuf*
+ Get_Name(Lock *self);
+
+ CharBuf*
+ Get_Host(Lock *self);
+
+ CharBuf*
+ Get_Lock_Path(Lock *self);
+
+ public void
+ Destroy(Lock *self);
+}
+
+class Lucy::Store::LockFileLock cnick LFLock
+ inherits Lucy::Store::Lock {
+
+ CharBuf *link_path;
+
+ inert incremented LockFileLock*
+ new(Folder *folder, const CharBuf *name, const CharBuf *host,
+ int32_t timeout = 0, int32_t interval = 100);
+
+ public inert LockFileLock*
+ init(LockFileLock *self, Folder *folder, const CharBuf *name,
+ const CharBuf *host, int32_t timeout = 0, int32_t interval = 100);
+
+ public bool_t
+ Shared(LockFileLock *self);
+
+ public bool_t
+ Request(LockFileLock *self);
+
+ public void
+ Release(LockFileLock *self);
+
+ public bool_t
+ Is_Locked(LockFileLock *self);
+
+ public void
+ Clear_Stale(LockFileLock *self);
+
+ /** Delete a given lock file which meets these conditions...
+ *
+ * - lock name matches.
+ * - host id matches.
+ *
+ * If delete_mine is false, don't delete a lock file which matches this
+ * process's pid. If delete_other is false, don't delete lock files which
+ * don't match this process's pid.
+ */
+ bool_t
+ Maybe_Delete_File(LockFileLock *self, const CharBuf *filepath,
+ bool_t delete_mine, bool_t delete_other);
+
+ public void
+ Destroy(LockFileLock *self);
+}
+
+/** Lock exception.
+ *
+ * LockErr is a subclass of L<Err|Lucy::Object::Err> which indicates
+ * that a file locking problem occurred.
+ */
+class Lucy::Store::LockErr inherits Lucy::Object::Err {
+
+ public inert incremented LockErr*
+ new(CharBuf *message);
+
+ public inert LockErr*
+ init(LockErr *self, CharBuf *message);
+
+ public incremented LockErr*
+ Make(LockErr *self);
+}
+
+
diff --git a/core/Lucy/Store/LockFactory.c b/core/Lucy/Store/LockFactory.c
new file mode 100644
index 0000000..2f1f362
--- /dev/null
+++ b/core/Lucy/Store/LockFactory.c
@@ -0,0 +1,63 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_LOCKFACTORY
+#include "Lucy/Util/ToolSet.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include "Lucy/Store/LockFactory.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/Lock.h"
+#include "Lucy/Store/SharedLock.h"
+
+LockFactory*
+LockFact_new(Folder *folder, const CharBuf *host) {
+ LockFactory *self = (LockFactory*)VTable_Make_Obj(LOCKFACTORY);
+ return LockFact_init(self, folder, host);
+}
+
+LockFactory*
+LockFact_init(LockFactory *self, Folder *folder, const CharBuf *host) {
+ self->folder = (Folder*)INCREF(folder);
+ self->host = CB_Clone(host);
+ return self;
+}
+
+void
+LockFact_destroy(LockFactory *self) {
+ DECREF(self->folder);
+ DECREF(self->host);
+ SUPER_DESTROY(self, LOCKFACTORY);
+}
+
+Lock*
+LockFact_make_lock(LockFactory *self, const CharBuf *name, int32_t timeout,
+ int32_t interval) {
+ return (Lock*)LFLock_new(self->folder, name, self->host, timeout,
+ interval);
+}
+
+Lock*
+LockFact_make_shared_lock(LockFactory *self, const CharBuf *name,
+ int32_t timeout, int32_t interval) {
+ return (Lock*)ShLock_new(self->folder, name, self->host, timeout,
+ interval);
+}
+
+
diff --git a/core/Lucy/Store/LockFactory.cfh b/core/Lucy/Store/LockFactory.cfh
new file mode 100644
index 0000000..a3ded37
--- /dev/null
+++ b/core/Lucy/Store/LockFactory.cfh
@@ -0,0 +1,74 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Create Locks.
+ *
+ * LockFactory is used to spin off interprocess mutex locks used by various
+ * index reading and writing components. The default implementation uses
+ * lockfiles, but LockFactory subclasses which are implemented using
+ * alternatives such as flock() are possible.
+ */
+
+class Lucy::Store::LockFactory cnick LockFact
+ inherits Lucy::Object::Obj {
+
+ Folder *folder;
+ CharBuf *host;
+
+ inert incremented LockFactory*
+ new(Folder *folder, const CharBuf *host);
+
+ /**
+ * @param folder A L<Lucy::Store::Folder>.
+ * @param host An identifier which should be unique per-machine.
+ */
+ public inert LockFactory*
+ init(LockFactory *self, Folder *folder, const CharBuf *host);
+
+ /** Return a Lock object, which, once Obtain() returns successfully,
+ * maintains an exclusive lock on a resource.
+ *
+ * @param name A file-system-friendly id which identifies the
+ * resource to be locked.
+ * @param timeout Time in milliseconds to keep retrying before abandoning
+ * the attempt to Obtain() a lock.
+ * @param interval Time in milliseconds between retries.
+ */
+ public incremented Lock*
+ Make_Lock(LockFactory *self, const CharBuf *name, int32_t timeout = 0,
+ int32_t interval = 100);
+
+ /** Return a Lock object for which Shared() returns true, and which
+ * maintains a non-exclusive lock on a resource once Obtain() returns
+ * success.
+ *
+ * @param name A file-system-friendly id which identifies the
+ * resource to be locked.
+ * @param timeout Time in milliseconds to keep retrying before abandoning
+ * the attempt to Obtain() a lock.
+ * @param interval Time in milliseconds between retries.
+ */
+ public incremented Lock*
+ Make_Shared_Lock(LockFactory *self, const CharBuf *name,
+ int32_t timeout = 0, int32_t interval = 100);
+
+ public void
+ Destroy(LockFactory *self);
+}
+
+
diff --git a/core/Lucy/Store/OutStream.c b/core/Lucy/Store/OutStream.c
new file mode 100644
index 0000000..1a38066
--- /dev/null
+++ b/core/Lucy/Store/OutStream.c
@@ -0,0 +1,330 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_OUTSTREAM
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/FSFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/RAMFileHandle.h"
+
+// Inlined version of OutStream_Write_Bytes.
+static INLINE void
+SI_write_bytes(OutStream *self, const void *bytes, size_t len);
+
+// Inlined version of OutStream_Write_C32.
+static INLINE void
+SI_write_c32(OutStream *self, uint32_t value);
+
+// Flush content in the buffer to the FileHandle.
+static void
+S_flush(OutStream *self);
+
+OutStream*
+OutStream_open(Obj *file) {
+ OutStream *self = (OutStream*)VTable_Make_Obj(OUTSTREAM);
+ return OutStream_do_open(self, file);
+}
+
+OutStream*
+OutStream_do_open(OutStream *self, Obj *file) {
+ // Init.
+ self->buf = (char*)MALLOCATE(IO_STREAM_BUF_SIZE);
+ self->buf_start = 0;
+ self->buf_pos = 0;
+
+ // Obtain a FileHandle.
+ if (Obj_Is_A(file, FILEHANDLE)) {
+ self->file_handle = (FileHandle*)INCREF(file);
+ }
+ else if (Obj_Is_A(file, RAMFILE)) {
+ self->file_handle
+ = (FileHandle*)RAMFH_open(NULL, FH_WRITE_ONLY, (RAMFile*)file);
+ }
+ else if (Obj_Is_A(file, CHARBUF)) {
+ self->file_handle = (FileHandle*)FSFH_open((CharBuf*)file,
+ FH_WRITE_ONLY | FH_CREATE | FH_EXCLUSIVE);
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Invalid type for param 'file': '%o'",
+ Obj_Get_Class_Name(file))));
+ DECREF(self);
+ return NULL;
+ }
+ if (!self->file_handle) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ // Derive filepath from FileHandle.
+ self->path = CB_Clone(FH_Get_Path(self->file_handle));
+
+ return self;
+}
+
+void
+OutStream_destroy(OutStream *self) {
+ if (self->file_handle != NULL) {
+ // Inlined flush, ignoring errors.
+ if (self->buf_pos) {
+ FH_Write(self->file_handle, self->buf, self->buf_pos);
+ }
+ DECREF(self->file_handle);
+ }
+ DECREF(self->path);
+ FREEMEM(self->buf);
+ SUPER_DESTROY(self, OUTSTREAM);
+}
+
+CharBuf*
+OutStream_get_path(OutStream *self) {
+ return self->path;
+}
+
+void
+OutStream_absorb(OutStream *self, InStream *instream) {
+ char buf[IO_STREAM_BUF_SIZE];
+ int64_t bytes_left = InStream_Length(instream);
+
+ // Read blocks of content into an intermediate buffer, than write them to
+ // the OutStream.
+ //
+ // TODO: optimize by utilizing OutStream's buffer directly, while still
+ // not flushing too frequently and keeping code complexity under control.
+ OutStream_Grow(self, OutStream_Tell(self) + bytes_left);
+ while (bytes_left) {
+ const size_t bytes_this_iter = bytes_left < IO_STREAM_BUF_SIZE
+ ? (size_t)bytes_left
+ : IO_STREAM_BUF_SIZE;
+ InStream_Read_Bytes(instream, buf, bytes_this_iter);
+ SI_write_bytes(self, buf, bytes_this_iter);
+ bytes_left -= bytes_this_iter;
+ }
+}
+
+void
+OutStream_grow(OutStream *self, int64_t length) {
+ if (!FH_Grow(self->file_handle, length)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+}
+
+int64_t
+OutStream_tell(OutStream *self) {
+ return self->buf_start + self->buf_pos;
+}
+
+int64_t
+OutStream_align(OutStream *self, int64_t modulus) {
+ int64_t len = OutStream_Tell(self);
+ int64_t filler_bytes = (modulus - (len % modulus)) % modulus;
+ while (filler_bytes--) { OutStream_Write_U8(self, 0); }
+ return OutStream_Tell(self);
+}
+
+void
+OutStream_flush(OutStream *self) {
+ S_flush(self);
+}
+
+static void
+S_flush(OutStream *self) {
+ if (self->file_handle == NULL) {
+ THROW(ERR, "Can't write to a closed OutStream for %o", self->path);
+ }
+ if (!FH_Write(self->file_handle, self->buf, self->buf_pos)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ self->buf_start += self->buf_pos;
+ self->buf_pos = 0;
+}
+
+int64_t
+OutStream_length(OutStream *self) {
+ return OutStream_tell(self);
+}
+
+void
+OutStream_write_bytes(OutStream *self, const void *bytes, size_t len) {
+ SI_write_bytes(self, bytes, len);
+}
+
+static INLINE void
+SI_write_bytes(OutStream *self, const void *bytes, size_t len) {
+ // If this data is larger than the buffer size, flush and write.
+ if (len >= IO_STREAM_BUF_SIZE) {
+ S_flush(self);
+ if (!FH_Write(self->file_handle, bytes, len)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ self->buf_start += len;
+ }
+ // If there's not enough room in the buffer, flush then add.
+ else if (self->buf_pos + len >= IO_STREAM_BUF_SIZE) {
+ S_flush(self);
+ memcpy((self->buf + self->buf_pos), bytes, len);
+ self->buf_pos += len;
+ }
+ // If there's room, just add these bytes to the buffer.
+ else {
+ memcpy((self->buf + self->buf_pos), bytes, len);
+ self->buf_pos += len;
+ }
+}
+
+static INLINE void
+SI_write_u8(OutStream *self, uint8_t value) {
+ if (self->buf_pos >= IO_STREAM_BUF_SIZE) {
+ S_flush(self);
+ }
+ self->buf[self->buf_pos++] = (char)value;
+}
+
+void
+OutStream_write_i8(OutStream *self, int8_t value) {
+ SI_write_u8(self, (uint8_t)value);
+}
+
+void
+OutStream_write_u8(OutStream *self, uint8_t value) {
+ SI_write_u8(self, value);
+}
+
+static INLINE void
+SI_write_u32(OutStream *self, uint32_t value) {
+#ifdef BIG_END
+ SI_write_bytes(self, &value, 4);
+#else
+ char buf[4];
+ char *buf_copy = buf;
+ NumUtil_encode_bigend_u32(value, &buf_copy);
+ SI_write_bytes(self, buf, 4);
+#endif
+}
+
+void
+OutStream_write_i32(OutStream *self, int32_t value) {
+ SI_write_u32(self, (uint32_t)value);
+}
+
+void
+OutStream_write_u32(OutStream *self, uint32_t value) {
+ SI_write_u32(self, value);
+}
+
+static INLINE void
+SI_write_u64(OutStream *self, uint64_t value) {
+#ifdef BIG_END
+ SI_write_bytes(self, &value, 8);
+#else
+ char buf[sizeof(uint64_t)];
+ char *buf_copy = buf;
+ NumUtil_encode_bigend_u64(value, &buf_copy);
+ SI_write_bytes(self, buf, sizeof(uint64_t));
+#endif
+}
+
+void
+OutStream_write_i64(OutStream *self, int64_t value) {
+ SI_write_u64(self, (uint64_t)value);
+}
+
+void
+OutStream_write_u64(OutStream *self, uint64_t value) {
+ SI_write_u64(self, value);
+}
+
+void
+OutStream_write_f32(OutStream *self, float value) {
+ char buf[sizeof(float)];
+ char *buf_copy = buf;
+ NumUtil_encode_bigend_f32(value, &buf_copy);
+ SI_write_bytes(self, buf_copy, sizeof(float));
+}
+
+void
+OutStream_write_f64(OutStream *self, double value) {
+ char buf[sizeof(double)];
+ char *buf_copy = buf;
+ NumUtil_encode_bigend_f64(value, &buf_copy);
+ SI_write_bytes(self, buf_copy, sizeof(double));
+}
+
+void
+OutStream_write_c32(OutStream *self, uint32_t value) {
+ SI_write_c32(self, value);
+}
+
+static INLINE void
+SI_write_c32(OutStream *self, uint32_t value) {
+ uint8_t buf[C32_MAX_BYTES];
+ uint8_t *ptr = buf + sizeof(buf) - 1;
+
+ // Write last byte first, which has no continue bit.
+ *ptr = value & 0x7f;
+ value >>= 7;
+
+ while (value) {
+ // Work backwards, writing bytes with continue bits set.
+ *--ptr = ((value & 0x7f) | 0x80);
+ value >>= 7;
+ }
+
+ SI_write_bytes(self, ptr, (buf + sizeof(buf)) - ptr);
+}
+
+void
+OutStream_write_c64(OutStream *self, uint64_t value) {
+ uint8_t buf[C64_MAX_BYTES];
+ uint8_t *ptr = buf + sizeof(buf) - 1;
+
+ // Write last byte first, which has no continue bit.
+ *ptr = value & 0x7f;
+ value >>= 7;
+
+ while (value) {
+ // Work backwards, writing bytes with continue bits set.
+ *--ptr = ((value & 0x7f) | 0x80);
+ value >>= 7;
+ }
+
+ SI_write_bytes(self, ptr, (buf + sizeof(buf)) - ptr);
+}
+
+void
+OutStream_write_string(OutStream *self, const char *string, size_t len) {
+ SI_write_c32(self, (uint32_t)len);
+ SI_write_bytes(self, string, len);
+}
+
+void
+OutStream_close(OutStream *self) {
+ if (self->file_handle) {
+ S_flush(self);
+ if (!FH_Close(self->file_handle)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ DECREF(self->file_handle);
+ self->file_handle = NULL;
+ }
+}
+
+
diff --git a/core/Lucy/Store/OutStream.cfh b/core/Lucy/Store/OutStream.cfh
new file mode 100644
index 0000000..d484d51
--- /dev/null
+++ b/core/Lucy/Store/OutStream.cfh
@@ -0,0 +1,157 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Write index files.
+ *
+ * OutStream objects are the primary interface for writing index files. They
+ * are media-agnostic wrappers around low-level, media-specific, unbuffered
+ * FileHandle objects, providing output buffering and routines for writing
+ * common constructs such as big-endian or compressed integers.
+ *
+ * OutStreams are write-once and cannot seek -- they must write all their data
+ * in order. Furthermore, each OutStream is associated with exactly one,
+ * unique FileHandle -- unlike InStreams, which can share a common FileHandle.
+ */
+class Lucy::Store::OutStream inherits Lucy::Object::Obj {
+
+ char *buf;
+ int64_t buf_start;
+ size_t buf_pos;
+ FileHandle *file_handle;
+ CharBuf *path;
+
+ inert incremented nullable OutStream*
+ open(Obj *file);
+
+ /** Return a new OutStream or set Err_error and return NULL on failure.
+ */
+ inert nullable OutStream*
+ do_open(OutStream *self, Obj *file);
+
+ /** Accessor for <code>path</code> member.
+ */
+ CharBuf*
+ Get_Path(OutStream *self);
+
+ /** Return the current file position.
+ */
+ final int64_t
+ Tell(OutStream *self);
+
+ /** Write 0 or more null bytes to the OutStream until its file position is
+ * a multiple of <code>modulus</code>.
+ *
+ * @return the new file position.
+ */
+ final int64_t
+ Align(OutStream *self, int64_t modulus);
+
+ /** Flush output buffer to target FileHandle.
+ */
+ final void
+ Flush(OutStream *self);
+
+ /** Return the current length of the file in bytes.
+ */
+ final int64_t
+ Length(OutStream *self);
+
+ /** Advisory call informing the OutStream that it should prepare to occupy
+ * <code>length</code> bytes.
+ */
+ void
+ Grow(OutStream *self, int64_t length);
+
+ /** Write <code>len</code> bytes from <code>buf</code> to the OutStream.
+ */
+ final void
+ Write_Bytes(OutStream *self, const void *buf, size_t len);
+
+ /** Write a signed 8-bit integer.
+ */
+ final void
+ Write_I8(OutStream *self, int8_t value);
+
+ /** Write an unsigned 8-bit integer.
+ */
+ final void
+ Write_U8(OutStream *self, uint8_t value);
+
+ /** Write a signed 32-bit integer.
+ */
+ final void
+ Write_I32(OutStream *self, int32_t value);
+
+ /** Write an unsigned 32-bit integer.
+ */
+ final void
+ Write_U32(OutStream *self, uint32_t value);
+
+ /** Write a signed 64-bit integer.
+ */
+ final void
+ Write_I64(OutStream *self, int64_t value);
+
+ /** Write an unsigned 64-bit integer.
+ */
+ final void
+ Write_U64(OutStream *self, uint64_t value);
+
+ /** Write a 32-bit integer using a compressed format.
+ */
+ final void
+ Write_C32(OutStream *self, uint32_t value);
+
+ /** Write a 64-bit integer using a compressed format.
+ */
+ final void
+ Write_C64(OutStream *self, uint64_t value);
+
+ /** Write an IEEE 764 32-bit floating point number in big-endian byte
+ * order.
+ */
+ final void
+ Write_F32(OutStream *self, float value);
+
+ /** Write an IEEE 764 64-bit double-precision floating point number in
+ * big-endian byte order.
+ */
+ final void
+ Write_F64(OutStream *self, double value);
+
+ /** Write a string as a C32 indicating length of content in bytes,
+ * followed by the content.
+ */
+ final void
+ Write_String(OutStream *self, const char *buf, size_t len);
+
+ /** Write the entire contents of an InStream to the OutStream.
+ */
+ void
+ Absorb(OutStream *self, InStream *instream);
+
+ /** Close down the stream.
+ */
+ void
+ Close(OutStream *self);
+
+ public void
+ Destroy(OutStream *self);
+}
+
+
diff --git a/core/Lucy/Store/RAMDirHandle.c b/core/Lucy/Store/RAMDirHandle.c
new file mode 100644
index 0000000..67c7674
--- /dev/null
+++ b/core/Lucy/Store/RAMDirHandle.c
@@ -0,0 +1,88 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFOLDER
+#define C_LUCY_RAMDIRHANDLE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/RAMDirHandle.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+RAMDirHandle*
+RAMDH_new(RAMFolder *folder) {
+ RAMDirHandle *self = (RAMDirHandle*)VTable_Make_Obj(RAMDIRHANDLE);
+ return RAMDH_init(self, folder);
+}
+
+RAMDirHandle*
+RAMDH_init(RAMDirHandle *self, RAMFolder *folder) {
+ DH_init((DirHandle*)self, RAMFolder_Get_Path(folder));
+ self->folder = (RAMFolder*)INCREF(folder);
+ self->elems = Hash_Keys(self->folder->entries);
+ self->tick = -1;
+ return self;
+}
+
+bool_t
+RAMDH_close(RAMDirHandle *self) {
+ if (self->elems) {
+ VA_Dec_RefCount(self->elems);
+ self->elems = NULL;
+ }
+ if (self->folder) {
+ RAMFolder_Dec_RefCount(self->folder);
+ self->folder = NULL;
+ }
+ return true;
+}
+
+bool_t
+RAMDH_next(RAMDirHandle *self) {
+ if (self->elems) {
+ self->tick++;
+ if (self->tick < (int32_t)VA_Get_Size(self->elems)) {
+ CharBuf *path = (CharBuf*)CERTIFY(
+ VA_Fetch(self->elems, self->tick), CHARBUF);
+ CB_Mimic(self->entry, (Obj*)path);
+ return true;
+ }
+ else {
+ self->tick--;
+ return false;
+ }
+ }
+ return false;
+}
+
+bool_t
+RAMDH_entry_is_dir(RAMDirHandle *self) {
+ if (self->elems) {
+ CharBuf *name = (CharBuf*)VA_Fetch(self->elems, self->tick);
+ if (name) {
+ return RAMFolder_Local_Is_Directory(self->folder, name);
+ }
+ }
+ return false;
+}
+
+bool_t
+RAMDH_entry_is_symlink(RAMDirHandle *self) {
+ UNUSED_VAR(self);
+ return false;
+}
+
+
diff --git a/core/Lucy/Store/RAMDirHandle.cfh b/core/Lucy/Store/RAMDirHandle.cfh
new file mode 100644
index 0000000..1dd87ae
--- /dev/null
+++ b/core/Lucy/Store/RAMDirHandle.cfh
@@ -0,0 +1,47 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** DirHandle for RAMFolder.
+ */
+class Lucy::Store::RAMDirHandle cnick RAMDH
+ inherits Lucy::Store::DirHandle {
+
+ RAMFolder *folder;
+ VArray *elems;
+ int32_t tick;
+
+ inert incremented RAMDirHandle*
+ new(RAMFolder *folder);
+
+ inert RAMDirHandle*
+ init(RAMDirHandle *self, RAMFolder *folder);
+
+ bool_t
+ Next(RAMDirHandle *self);
+
+ bool_t
+ Entry_Is_Dir(RAMDirHandle *self);
+
+ bool_t
+ Entry_Is_Symlink(RAMDirHandle *self);
+
+ bool_t
+ Close(RAMDirHandle *self);
+}
+
+
diff --git a/core/Lucy/Store/RAMFile.c b/core/Lucy/Store/RAMFile.c
new file mode 100644
index 0000000..b2f25d6
--- /dev/null
+++ b/core/Lucy/Store/RAMFile.c
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFILE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/RAMFile.h"
+
+RAMFile*
+RAMFile_new(ByteBuf *contents, bool_t read_only) {
+ RAMFile *self = (RAMFile*)VTable_Make_Obj(RAMFILE);
+ return RAMFile_init(self, contents, read_only);
+}
+
+RAMFile*
+RAMFile_init(RAMFile *self, ByteBuf *contents, bool_t read_only) {
+ self->contents = contents ? (ByteBuf*)INCREF(contents) : BB_new(0);
+ self->read_only = read_only;
+ return self;
+}
+
+void
+RAMFile_destroy(RAMFile *self) {
+ DECREF(self->contents);
+ SUPER_DESTROY(self, RAMFILE);
+}
+
+ByteBuf*
+RAMFile_get_contents(RAMFile *self) {
+ return self->contents;
+}
+
+bool_t
+RAMFile_read_only(RAMFile *self) {
+ return self->read_only;
+}
+
+void
+RAMFile_set_read_only(RAMFile *self, bool_t read_only) {
+ self->read_only = read_only;
+}
+
+
diff --git a/core/Lucy/Store/RAMFile.cfh b/core/Lucy/Store/RAMFile.cfh
new file mode 100644
index 0000000..d8c0a06
--- /dev/null
+++ b/core/Lucy/Store/RAMFile.cfh
@@ -0,0 +1,55 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Backing storage used by RAMFolder and RAMFileHandle.
+ */
+class Lucy::Store::RAMFile inherits Lucy::Object::Obj {
+
+ bool_t read_only;
+ ByteBuf *contents;
+
+ inert incremented RAMFile*
+ new(ByteBuf *contents = NULL, bool_t read_only = false);
+
+ /**
+ * @param contents Existing file contents, if any.
+ * @param read_only Indicate that the file contents may not be modified.
+ */
+ inert RAMFile*
+ init(RAMFile *self, ByteBuf *contents = NULL, bool_t read_only = false);
+
+ /** Accessor for the file's contents.
+ */
+ ByteBuf*
+ Get_Contents(RAMFile *self);
+
+ /** Accessor for <code>read_only</code> property.
+ */
+ bool_t
+ Read_Only(RAMFile *self);
+
+ /** Set the object's <code>read_only</code> property.
+ */
+ void
+ Set_Read_Only(RAMFile *self, bool_t read_only);
+
+ public void
+ Destroy(RAMFile *self);
+}
+
+
diff --git a/core/Lucy/Store/RAMFileHandle.c b/core/Lucy/Store/RAMFileHandle.c
new file mode 100644
index 0000000..7d48ba7
--- /dev/null
+++ b/core/Lucy/Store/RAMFileHandle.c
@@ -0,0 +1,179 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFILEHANDLE
+#define C_LUCY_RAMFILE
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/RAMFileHandle.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/FileWindow.h"
+
+RAMFileHandle*
+RAMFH_open(const CharBuf *path, uint32_t flags, RAMFile *file) {
+ RAMFileHandle *self = (RAMFileHandle*)VTable_Make_Obj(RAMFILEHANDLE);
+ return RAMFH_do_open(self, path, flags, file);
+}
+
+RAMFileHandle*
+RAMFH_do_open(RAMFileHandle *self, const CharBuf *path, uint32_t flags,
+ RAMFile *file) {
+ bool_t must_create
+ = (flags & (FH_CREATE | FH_EXCLUSIVE)) == (FH_CREATE | FH_EXCLUSIVE)
+ ? true : false;
+ bool_t can_create
+ = (flags & (FH_CREATE | FH_WRITE_ONLY)) == (FH_CREATE | FH_WRITE_ONLY)
+ ? true : false;
+
+ FH_do_open((FileHandle*)self, path, flags);
+
+ // Obtain a RAMFile.
+ if (file) {
+ if (must_create) {
+ Err_set_error(Err_new(CB_newf("File '%o' exists, but FH_EXCLUSIVE flag supplied", path)));
+ DECREF(self);
+ return NULL;
+ }
+ self->ram_file = (RAMFile*)INCREF(file);
+ }
+ else if (can_create) {
+ self->ram_file = RAMFile_new(NULL, false);
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("Must supply either RAMFile or FH_CREATE | FH_WRITE_ONLY")));
+ DECREF(self);
+ return NULL;
+ }
+
+ // Prevent writes to to the RAMFile if FH_READ_ONLY was specified.
+ if (flags & FH_READ_ONLY) {
+ RAMFile_Set_Read_Only(self->ram_file, true);
+ }
+
+ self->len = BB_Get_Size(self->ram_file->contents);
+
+ return self;
+}
+
+void
+RAMFH_destroy(RAMFileHandle *self) {
+ DECREF(self->ram_file);
+ SUPER_DESTROY(self, RAMFILEHANDLE);
+}
+
+bool_t
+RAMFH_window(RAMFileHandle *self, FileWindow *window, int64_t offset,
+ int64_t len) {
+ int64_t end = offset + len;
+ if (!(self->flags & FH_READ_ONLY)) {
+ Err_set_error(Err_new(CB_newf("Can't read from write-only handle")));
+ return false;
+ }
+ else if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from negative offset %i64",
+ offset)));
+ return false;
+ }
+ else if (end > self->len) {
+ Err_set_error(Err_new(CB_newf("Tried to read past EOF: offset %i64 + request %i64 > len %i64",
+ offset, len, self->len)));
+ return false;
+ }
+ else {
+ char *const buf = BB_Get_Buf(self->ram_file->contents) + offset;
+ FileWindow_Set_Window(window, buf, offset, len);
+ return true;
+ }
+}
+
+bool_t
+RAMFH_release_window(RAMFileHandle *self, FileWindow *window) {
+ UNUSED_VAR(self);
+ FileWindow_Set_Window(window, NULL, 0, 0);
+ return true;
+}
+
+bool_t
+RAMFH_read(RAMFileHandle *self, char *dest, int64_t offset, size_t len) {
+ int64_t end = offset + len;
+ if (!(self->flags & FH_READ_ONLY)) {
+ Err_set_error(Err_new(CB_newf("Can't read from write-only handle")));
+ return false;
+ }
+ else if (offset < 0) {
+ Err_set_error(Err_new(CB_newf("Can't read from a negative offset %i64",
+ offset)));
+ return false;
+ }
+ else if (end > self->len) {
+ Err_set_error(Err_new(CB_newf("Attempt to read %u64 bytes starting at %i64 goes past EOF %u64",
+ (uint64_t)len, offset, self->len)));
+ return false;
+ }
+ else {
+ char *const source = BB_Get_Buf(self->ram_file->contents) + offset;
+ memcpy(dest, source, len);
+ return true;
+ }
+}
+
+bool_t
+RAMFH_write(RAMFileHandle *self, const void *data, size_t len) {
+ if (self->ram_file->read_only) {
+ Err_set_error(Err_new(CB_newf("Attempt to write to read-only RAMFile")));
+ return false;
+ }
+ BB_Cat_Bytes(self->ram_file->contents, data, len);
+ self->len += len;
+ return true;
+}
+
+bool_t
+RAMFH_grow(RAMFileHandle *self, int64_t len) {
+ if (len > I32_MAX) {
+ Err_set_error(Err_new(CB_newf("Can't support RAM files of size %i64 (> %i32)",
+ len, (int32_t)I32_MAX)));
+ return false;
+ }
+ else if (self->ram_file->read_only) {
+ Err_set_error(Err_new(CB_newf("Can't grow read-only RAMFile '%o'",
+ self->path)));
+ return false;
+ }
+ else {
+ BB_Grow(self->ram_file->contents, (size_t)len);
+ return true;
+ }
+}
+
+RAMFile*
+RAMFH_get_file(RAMFileHandle *self) {
+ return self->ram_file;
+}
+
+int64_t
+RAMFH_length(RAMFileHandle *self) {
+ return self->len;
+}
+
+bool_t
+RAMFH_close(RAMFileHandle *self) {
+ UNUSED_VAR(self);
+ return true;
+}
+
+
diff --git a/core/Lucy/Store/RAMFileHandle.cfh b/core/Lucy/Store/RAMFileHandle.cfh
new file mode 100644
index 0000000..2c1ea03
--- /dev/null
+++ b/core/Lucy/Store/RAMFileHandle.cfh
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** In-memory FileHandle.
+ *
+ * RAM-based implementation of FileHandle, to be used in conjunction with
+ * RAMFolder.
+ */
+class Lucy::Store::RAMFileHandle cnick RAMFH
+ inherits Lucy::Store::FileHandle {
+
+ RAMFile *ram_file;
+ int64_t len;
+
+ inert incremented nullable RAMFileHandle*
+ open(const CharBuf *path = NULL, uint32_t flags, RAMFile *file = NULL);
+
+ /**
+ * Return a new RAMFileHandle, or set Err_error and return NULL on
+ * failure.
+ *
+ * @param path Filepath.
+ * @param flags FileHandle flags.
+ * @param file An existing RAMFile; if not supplied, the FH_CREATE flag
+ * must be passed or an error will occur.
+ */
+ inert nullable RAMFileHandle*
+ do_open(RAMFileHandle *self, const CharBuf *path = NULL, uint32_t flags,
+ RAMFile *file = NULL);
+
+ /** Access the backing RAMFile.
+ */
+ RAMFile*
+ Get_File(RAMFileHandle *self);
+
+ bool_t
+ Grow(RAMFileHandle *self, int64_t len);
+
+ public void
+ Destroy(RAMFileHandle *self);
+
+ bool_t
+ Window(RAMFileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+ bool_t
+ Release_Window(RAMFileHandle *self, FileWindow *window);
+
+ bool_t
+ Read(RAMFileHandle *self, char *dest, int64_t offset, size_t len);
+
+ bool_t
+ Write(RAMFileHandle *self, const void *data, size_t len);
+
+ int64_t
+ Length(RAMFileHandle *self);
+
+ bool_t
+ Close(RAMFileHandle *self);
+}
+
+
diff --git a/core/Lucy/Store/RAMFolder.c b/core/Lucy/Store/RAMFolder.c
new file mode 100644
index 0000000..909c2a3
--- /dev/null
+++ b/core/Lucy/Store/RAMFolder.c
@@ -0,0 +1,350 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMDirHandle.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/RAMFileHandle.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+// Return the concatenation of the Folder's path and the supplied path.
+static CharBuf*
+S_fullpath(RAMFolder *self, const CharBuf *path);
+
+RAMFolder*
+RAMFolder_new(const CharBuf *path) {
+ RAMFolder *self = (RAMFolder*)VTable_Make_Obj(RAMFOLDER);
+ return RAMFolder_init(self, path);
+}
+
+RAMFolder*
+RAMFolder_init(RAMFolder *self, const CharBuf *path) {
+ Folder_init((Folder*)self, path);
+ return self;
+}
+
+void
+RAMFolder_initialize(RAMFolder *self) {
+ UNUSED_VAR(self);
+}
+
+bool_t
+RAMFolder_check(RAMFolder *self) {
+ UNUSED_VAR(self);
+ return true;
+}
+
+bool_t
+RAMFolder_local_mkdir(RAMFolder *self, const CharBuf *name) {
+ if (Hash_Fetch(self->entries, (Obj*)name)) {
+ Err_set_error(Err_new(CB_newf("Can't MkDir, '%o' already exists",
+ name)));
+ return false;
+ }
+ else {
+ CharBuf *fullpath = S_fullpath(self, name);
+ Hash_Store(self->entries, (Obj*)name,
+ (Obj*)RAMFolder_new(fullpath));
+ DECREF(fullpath);
+ return true;
+ }
+}
+
+FileHandle*
+RAMFolder_local_open_filehandle(RAMFolder *self, const CharBuf *name,
+ uint32_t flags) {
+ RAMFileHandle *fh;
+ CharBuf *fullpath = S_fullpath(self, name);
+ RAMFile *file = (RAMFile*)Hash_Fetch(self->entries, (Obj*)name);
+ bool_t can_create
+ = (flags & (FH_WRITE_ONLY | FH_CREATE)) == (FH_WRITE_ONLY | FH_CREATE)
+ ? true : false;
+
+ // Make sure the filepath isn't a directory, and that it either exists
+ // or we have permission to create it.
+ if (file) {
+ if (!RAMFile_Is_A(file, RAMFILE)) {
+ Err_set_error(Err_new(CB_newf("Not a file: '%o'", fullpath)));
+ DECREF(fullpath);
+ return NULL;
+ }
+ }
+ else if (!can_create) {
+ Err_set_error(Err_new(CB_newf("File not found: '%o'", fullpath)));
+ DECREF(fullpath);
+ return NULL;
+ }
+
+ // Open the file and store it if it was just created.
+ fh = RAMFH_open(fullpath, flags, file);
+ if (fh) {
+ if (!file) {
+ file = RAMFH_Get_File(fh);
+ Hash_Store(self->entries, (Obj*)name, INCREF(file));
+ }
+ }
+ else {
+ Err *error = Err_get_error();
+ ERR_ADD_FRAME(error);
+ }
+
+ DECREF(fullpath);
+
+ return (FileHandle*)fh;
+}
+
+DirHandle*
+RAMFolder_local_open_dir(RAMFolder *self) {
+ RAMDirHandle *dh = RAMDH_new(self);
+ if (!dh) { ERR_ADD_FRAME(Err_get_error()); }
+ return (DirHandle*)dh;
+}
+
+bool_t
+RAMFolder_local_exists(RAMFolder *self, const CharBuf *name) {
+ return !!Hash_Fetch(self->entries, (Obj*)name);
+}
+
+bool_t
+RAMFolder_local_is_directory(RAMFolder *self, const CharBuf *name) {
+ Obj *entry = Hash_Fetch(self->entries, (Obj*)name);
+ if (entry && Obj_Is_A(entry, FOLDER)) { return true; }
+ return false;
+}
+
+#define OP_RENAME 1
+#define OP_HARD_LINK 2
+
+static bool_t
+S_rename_or_hard_link(RAMFolder *self, const CharBuf* from, const CharBuf *to,
+ Folder *from_folder, Folder *to_folder,
+ ZombieCharBuf *from_name, ZombieCharBuf *to_name,
+ int op) {
+ Obj *elem = NULL;
+ RAMFolder *inner_from_folder = NULL;
+ RAMFolder *inner_to_folder = NULL;
+ UNUSED_VAR(self);
+
+ // Make sure the source and destination folders exist.
+ if (!from_folder) {
+ Err_set_error(Err_new(CB_newf("File not found: '%o'", from)));
+ return false;
+ }
+ if (!to_folder) {
+ Err_set_error(Err_new(CB_newf("Invalid file path (can't find dir): '%o'",
+ to)));
+ return false;
+ }
+
+ // Extract RAMFolders from compound reader wrappers, if necessary.
+ if (Folder_Is_A(from_folder, COMPOUNDFILEREADER)) {
+ inner_from_folder = (RAMFolder*)CFReader_Get_Real_Folder(
+ (CompoundFileReader*)from_folder);
+ }
+ else {
+ inner_from_folder = (RAMFolder*)from_folder;
+ }
+ if (Folder_Is_A(to_folder, COMPOUNDFILEREADER)) {
+ inner_to_folder = (RAMFolder*)CFReader_Get_Real_Folder(
+ (CompoundFileReader*)to_folder);
+ }
+ else {
+ inner_to_folder = (RAMFolder*)to_folder;
+ }
+ if (!RAMFolder_Is_A(inner_from_folder, RAMFOLDER)) {
+ Err_set_error(Err_new(CB_newf("Not a RAMFolder, but a '%o'",
+ Obj_Get_Class_Name((Obj*)inner_from_folder))));
+ return false;
+ }
+ if (!RAMFolder_Is_A(inner_to_folder, RAMFOLDER)) {
+ Err_set_error(Err_new(CB_newf("Not a RAMFolder, but a '%o'",
+ Obj_Get_Class_Name((Obj*)inner_to_folder))));
+ return false;
+ }
+
+ // Find the original element.
+ elem = Hash_Fetch(inner_from_folder->entries, (Obj*)from_name);
+ if (!elem) {
+ if (Folder_Is_A(from_folder, COMPOUNDFILEREADER)
+ && Folder_Local_Exists(from_folder, (CharBuf*)from_name)
+ ) {
+ Err_set_error(Err_new(CB_newf("Source file '%o' is virtual",
+ from)));
+ }
+ else {
+ Err_set_error(Err_new(CB_newf("File not found: '%o'", from)));
+ }
+ return false;
+ }
+
+ // Execute the rename/hard-link.
+ if (op == OP_RENAME) {
+ Obj *existing = Hash_Fetch(inner_to_folder->entries, (Obj*)to_name);
+ if (existing) {
+ bool_t conflict = false;
+
+ // Return success fast if file is copied on top of itself.
+ if (inner_from_folder == inner_to_folder
+ && ZCB_Equals(from_name, (Obj*)to_name)
+ ) {
+ return true;
+ }
+
+ // Don't allow clobbering of different entry type.
+ if (Obj_Is_A(elem, RAMFILE)) {
+ if (!Obj_Is_A(existing, RAMFILE)) {
+ conflict = true;
+ }
+ }
+ else if (Obj_Is_A(elem, FOLDER)) {
+ if (!Obj_Is_A(existing, FOLDER)) {
+ conflict = true;
+ }
+ }
+ if (conflict) {
+ Err_set_error(Err_new(CB_newf("Can't clobber a %o with a %o",
+ Obj_Get_Class_Name(existing),
+ Obj_Get_Class_Name(elem))));
+ return false;
+ }
+ }
+
+ // Perform the store first, then the delete. Inform Folder objects
+ // about the relocation.
+ Hash_Store(inner_to_folder->entries, (Obj*)to_name, INCREF(elem));
+ DECREF(Hash_Delete(inner_from_folder->entries, (Obj*)from_name));
+ if (Obj_Is_A(elem, FOLDER)) {
+ CharBuf *newpath = S_fullpath(inner_to_folder, (CharBuf*)to_name);
+ Folder_Set_Path((Folder*)elem, newpath);
+ DECREF(newpath);
+ }
+ }
+ else if (op == OP_HARD_LINK) {
+ if (!Obj_Is_A(elem, RAMFILE)) {
+ Err_set_error(Err_new(CB_newf("'%o' isn't a file, it's a %o",
+ from, Obj_Get_Class_Name(elem))));
+ return false;
+ }
+ else {
+ Obj *existing
+ = Hash_Fetch(inner_to_folder->entries, (Obj*)to_name);
+ if (existing) {
+ Err_set_error(Err_new(CB_newf("'%o' already exists", to)));
+ return false;
+ }
+ else {
+ Hash_Store(inner_to_folder->entries, (Obj*)to_name,
+ INCREF(elem));
+ }
+ }
+ }
+ else {
+ THROW(ERR, "Unexpected op: %i32", (int32_t)op);
+ }
+
+ return true;
+}
+
+bool_t
+RAMFolder_rename(RAMFolder *self, const CharBuf* from, const CharBuf *to) {
+ Folder *from_folder = RAMFolder_Enclosing_Folder(self, from);
+ Folder *to_folder = RAMFolder_Enclosing_Folder(self, to);
+ ZombieCharBuf *from_name = IxFileNames_local_part(from, ZCB_BLANK());
+ ZombieCharBuf *to_name = IxFileNames_local_part(to, ZCB_BLANK());
+ bool_t result = S_rename_or_hard_link(self, from, to, from_folder,
+ to_folder, from_name, to_name,
+ OP_RENAME);
+ if (!result) { ERR_ADD_FRAME(Err_get_error()); }
+ return result;
+}
+
+bool_t
+RAMFolder_hard_link(RAMFolder *self, const CharBuf *from, const CharBuf *to) {
+ Folder *from_folder = RAMFolder_Enclosing_Folder(self, from);
+ Folder *to_folder = RAMFolder_Enclosing_Folder(self, to);
+ ZombieCharBuf *from_name = IxFileNames_local_part(from, ZCB_BLANK());
+ ZombieCharBuf *to_name = IxFileNames_local_part(to, ZCB_BLANK());
+ bool_t result = S_rename_or_hard_link(self, from, to, from_folder,
+ to_folder, from_name, to_name,
+ OP_HARD_LINK);
+ if (!result) { ERR_ADD_FRAME(Err_get_error()); }
+ return result;
+}
+
+bool_t
+RAMFolder_local_delete(RAMFolder *self, const CharBuf *name) {
+ Obj *entry = Hash_Fetch(self->entries, (Obj*)name);
+ if (entry) {
+ if (Obj_Is_A(entry, RAMFILE)) {
+ ;
+ }
+ else if (Obj_Is_A(entry, FOLDER)) {
+ RAMFolder *inner_folder;
+ if (Obj_Is_A(entry, COMPOUNDFILEREADER)) {
+ inner_folder = (RAMFolder*)CERTIFY(
+ CFReader_Get_Real_Folder((CompoundFileReader*)entry),
+ RAMFOLDER);
+ }
+ else {
+ inner_folder = (RAMFolder*)CERTIFY(entry, RAMFOLDER);
+ }
+ if (Hash_Get_Size(inner_folder->entries)) {
+ // Can't delete non-empty dir.
+ return false;
+ }
+ }
+ else {
+ return false;
+ }
+ DECREF(Hash_Delete(self->entries, (Obj*)name));
+ return true;
+ }
+ else {
+ return false;
+ }
+}
+
+Folder*
+RAMFolder_local_find_folder(RAMFolder *self, const CharBuf *path) {
+ Folder *local_folder = (Folder*)Hash_Fetch(self->entries, (Obj*)path);
+ if (local_folder && Folder_Is_A(local_folder, FOLDER)) {
+ return local_folder;
+ }
+ return NULL;
+}
+
+void
+RAMFolder_close(RAMFolder *self) {
+ UNUSED_VAR(self);
+}
+
+static CharBuf*
+S_fullpath(RAMFolder *self, const CharBuf *path) {
+ if (CB_Get_Size(self->path)) {
+ return CB_newf("%o/%o", self->path, path);
+ }
+ else {
+ return CB_Clone(path);
+ }
+}
+
+
diff --git a/core/Lucy/Store/RAMFolder.cfh b/core/Lucy/Store/RAMFolder.cfh
new file mode 100644
index 0000000..fb73747
--- /dev/null
+++ b/core/Lucy/Store/RAMFolder.cfh
@@ -0,0 +1,73 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** In-memory Folder implementation.
+ *
+ * RAMFolder is an entirely in-memory implementation of
+ * L<Lucy::Store::Folder>, primarily used for testing and development.
+ */
+
+class Lucy::Store::RAMFolder inherits Lucy::Store::Folder {
+
+ inert incremented RAMFolder*
+ new(const CharBuf *path = NULL);
+
+ /**
+ * @param path Relative path, used for subfolders.
+ */
+ public inert RAMFolder*
+ init(RAMFolder *self, const CharBuf *path = NULL);
+
+ public void
+ Initialize(RAMFolder *self);
+
+ public bool_t
+ Check(RAMFolder *self);
+
+ public void
+ Close(RAMFolder *self);
+
+ incremented nullable FileHandle*
+ Local_Open_FileHandle(RAMFolder *self, const CharBuf *name, uint32_t flags);
+
+ incremented nullable DirHandle*
+ Local_Open_Dir(RAMFolder *self);
+
+ bool_t
+ Local_MkDir(RAMFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Exists(RAMFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Is_Directory(RAMFolder *self, const CharBuf *name);
+
+ nullable Folder*
+ Local_Find_Folder(RAMFolder *self, const CharBuf *name);
+
+ bool_t
+ Local_Delete(RAMFolder *self, const CharBuf *name);
+
+ public bool_t
+ Rename(RAMFolder *self, const CharBuf* from, const CharBuf *to);
+
+ public bool_t
+ Hard_Link(RAMFolder *self, const CharBuf *from, const CharBuf *to);
+}
+
+
diff --git a/core/Lucy/Store/SharedLock.c b/core/Lucy/Store/SharedLock.c
new file mode 100644
index 0000000..d1a6ec2
--- /dev/null
+++ b/core/Lucy/Store/SharedLock.c
@@ -0,0 +1,165 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SHAREDLOCK
+#include "Lucy/Util/ToolSet.h"
+
+#include <errno.h>
+#include <stdio.h>
+#include <ctype.h>
+
+#include "Lucy/Store/SharedLock.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+
+SharedLock*
+ShLock_new(Folder *folder, const CharBuf *name, const CharBuf *host,
+ int32_t timeout, int32_t interval) {
+ SharedLock *self = (SharedLock*)VTable_Make_Obj(SHAREDLOCK);
+ return ShLock_init(self, folder, name, host, timeout, interval);
+}
+
+SharedLock*
+ShLock_init(SharedLock *self, Folder *folder, const CharBuf *name,
+ const CharBuf *host, int32_t timeout, int32_t interval) {
+ LFLock_init((LockFileLock*)self, folder, name, host, timeout, interval);
+
+ // Override.
+ DECREF(self->lock_path);
+ self->lock_path = (CharBuf*)INCREF(&EMPTY);
+
+ return self;
+}
+
+bool_t
+ShLock_shared(SharedLock *self) {
+ UNUSED_VAR(self);
+ return true;
+}
+
+bool_t
+ShLock_request(SharedLock *self) {
+ uint32_t i = 0;
+ ShLock_request_t super_request
+ = (ShLock_request_t)SUPER_METHOD(SHAREDLOCK, ShLock, Request);
+
+ // EMPTY lock_path indicates whether this particular instance is locked.
+ if (self->lock_path != (CharBuf*)&EMPTY
+ && Folder_Exists(self->folder, self->lock_path)
+ ) {
+ // Don't allow double obtain.
+ Err_set_error((Err*)LockErr_new(CB_newf("Lock already obtained via '%o'",
+ self->lock_path)));
+ return false;
+ }
+
+ DECREF(self->lock_path);
+ self->lock_path = CB_new(CB_Get_Size(self->name) + 10);
+ do {
+ CB_setf(self->lock_path, "locks/%o-%u32.lock", self->name, ++i);
+ } while (Folder_Exists(self->folder, self->lock_path));
+
+ bool_t success = super_request(self);
+ if (!success) { ERR_ADD_FRAME(Err_get_error()); }
+ return success;
+}
+
+void
+ShLock_release(SharedLock *self) {
+ if (self->lock_path != (CharBuf*)&EMPTY) {
+ ShLock_release_t super_release
+ = (ShLock_release_t)SUPER_METHOD(SHAREDLOCK, ShLock, Release);
+ super_release(self);
+
+ // Empty out lock_path.
+ DECREF(self->lock_path);
+ self->lock_path = (CharBuf*)INCREF(&EMPTY);
+ }
+}
+
+
+void
+ShLock_clear_stale(SharedLock *self) {
+ DirHandle *dh;
+ CharBuf *entry;
+ CharBuf *candidate = NULL;
+ CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
+
+ if (Folder_Find_Folder(self->folder, lock_dir_name)) {
+ dh = Folder_Open_Dir(self->folder, lock_dir_name);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+ entry = DH_Get_Entry(dh);
+ }
+ else {
+ return;
+ }
+
+ // Take a stab at any file that begins with our lock name.
+ while (DH_Next(dh)) {
+ if (CB_Starts_With(entry, self->name)
+ && CB_Ends_With_Str(entry, ".lock", 5)
+ ) {
+ candidate = candidate ? candidate : CB_new(0);
+ CB_setf(candidate, "%o/%o", lock_dir_name, entry);
+ ShLock_Maybe_Delete_File(self, candidate, false, true);
+ }
+ }
+
+ DECREF(candidate);
+ DECREF(dh);
+}
+
+bool_t
+ShLock_is_locked(SharedLock *self) {
+ DirHandle *dh;
+ CharBuf *entry;
+
+ CharBuf *lock_dir_name = (CharBuf*)ZCB_WRAP_STR("locks", 5);
+ if (Folder_Find_Folder(self->folder, lock_dir_name)) {
+ dh = Folder_Open_Dir(self->folder, lock_dir_name);
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+ entry = DH_Get_Entry(dh);
+ }
+ else {
+ return false;
+ }
+
+ while (DH_Next(dh)) {
+ // Translation: $locked = 1 if $entry =~ /^\Q$name-\d+\.lock$/
+ if (CB_Starts_With(entry, self->name)
+ && CB_Ends_With_Str(entry, ".lock", 5)
+ ) {
+ ZombieCharBuf *scratch = ZCB_WRAP(entry);
+ ZCB_Chop(scratch, sizeof(".lock") - 1);
+ while (isdigit(ZCB_Code_Point_From(scratch, 1))) {
+ ZCB_Chop(scratch, 1);
+ }
+ if (ZCB_Code_Point_From(scratch, 1) == '-') {
+ ZCB_Chop(scratch, 1);
+ if (ZCB_Equals(scratch, (Obj*)self->name)) {
+ DECREF(dh);
+ return true;
+ }
+ }
+ }
+ }
+
+ DECREF(dh);
+ return false;
+}
+
+
diff --git a/core/Lucy/Store/SharedLock.cfh b/core/Lucy/Store/SharedLock.cfh
new file mode 100644
index 0000000..0ef61ff
--- /dev/null
+++ b/core/Lucy/Store/SharedLock.cfh
@@ -0,0 +1,77 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Shared (read) lock.
+ *
+ * SharedLock's interface is nearly identical to that of its parent class
+ * L<Lucy::Store::Lock>, taking the same constructor arguments and
+ * implementing the same list of methods. It differs from Lock only in the
+ * semantics of two methods.
+ *
+ * First, Obtain() will not fail if another lock is held against the resource
+ * identified by <code>name</code> (though it might fail for other reasons).
+ *
+ * Second, Is_Locked() returns true so long as some lock, somewhere is holding
+ * a lock on <code>name</code>. That lock could be this instance, or it could
+ * be another -- so is entirely possible to call Release() successfully on a
+ * SharedLock object yet still have Is_Locked() return true.
+ *
+ * As currently implemented, SharedLock differs from Lock in that each caller
+ * gets its own lockfile. Lockfiles still have filenames which begin with the
+ * lock name and end with ".lock", but each is also assigned a unique number
+ * which gets pasted between: "foo-44.lock" instead of "foo.lock". A
+ * SharedLock is considered fully released when no lock files with a given
+ * lock name are left.
+ */
+class Lucy::Store::SharedLock cnick ShLock
+ inherits Lucy::Store::LockFileLock {
+
+ inert incremented SharedLock*
+ new(Folder *folder, const CharBuf *name, const CharBuf *host,
+ int32_t timeout = 0, int32_t interval = 100);
+
+ /**
+ * @param folder The Lucy::Store::Folder where the lock file will
+ * reside.
+ * @param name String identifying the resource to be locked.
+ * @param host An identifier which should be unique per-machine.
+ * @param timeout Time in milliseconds to keep retrying before abandoning
+ * the attempt to Obtain() a lock.
+ * @param interval Time in milliseconds between retries.
+ */
+ public inert SharedLock*
+ init(SharedLock *self, Folder *folder, const CharBuf *name,
+ const CharBuf *host, int32_t timeout = 0, int32_t interval = 100);
+
+ public bool_t
+ Shared(SharedLock *self);
+
+ public bool_t
+ Request(SharedLock *self);
+
+ public void
+ Release(SharedLock *self);
+
+ public bool_t
+ Is_Locked(SharedLock *self);
+
+ public void
+ Clear_Stale(SharedLock *self);
+}
+
+
diff --git a/core/Lucy/Test.c b/core/Lucy/Test.c
new file mode 100644
index 0000000..0457683
--- /dev/null
+++ b/core/Lucy/Test.c
@@ -0,0 +1,295 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define C_LUCY_TESTBATCH
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+
+TestBatch*
+TestBatch_new(int64_t num_tests) {
+ TestBatch *self = (TestBatch*)VTable_Make_Obj(TESTBATCH);
+ return TestBatch_init(self, num_tests);
+}
+
+TestBatch*
+TestBatch_init(TestBatch *self, int64_t num_tests) {
+ // Assign.
+ self->num_tests = num_tests;
+
+ // Initialize.
+ self->test_num = 0;
+ self->num_passed = 0;
+ self->num_failed = 0;
+ self->num_skipped = 0;
+
+ // Unbuffer stdout. TODO: move this elsewhere.
+ int check_val = setvbuf(stdout, NULL, _IONBF, 0);
+ if (check_val != 0) {
+ fprintf(stderr, "Failed when trying to unbuffer stdout\n");
+ }
+
+ return self;
+}
+
+void
+TestBatch_plan(TestBatch *self) {
+ printf("1..%" I64P "\n", self->num_tests);
+}
+
+bool_t
+TestBatch_test_true(void *vself, bool_t condition, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VTest_True((TestBatch*)vself, condition,
+ pattern, args);
+ va_end(args);
+ return result;
+}
+
+bool_t
+TestBatch_test_false(void *vself, bool_t condition, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VTest_False((TestBatch*)vself, condition,
+ pattern, args);
+ va_end(args);
+ return result;
+}
+
+bool_t
+TestBatch_test_int_equals(void *vself, long got, long expected,
+ const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VTest_Int_Equals((TestBatch*)vself, got,
+ expected, pattern, args);
+ va_end(args);
+ return result;
+}
+
+bool_t
+TestBatch_test_float_equals(void *vself, double got, double expected,
+ const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VTest_Float_Equals((TestBatch*)vself, got,
+ expected, pattern, args);
+ va_end(args);
+ return result;
+}
+
+bool_t
+TestBatch_test_string_equals(void *vself, const char *got,
+ const char *expected, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VTest_String_Equals((TestBatch*)vself, got,
+ expected, pattern, args);
+ va_end(args);
+ return result;
+}
+
+bool_t
+TestBatch_pass(void *vself, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VPass((TestBatch*)vself, pattern, args);
+ va_end(args);
+ return result;
+}
+
+bool_t
+TestBatch_fail(void *vself, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ bool_t result = TestBatch_VFail((TestBatch*)vself, pattern, args);
+ va_end(args);
+ return result;
+}
+
+void
+TestBatch_skip(void *vself, const char *pattern, ...) {
+ va_list args;
+ va_start(args, pattern);
+ TestBatch_VSkip((TestBatch*)vself, pattern, args);
+ va_end(args);
+}
+
+bool_t
+TestBatch_vtest_true(TestBatch *self, bool_t condition, const char *pattern,
+ va_list args) {
+ // Increment test number.
+ self->test_num++;
+
+ // Test condition and pass or fail.
+ if (condition) {
+ self->num_passed++;
+ printf("ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return true;
+ }
+ else {
+ self->num_failed++;
+ printf("not ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return false;
+ }
+}
+
+bool_t
+TestBatch_vtest_false(TestBatch *self, bool_t condition,
+ const char *pattern, va_list args) {
+ // Increment test number.
+ self->test_num++;
+
+ // Test condition and pass or fail.
+ if (!condition) {
+ self->num_passed++;
+ printf("ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return true;
+ }
+ else {
+ self->num_failed++;
+ printf("not ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return false;
+ }
+}
+
+bool_t
+TestBatch_vtest_int_equals(TestBatch *self, long got, long expected,
+ const char *pattern, va_list args) {
+ // Increment test number.
+ self->test_num++;
+
+ // Test condition and pass or fail.
+ if (expected == got) {
+ self->num_passed++;
+ printf("ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return true;
+ }
+ else {
+ self->num_failed++;
+ printf("not ok %" I64P " - Expected '%ld', got '%ld'\n ",
+ self->test_num, expected, got);
+ vprintf(pattern, args);
+ printf("\n");
+ return false;
+ }
+}
+
+bool_t
+TestBatch_vtest_float_equals(TestBatch *self, double got, double expected,
+ const char *pattern, va_list args) {
+ double diff = expected / got;
+
+ // Increment test number.
+ self->test_num++;
+
+ // Evaluate condition and pass or fail.
+ if (diff > 0.00001) {
+ self->num_passed++;
+ printf("ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return true;
+ }
+ else {
+ self->num_failed++;
+ printf("not ok %" I64P " - Expected '%f', got '%f'\n ",
+ self->test_num, expected, got);
+ vprintf(pattern, args);
+ printf("\n");
+ return false;
+ }
+}
+
+bool_t
+TestBatch_vtest_string_equals(TestBatch *self, const char *got,
+ const char *expected, const char *pattern,
+ va_list args) {
+ // Increment test number.
+ self->test_num++;
+
+ // Test condition and pass or fail.
+ if (strcmp(expected, got) == 0) {
+ self->num_passed++;
+ printf("ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ return true;
+ }
+ else {
+ self->num_failed++;
+ printf("not ok %" I64P " - Expected '%s', got '%s'\n ",
+ self->test_num, expected, got);
+ vprintf(pattern, args);
+ printf("\n");
+ return false;
+ }
+}
+
+bool_t
+TestBatch_vpass(TestBatch *self, const char *pattern, va_list args) {
+ // Increment test number.
+ self->test_num++;
+
+ // Update counter, indicate pass.
+ self->num_passed++;
+ printf("ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+
+ return true;
+}
+
+bool_t
+TestBatch_vfail(TestBatch *self, const char *pattern, va_list args) {
+ // Increment test number.
+ self->test_num++;
+
+ // Update counter, indicate failure.
+ self->num_failed++;
+ printf("not ok %" I64P " - ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+
+ return false;
+}
+
+void
+TestBatch_vskip(TestBatch *self, const char *pattern, va_list args) {
+ self->test_num++;
+ printf("ok %" I64P " # SKIP ", self->test_num);
+ vprintf(pattern, args);
+ printf("\n");
+ self->num_skipped++;
+}
+
+
diff --git a/core/Lucy/Test.cfh b/core/Lucy/Test.cfh
new file mode 100644
index 0000000..1b14224
--- /dev/null
+++ b/core/Lucy/Test.cfh
@@ -0,0 +1,109 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Testing framework.
+ */
+inert class Lucy::Test { }
+
+class Lucy::Test::TestBatch inherits Lucy::Object::Obj {
+ int64_t test_num;
+ int64_t num_tests;
+ int64_t num_passed;
+ int64_t num_failed;
+ int64_t num_skipped;
+
+ inert incremented TestBatch*
+ new(int64_t num_tests);
+
+ inert TestBatch*
+ init(TestBatch *self, int64_t num_tests);
+
+ void
+ Plan(TestBatch *self);
+
+ inert bool_t
+ test_true(void *vself, bool_t condition, const char *pattern, ...);
+
+ inert bool_t
+ test_false(void *vself, bool_t condition, const char *pattern, ...);
+
+ inert bool_t
+ test_int_equals(void *vself, long got, long expected,
+ const char *pattern, ...);
+
+ inert bool_t
+ test_float_equals(void *vself, double got, double expected,
+ const char *pattern, ...);
+
+ inert bool_t
+ test_string_equals(void *vself, const char *got, const char *expected,
+ const char *pattern, ...);
+
+ inert bool_t
+ pass(void *vself, const char *pattern, ...);
+
+ inert bool_t
+ fail(void *vself, const char *pattern, ...);
+
+ inert void
+ skip(void *vself, const char *pattern, ...);
+
+ bool_t
+ VTest_True(TestBatch *self, bool_t condition, const char *pattern,
+ va_list args);
+
+ bool_t
+ VTest_False(TestBatch *self, bool_t condition, const char *pattern,
+ va_list args);
+
+ bool_t
+ VTest_Int_Equals(TestBatch *self, long got, long expected,
+ const char *pattern, va_list args);
+
+ bool_t
+ VTest_Float_Equals(TestBatch *self, double got, double expected,
+ const char *pattern, va_list args);
+
+ bool_t
+ VTest_String_Equals(TestBatch *self, const char *got, const char *expected,
+ const char *pattern, va_list args);
+
+ bool_t
+ VPass(TestBatch *self, const char *pattern, va_list args);
+
+ bool_t
+ VFail(TestBatch *self, const char *pattern, va_list args);
+
+ void
+ VSkip(TestBatch *self, const char *pattern, va_list args);
+}
+
+__C__
+#ifdef LUCY_USE_SHORT_NAMES
+ #define TEST_TRUE lucy_TestBatch_test_true
+ #define TEST_FALSE lucy_TestBatch_test_false
+ #define TEST_INT_EQ lucy_TestBatch_test_int_equals
+ #define TEST_FLOAT_EQ lucy_TestBatch_test_float_equals
+ #define TEST_STR_EQ lucy_TestBatch_test_string_equals
+ #define PASS lucy_TestBatch_pass
+ #define FAIL lucy_TestBatch_fail
+ #define SKIP lucy_TestBatch_skip
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Test/Analysis/TestAnalyzer.c b/core/Lucy/Test/Analysis/TestAnalyzer.c
new file mode 100644
index 0000000..3edf4fe
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestAnalyzer.c
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Analysis/TestAnalyzer.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Analysis/Inversion.h"
+
+DummyAnalyzer*
+DummyAnalyzer_new() {
+ DummyAnalyzer *self = (DummyAnalyzer*)VTable_Make_Obj(DUMMYANALYZER);
+ return DummyAnalyzer_init(self);
+}
+
+DummyAnalyzer*
+DummyAnalyzer_init(DummyAnalyzer *self) {
+ return (DummyAnalyzer*)Analyzer_init((Analyzer*)self);
+}
+
+Inversion*
+DummyAnalyzer_transform(DummyAnalyzer *self, Inversion *inversion) {
+ UNUSED_VAR(self);
+ return (Inversion*)INCREF(inversion);
+}
+
+static void
+test_analysis(TestBatch *batch) {
+ DummyAnalyzer *analyzer = DummyAnalyzer_new();
+ CharBuf *source = CB_newf("foo bar baz");
+ VArray *wanted = VA_new(1);
+ VA_Push(wanted, (Obj*)CB_newf("foo bar baz"));
+ TestUtils_test_analyzer(batch, (Analyzer*)analyzer, source, wanted,
+ "test basic analysis");
+ DECREF(wanted);
+ DECREF(source);
+ DECREF(analyzer);
+}
+
+void
+TestAnalyzer_run_tests() {
+ TestBatch *batch = TestBatch_new(3);
+
+ TestBatch_Plan(batch);
+
+ test_analysis(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Analysis/TestAnalyzer.cfh b/core/Lucy/Test/Analysis/TestAnalyzer.cfh
new file mode 100644
index 0000000..53ddd62
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestAnalyzer.cfh
@@ -0,0 +1,34 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Analysis::TestAnalyzer {
+ inert void
+ run_tests();
+}
+
+class Lucy::Test::Analysis::DummyAnalyzer inherits Lucy::Analysis::Analyzer {
+ inert incremented DummyAnalyzer*
+ new();
+
+ inert DummyAnalyzer*
+ init(DummyAnalyzer *self);
+
+ public incremented Inversion*
+ Transform(DummyAnalyzer *self, Inversion *inversion);
+}
+
diff --git a/core/Lucy/Test/Analysis/TestCaseFolder.c b/core/Lucy/Test/Analysis/TestCaseFolder.c
new file mode 100644
index 0000000..94125fc
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestCaseFolder.c
@@ -0,0 +1,69 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTCASEFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Analysis/TestCaseFolder.h"
+#include "Lucy/Analysis/CaseFolder.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ CaseFolder *case_folder = CaseFolder_new();
+ CaseFolder *other = CaseFolder_new();
+ Obj *dump = (Obj*)CaseFolder_Dump(case_folder);
+ CaseFolder *clone = (CaseFolder*)CaseFolder_Load(other, dump);
+
+ TEST_TRUE(batch, CaseFolder_Equals(case_folder, (Obj*)other), "Equals");
+ TEST_FALSE(batch, CaseFolder_Equals(case_folder, (Obj*)&EMPTY), "Not Equals");
+ TEST_TRUE(batch, CaseFolder_Equals(case_folder, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(case_folder);
+ DECREF(other);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+static void
+test_analysis(TestBatch *batch) {
+ CaseFolder *case_folder = CaseFolder_new();
+ CharBuf *source = CB_newf("caPiTal ofFensE");
+ VArray *wanted = VA_new(1);
+ VA_Push(wanted, (Obj*)CB_newf("capital offense"));
+ TestUtils_test_analyzer(batch, (Analyzer*)case_folder, source, wanted,
+ "lowercase plain text");
+ DECREF(wanted);
+ DECREF(source);
+ DECREF(case_folder);
+}
+
+void
+TestCaseFolder_run_tests() {
+ TestBatch *batch = TestBatch_new(6);
+
+ TestBatch_Plan(batch);
+
+ test_Dump_Load_and_Equals(batch);
+ test_analysis(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Analysis/TestCaseFolder.cfh b/core/Lucy/Test/Analysis/TestCaseFolder.cfh
new file mode 100644
index 0000000..b82e109
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestCaseFolder.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Analysis::TestCaseFolder {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Analysis/TestPolyAnalyzer.c b/core/Lucy/Test/Analysis/TestPolyAnalyzer.c
new file mode 100644
index 0000000..c52bbc8
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestPolyAnalyzer.c
@@ -0,0 +1,171 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTPOLYANALYZER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Analysis/TestPolyAnalyzer.h"
+#include "Lucy/Analysis/PolyAnalyzer.h"
+#include "Lucy/Analysis/CaseFolder.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
+#include "Lucy/Analysis/SnowballStemmer.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ CharBuf *EN = (CharBuf*)ZCB_WRAP_STR("en", 2);
+ CharBuf *ES = (CharBuf*)ZCB_WRAP_STR("es", 2);
+ PolyAnalyzer *analyzer = PolyAnalyzer_new(EN, NULL);
+ PolyAnalyzer *other = PolyAnalyzer_new(ES, NULL);
+ Obj *dump = (Obj*)PolyAnalyzer_Dump(analyzer);
+ Obj *other_dump = (Obj*)PolyAnalyzer_Dump(other);
+ PolyAnalyzer *clone = (PolyAnalyzer*)PolyAnalyzer_Load(other, dump);
+ PolyAnalyzer *other_clone
+ = (PolyAnalyzer*)PolyAnalyzer_Load(other, other_dump);
+
+ TEST_FALSE(batch, PolyAnalyzer_Equals(analyzer, (Obj*)other),
+ "Equals() false with different language");
+ TEST_TRUE(batch, PolyAnalyzer_Equals(analyzer, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_TRUE(batch, PolyAnalyzer_Equals(other, (Obj*)other_clone),
+ "Dump => Load round trip");
+
+ DECREF(analyzer);
+ DECREF(dump);
+ DECREF(clone);
+ DECREF(other);
+ DECREF(other_dump);
+ DECREF(other_clone);
+}
+
+static void
+test_analysis(TestBatch *batch) {
+ CharBuf *EN = (CharBuf*)ZCB_WRAP_STR("en", 2);
+ CharBuf *source_text = CB_newf("Eats, shoots and leaves.");
+ CaseFolder *case_folder = CaseFolder_new();
+ RegexTokenizer *tokenizer = RegexTokenizer_new(NULL);
+ SnowballStopFilter *stopfilter = SnowStop_new(EN, NULL);
+ SnowballStemmer *stemmer = SnowStemmer_new(EN);
+
+ {
+ VArray *analyzers = VA_new(0);
+ PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
+ VArray *expected = VA_new(1);
+ VA_Push(expected, INCREF(source_text));
+ TestUtils_test_analyzer(batch, (Analyzer*)polyanalyzer, source_text,
+ expected, "No sub analyzers");
+ DECREF(expected);
+ DECREF(polyanalyzer);
+ DECREF(analyzers);
+ }
+
+ {
+ VArray *analyzers = VA_new(0);
+ VA_Push(analyzers, INCREF(case_folder));
+ PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
+ VArray *expected = VA_new(1);
+ VA_Push(expected, (Obj*)CB_newf("eats, shoots and leaves."));
+ TestUtils_test_analyzer(batch, (Analyzer*)polyanalyzer, source_text,
+ expected, "With CaseFolder");
+ DECREF(expected);
+ DECREF(polyanalyzer);
+ DECREF(analyzers);
+ }
+
+ {
+ VArray *analyzers = VA_new(0);
+ VA_Push(analyzers, INCREF(case_folder));
+ VA_Push(analyzers, INCREF(tokenizer));
+ PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
+ VArray *expected = VA_new(1);
+ VA_Push(expected, (Obj*)CB_newf("eats"));
+ VA_Push(expected, (Obj*)CB_newf("shoots"));
+ VA_Push(expected, (Obj*)CB_newf("and"));
+ VA_Push(expected, (Obj*)CB_newf("leaves"));
+ TestUtils_test_analyzer(batch, (Analyzer*)polyanalyzer, source_text,
+ expected, "With RegexTokenizer");
+ DECREF(expected);
+ DECREF(polyanalyzer);
+ DECREF(analyzers);
+ }
+
+ {
+ VArray *analyzers = VA_new(0);
+ VA_Push(analyzers, INCREF(case_folder));
+ VA_Push(analyzers, INCREF(tokenizer));
+ VA_Push(analyzers, INCREF(stopfilter));
+ PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
+ VArray *expected = VA_new(1);
+ VA_Push(expected, (Obj*)CB_newf("eats"));
+ VA_Push(expected, (Obj*)CB_newf("shoots"));
+ VA_Push(expected, (Obj*)CB_newf("leaves"));
+ TestUtils_test_analyzer(batch, (Analyzer*)polyanalyzer, source_text,
+ expected, "With SnowballStopFilter");
+ DECREF(expected);
+ DECREF(polyanalyzer);
+ DECREF(analyzers);
+ }
+
+ {
+ VArray *analyzers = VA_new(0);
+ VA_Push(analyzers, INCREF(case_folder));
+ VA_Push(analyzers, INCREF(tokenizer));
+ VA_Push(analyzers, INCREF(stopfilter));
+ VA_Push(analyzers, INCREF(stemmer));
+ PolyAnalyzer *polyanalyzer = PolyAnalyzer_new(NULL, analyzers);
+ VArray *expected = VA_new(1);
+ VA_Push(expected, (Obj*)CB_newf("eat"));
+ VA_Push(expected, (Obj*)CB_newf("shoot"));
+ VA_Push(expected, (Obj*)CB_newf("leav"));
+ TestUtils_test_analyzer(batch, (Analyzer*)polyanalyzer, source_text,
+ expected, "With SnowballStemmer");
+ DECREF(expected);
+ DECREF(polyanalyzer);
+ DECREF(analyzers);
+ }
+
+ DECREF(stemmer);
+ DECREF(stopfilter);
+ DECREF(tokenizer);
+ DECREF(case_folder);
+ DECREF(source_text);
+}
+
+static void
+test_Get_Analyzers(TestBatch *batch) {
+ VArray *analyzers = VA_new(0);
+ PolyAnalyzer *analyzer = PolyAnalyzer_new(NULL, analyzers);
+ TEST_TRUE(batch, PolyAnalyzer_Get_Analyzers(analyzer) == analyzers,
+ "Get_Analyzers()");
+ DECREF(analyzer);
+ DECREF(analyzers);
+}
+
+void
+TestPolyAnalyzer_run_tests() {
+ TestBatch *batch = TestBatch_new(19);
+
+ TestBatch_Plan(batch);
+
+ test_Dump_Load_and_Equals(batch);
+ test_analysis(batch);
+ test_Get_Analyzers(batch);
+
+ DECREF(batch);
+}
+
diff --git a/core/Lucy/Test/Analysis/TestPolyAnalyzer.cfh b/core/Lucy/Test/Analysis/TestPolyAnalyzer.cfh
new file mode 100644
index 0000000..c2983c7
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestPolyAnalyzer.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Analysis::TestPolyAnalyzer {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Analysis/TestRegexTokenizer.c b/core/Lucy/Test/Analysis/TestRegexTokenizer.c
new file mode 100644
index 0000000..ec6e507
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestRegexTokenizer.c
@@ -0,0 +1,70 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTREGEXTOKENIZER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Analysis/TestRegexTokenizer.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ ZombieCharBuf *word_char_pattern = ZCB_WRAP_STR("\\w+", 3);
+ ZombieCharBuf *whitespace_pattern = ZCB_WRAP_STR("\\S+", 3);
+ RegexTokenizer *word_char_tokenizer
+ = RegexTokenizer_new((CharBuf*)word_char_pattern);
+ RegexTokenizer *whitespace_tokenizer
+ = RegexTokenizer_new((CharBuf*)whitespace_pattern);
+ Obj *word_char_dump = RegexTokenizer_Dump(word_char_tokenizer);
+ Obj *whitespace_dump = RegexTokenizer_Dump(whitespace_tokenizer);
+ RegexTokenizer *word_char_clone
+ = RegexTokenizer_Load(whitespace_tokenizer, word_char_dump);
+ RegexTokenizer *whitespace_clone
+ = RegexTokenizer_Load(whitespace_tokenizer, whitespace_dump);
+
+ TEST_FALSE(batch,
+ RegexTokenizer_Equals(word_char_tokenizer, (Obj*)whitespace_tokenizer),
+ "Equals() false with different pattern");
+ TEST_TRUE(batch,
+ RegexTokenizer_Equals(word_char_tokenizer, (Obj*)word_char_clone),
+ "Dump => Load round trip");
+ TEST_TRUE(batch,
+ RegexTokenizer_Equals(whitespace_tokenizer, (Obj*)whitespace_clone),
+ "Dump => Load round trip");
+
+ DECREF(word_char_tokenizer);
+ DECREF(word_char_dump);
+ DECREF(word_char_clone);
+ DECREF(whitespace_tokenizer);
+ DECREF(whitespace_dump);
+ DECREF(whitespace_clone);
+}
+
+void
+TestRegexTokenizer_run_tests() {
+ TestBatch *batch = TestBatch_new(3);
+
+ TestBatch_Plan(batch);
+
+ test_Dump_Load_and_Equals(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Analysis/TestRegexTokenizer.cfh b/core/Lucy/Test/Analysis/TestRegexTokenizer.cfh
new file mode 100644
index 0000000..3975e00
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestRegexTokenizer.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Analysis::TestRegexTokenizer {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Analysis/TestSnowballStemmer.c b/core/Lucy/Test/Analysis/TestSnowballStemmer.c
new file mode 100644
index 0000000..aa05b84
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestSnowballStemmer.c
@@ -0,0 +1,111 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTSNOWBALLSTEMMER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Analysis/TestSnowballStemmer.h"
+#include "Lucy/Analysis/SnowballStemmer.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Util/Json.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ CharBuf *EN = (CharBuf*)ZCB_WRAP_STR("en", 2);
+ CharBuf *ES = (CharBuf*)ZCB_WRAP_STR("es", 2);
+ SnowballStemmer *stemmer = SnowStemmer_new(EN);
+ SnowballStemmer *other = SnowStemmer_new(ES);
+ Obj *dump = (Obj*)SnowStemmer_Dump(stemmer);
+ Obj *other_dump = (Obj*)SnowStemmer_Dump(other);
+ SnowballStemmer *clone = (SnowballStemmer*)SnowStemmer_Load(other, dump);
+ SnowballStemmer *other_clone = (SnowballStemmer*)SnowStemmer_Load(other, other_dump);
+
+ TEST_FALSE(batch,
+ SnowStemmer_Equals(stemmer, (Obj*)other),
+ "Equals() false with different language");
+ TEST_TRUE(batch,
+ SnowStemmer_Equals(stemmer, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_TRUE(batch,
+ SnowStemmer_Equals(other, (Obj*)other_clone),
+ "Dump => Load round trip");
+
+ DECREF(stemmer);
+ DECREF(dump);
+ DECREF(clone);
+ DECREF(other);
+ DECREF(other_dump);
+ DECREF(other_clone);
+}
+
+static void
+test_stemming(TestBatch *batch) {
+ CharBuf *path = CB_newf("modules");
+ FSFolder *modules_folder = FSFolder_new(path);
+ if (!FSFolder_Check(modules_folder)) {
+ DECREF(modules_folder);
+ CB_setf(path, "../modules");
+ modules_folder = FSFolder_new(path);
+ if (!FSFolder_Check(modules_folder)) {
+ THROW(ERR, "Can't open modules folder");
+ }
+ }
+ CB_setf(path, "analysis/snowstem/source/test/tests.json");
+ Hash *tests = (Hash*)Json_slurp_json((Folder*)modules_folder, path);
+ if (!tests) { RETHROW(Err_get_error()); }
+
+ CharBuf *iso;
+ Hash *lang_data;
+ Hash_Iterate(tests);
+ while (Hash_Next(tests, (Obj**)&iso, (Obj**)&lang_data)) {
+ VArray *words = (VArray*)Hash_Fetch_Str(lang_data, "words", 5);
+ VArray *stems = (VArray*)Hash_Fetch_Str(lang_data, "stems", 5);
+ SnowballStemmer *stemmer = SnowStemmer_new(iso);
+ for (uint32_t i = 0, max = VA_Get_Size(words); i < max; i++) {
+ CharBuf *word = (CharBuf*)VA_Fetch(words, i);
+ VArray *got = SnowStemmer_Split(stemmer, word);
+ CharBuf *stem = (CharBuf*)VA_Fetch(got, 0);
+ TEST_TRUE(batch,
+ stem
+ && CB_Is_A(stem, CHARBUF)
+ && CB_Equals(stem, VA_Fetch(stems, i)),
+ "Stem %s: %s", CB_Get_Ptr8(iso), CB_Get_Ptr8(word)
+ );
+ DECREF(got);
+ }
+ DECREF(stemmer);
+ }
+
+ DECREF(tests);
+ DECREF(modules_folder);
+ DECREF(path);
+}
+
+void
+TestSnowStemmer_run_tests() {
+ TestBatch *batch = TestBatch_new(153);
+
+ TestBatch_Plan(batch);
+
+ test_Dump_Load_and_Equals(batch);
+ test_stemming(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Analysis/TestSnowballStemmer.cfh b/core/Lucy/Test/Analysis/TestSnowballStemmer.cfh
new file mode 100644
index 0000000..8dbc5db
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestSnowballStemmer.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Analysis::TestSnowballStemmer cnick TestSnowStemmer {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Analysis/TestSnowballStopFilter.c b/core/Lucy/Test/Analysis/TestSnowballStopFilter.c
new file mode 100644
index 0000000..a414e64
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestSnowballStopFilter.c
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTSNOWBALLSTOPFILTER
+#include "Lucy/Util/ToolSet.h"
+#include <stdarg.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Analysis/TestSnowballStopFilter.h"
+#include "Lucy/Analysis/SnowballStopFilter.h"
+
+static SnowballStopFilter*
+S_make_stopfilter(void *unused, ...) {
+ va_list args;
+ SnowballStopFilter *self = (SnowballStopFilter*)VTable_Make_Obj(SNOWBALLSTOPFILTER);
+ Hash *stoplist = Hash_new(0);
+ char *stopword;
+
+ va_start(args, unused);
+ while (NULL != (stopword = va_arg(args, char*))) {
+ Hash_Store_Str(stoplist, stopword, strlen(stopword), INCREF(&EMPTY));
+ }
+ va_end(args);
+
+ self = SnowStop_init(self, NULL, stoplist);
+ DECREF(stoplist);
+ return self;
+}
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ SnowballStopFilter *stopfilter =
+ S_make_stopfilter(NULL, "foo", "bar", "baz", NULL);
+ SnowballStopFilter *other =
+ S_make_stopfilter(NULL, "foo", "bar", NULL);
+ Obj *dump = SnowStop_Dump(stopfilter);
+ Obj *other_dump = SnowStop_Dump(other);
+ SnowballStopFilter *clone = (SnowballStopFilter*)SnowStop_Load(other, dump);
+ SnowballStopFilter *other_clone = (SnowballStopFilter*)SnowStop_Load(other, other_dump);
+
+ TEST_FALSE(batch,
+ SnowStop_Equals(stopfilter, (Obj*)other),
+ "Equals() false with different stoplist");
+ TEST_TRUE(batch,
+ SnowStop_Equals(stopfilter, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_TRUE(batch,
+ SnowStop_Equals(other, (Obj*)other_clone),
+ "Dump => Load round trip");
+
+ DECREF(stopfilter);
+ DECREF(dump);
+ DECREF(clone);
+ DECREF(other);
+ DECREF(other_dump);
+ DECREF(other_clone);
+}
+
+void
+TestSnowStop_run_tests() {
+ TestBatch *batch = TestBatch_new(3);
+
+ TestBatch_Plan(batch);
+
+ test_Dump_Load_and_Equals(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh b/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh
new file mode 100644
index 0000000..5d6fd4e
--- /dev/null
+++ b/core/Lucy/Test/Analysis/TestSnowballStopFilter.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Analysis::TestSnowballStopFilter cnick TestSnowStop {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Index/TestDocWriter.c b/core/Lucy/Test/Index/TestDocWriter.c
new file mode 100644
index 0000000..76d6fe3
--- /dev/null
+++ b/core/Lucy/Test/Index/TestDocWriter.c
@@ -0,0 +1,32 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTDOCWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestDocWriter.h"
+#include "Lucy/Index/DocWriter.h"
+
+void
+TestDocWriter_run_tests() {
+ TestBatch *batch = TestBatch_new(1);
+ TestBatch_Plan(batch);
+ PASS(batch, "placeholder");
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Index/TestDocWriter.cfh b/core/Lucy/Test/Index/TestDocWriter.cfh
new file mode 100644
index 0000000..5cb2b76
--- /dev/null
+++ b/core/Lucy/Test/Index/TestDocWriter.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestDocWriter {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Index/TestHighlightWriter.c b/core/Lucy/Test/Index/TestHighlightWriter.c
new file mode 100644
index 0000000..464da67
--- /dev/null
+++ b/core/Lucy/Test/Index/TestHighlightWriter.c
@@ -0,0 +1,32 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTHIGHLIGHTWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestHighlightWriter.h"
+#include "Lucy/Index/HighlightWriter.h"
+
+void
+TestHLWriter_run_tests() {
+ TestBatch *batch = TestBatch_new(1);
+ TestBatch_Plan(batch);
+ PASS(batch, "Placeholder");
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Index/TestHighlightWriter.cfh b/core/Lucy/Test/Index/TestHighlightWriter.cfh
new file mode 100644
index 0000000..08e2abe
--- /dev/null
+++ b/core/Lucy/Test/Index/TestHighlightWriter.cfh
@@ -0,0 +1,26 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestHighlightWriter
+ cnick TestHLWriter {
+
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Index/TestIndexManager.c b/core/Lucy/Test/Index/TestIndexManager.c
new file mode 100644
index 0000000..ba75e3d
--- /dev/null
+++ b/core/Lucy/Test/Index/TestIndexManager.c
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestIndexManager.h"
+#include "Lucy/Index/IndexManager.h"
+
+static void
+test_Choose_Sparse(TestBatch *batch) {
+ IndexManager *manager = IxManager_new(NULL, NULL);
+
+ for (uint32_t num_segs = 2; num_segs < 20; num_segs++) {
+ I32Array *doc_counts = I32Arr_new_blank(num_segs);
+ for (uint32_t j = 0; j < num_segs; j++) {
+ I32Arr_Set(doc_counts, j, 1000);
+ }
+ uint32_t threshold = IxManager_Choose_Sparse(manager, doc_counts);
+ TEST_TRUE(batch, threshold != 1,
+ "Either don't merge, or merge two segments: %u segs, thresh %u",
+ (unsigned)num_segs, (unsigned)threshold);
+
+ if (num_segs != 12 && num_segs != 13) { // when 2 is correct
+ I32Arr_Set(doc_counts, 0, 1);
+ threshold = IxManager_Choose_Sparse(manager, doc_counts);
+ TEST_TRUE(batch, threshold != 2,
+ "Don't include big next seg: %u segs, thresh %u",
+ (unsigned)num_segs, (unsigned)threshold);
+ }
+
+ DECREF(doc_counts);
+ }
+
+ DECREF(manager);
+}
+
+void
+TestIxManager_run_tests() {
+ TestBatch *batch = TestBatch_new(34);
+ TestBatch_Plan(batch);
+ test_Choose_Sparse(batch);
+ DECREF(batch);
+}
+
diff --git a/core/Lucy/Test/Index/TestIndexManager.cfh b/core/Lucy/Test/Index/TestIndexManager.cfh
new file mode 100644
index 0000000..dc22069
--- /dev/null
+++ b/core/Lucy/Test/Index/TestIndexManager.cfh
@@ -0,0 +1,25 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestIndexManager
+ cnick TestIxManager {
+
+ inert void
+ run_tests();
+}
+
diff --git a/core/Lucy/Test/Index/TestPolyReader.c b/core/Lucy/Test/Index/TestPolyReader.c
new file mode 100644
index 0000000..51e35fd
--- /dev/null
+++ b/core/Lucy/Test/Index/TestPolyReader.c
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTPOLYREADER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestPolyReader.h"
+#include "Lucy/Index/PolyReader.h"
+
+static void
+test_sub_tick(TestBatch *batch) {
+ size_t num_segs = 255;
+ int32_t *ints = (int32_t*)MALLOCATE(num_segs * sizeof(int32_t));
+ size_t i;
+ for (i = 0; i < num_segs; i++) {
+ ints[i] = i;
+ }
+ I32Array *offsets = I32Arr_new(ints, num_segs);
+ for (i = 1; i < num_segs; i++) {
+ if (PolyReader_sub_tick(offsets, i) != i - 1) { break; }
+ }
+ TEST_INT_EQ(batch, i, num_segs, "got all sub_tick() calls right");
+ DECREF(offsets);
+ FREEMEM(ints);
+}
+
+void
+TestPolyReader_run_tests() {
+ TestBatch *batch = TestBatch_new(1);
+ TestBatch_Plan(batch);
+
+ test_sub_tick(batch);
+
+ DECREF(batch);
+}
+
diff --git a/core/Lucy/Test/Index/TestPolyReader.cfh b/core/Lucy/Test/Index/TestPolyReader.cfh
new file mode 100644
index 0000000..4bf4ec8
--- /dev/null
+++ b/core/Lucy/Test/Index/TestPolyReader.cfh
@@ -0,0 +1,23 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestPolyReader {
+ inert void
+ run_tests();
+}
+
diff --git a/core/Lucy/Test/Index/TestPostingListWriter.c b/core/Lucy/Test/Index/TestPostingListWriter.c
new file mode 100644
index 0000000..90aee46
--- /dev/null
+++ b/core/Lucy/Test/Index/TestPostingListWriter.c
@@ -0,0 +1,31 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTPOSTINGLISTWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestPostingListWriter.h"
+
+void
+TestPListWriter_run_tests() {
+ TestBatch *batch = TestBatch_new(1);
+ TestBatch_Plan(batch);
+ PASS(batch, "Placeholder");
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Index/TestPostingListWriter.cfh b/core/Lucy/Test/Index/TestPostingListWriter.cfh
new file mode 100644
index 0000000..af5401c
--- /dev/null
+++ b/core/Lucy/Test/Index/TestPostingListWriter.cfh
@@ -0,0 +1,26 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestPostingListWriter
+ cnick TestPListWriter {
+
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Index/TestSegWriter.c b/core/Lucy/Test/Index/TestSegWriter.c
new file mode 100644
index 0000000..31ac12c
--- /dev/null
+++ b/core/Lucy/Test/Index/TestSegWriter.c
@@ -0,0 +1,32 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTSEGWRITER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestSegWriter.h"
+#include "Lucy/Index/SegWriter.h"
+
+void
+TestSegWriter_run_tests() {
+ TestBatch *batch = TestBatch_new(1);
+ TestBatch_Plan(batch);
+ PASS(batch, "placeholder");
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Index/TestSegWriter.cfh b/core/Lucy/Test/Index/TestSegWriter.cfh
new file mode 100644
index 0000000..121b182
--- /dev/null
+++ b/core/Lucy/Test/Index/TestSegWriter.cfh
@@ -0,0 +1,25 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestSegWriter {
+
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Index/TestSegment.c b/core/Lucy/Test/Index/TestSegment.c
new file mode 100644
index 0000000..6d164bc
--- /dev/null
+++ b/core/Lucy/Test/Index/TestSegment.c
@@ -0,0 +1,166 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTSEG
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestSegment.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Store/RAMFolder.h"
+
+static void
+test_fields(TestBatch *batch) {
+ Segment *segment = Seg_new(1);
+ ZombieCharBuf *foo = ZCB_WRAP_STR("foo", 3);
+ ZombieCharBuf *bar = ZCB_WRAP_STR("bar", 3);
+ ZombieCharBuf *baz = ZCB_WRAP_STR("baz", 3);
+ int32_t field_num;
+
+ field_num = Seg_Add_Field(segment, (CharBuf*)foo);
+ TEST_TRUE(batch, field_num == 1,
+ "Add_Field returns field number, and field numbers start at 1");
+ field_num = Seg_Add_Field(segment, (CharBuf*)bar);
+ TEST_TRUE(batch, field_num == 2, "add a second field");
+ field_num = Seg_Add_Field(segment, (CharBuf*)foo);
+ TEST_TRUE(batch, field_num == 1,
+ "Add_Field returns existing field number if field is already known");
+
+ TEST_TRUE(batch, ZCB_Equals(bar, (Obj*)Seg_Field_Name(segment, 2)),
+ "Field_Name");
+ TEST_TRUE(batch, Seg_Field_Name(segment, 3) == NULL,
+ "Field_Name returns NULL for unknown field number");
+ TEST_TRUE(batch, Seg_Field_Num(segment, (CharBuf*)bar) == 2,
+ "Field_Num");
+ TEST_TRUE(batch, Seg_Field_Num(segment, (CharBuf*)baz) == 0,
+ "Field_Num returns 0 for unknown field name");
+
+ DECREF(segment);
+}
+
+static void
+test_metadata_storage(TestBatch *batch) {
+ Segment *segment = Seg_new(1);
+ CharBuf *got;
+
+ Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
+ got = (CharBuf*)Seg_Fetch_Metadata_Str(segment, "foo", 3);
+ TEST_TRUE(batch,
+ got
+ && CB_Is_A(got, CHARBUF)
+ && CB_Equals_Str(got, "bar", 3),
+ "metadata round trip"
+ );
+ DECREF(segment);
+}
+
+static void
+test_seg_name_and_num(TestBatch *batch) {
+ Segment *segment_z = Seg_new(35);
+ CharBuf *seg_z_name = Seg_num_to_name(35);
+ TEST_TRUE(batch, Seg_Get_Number(segment_z) == I64_C(35), "Get_Number");
+ TEST_TRUE(batch, CB_Equals_Str(Seg_Get_Name(segment_z), "seg_z", 5),
+ "Get_Name");
+ TEST_TRUE(batch, CB_Equals_Str(seg_z_name, "seg_z", 5),
+ "num_to_name");
+ DECREF(seg_z_name);
+ DECREF(segment_z);
+}
+
+static void
+test_count(TestBatch *batch) {
+ Segment *segment = Seg_new(100);
+
+ TEST_TRUE(batch, Seg_Get_Count(segment) == 0, "count starts off at 0");
+ Seg_Set_Count(segment, 120);
+ TEST_TRUE(batch, Seg_Get_Count(segment) == 120, "Set_Count");
+ TEST_TRUE(batch, Seg_Increment_Count(segment, 10) == 130,
+ "Increment_Count");
+
+ DECREF(segment);
+}
+
+static void
+test_Compare_To(TestBatch *batch) {
+ Segment *segment_1 = Seg_new(1);
+ Segment *segment_2 = Seg_new(2);
+ Segment *also_segment_2 = Seg_new(2);
+
+ TEST_TRUE(batch, Seg_Compare_To(segment_1, (Obj*)segment_2) < 0,
+ "Compare_To 1 < 2");
+ TEST_TRUE(batch, Seg_Compare_To(segment_2, (Obj*)segment_1) > 0,
+ "Compare_To 1 < 2");
+ TEST_TRUE(batch, Seg_Compare_To(segment_1, (Obj*)segment_1) == 0,
+ "Compare_To identity");
+ TEST_TRUE(batch, Seg_Compare_To(segment_2, (Obj*)also_segment_2) == 0,
+ "Compare_To 2 == 2");
+
+ DECREF(segment_1);
+ DECREF(segment_2);
+ DECREF(also_segment_2);
+}
+
+static void
+test_Write_File_and_Read_File(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ Segment *segment = Seg_new(100);
+ Segment *got = Seg_new(100);
+ CharBuf *meta;
+ CharBuf *flotsam = (CharBuf*)ZCB_WRAP_STR("flotsam", 7);
+ CharBuf *jetsam = (CharBuf*)ZCB_WRAP_STR("jetsam", 6);
+
+ Seg_Set_Count(segment, 111);
+ Seg_Store_Metadata_Str(segment, "foo", 3, (Obj*)CB_newf("bar"));
+ Seg_Add_Field(segment, flotsam);
+ Seg_Add_Field(segment, jetsam);
+
+ RAMFolder_MkDir(folder, Seg_Get_Name(segment));
+ Seg_Write_File(segment, (Folder*)folder);
+ Seg_Read_File(got, (Folder*)folder);
+
+ TEST_TRUE(batch, Seg_Get_Count(got) == Seg_Get_Count(segment),
+ "Round-trip count through file");
+ TEST_TRUE(batch,
+ Seg_Field_Num(got, jetsam) == Seg_Field_Num(segment, jetsam),
+ "Round trip field names through file");
+ meta = (CharBuf*)Seg_Fetch_Metadata_Str(got, "foo", 3);
+ TEST_TRUE(batch,
+ meta
+ && CB_Is_A(meta, CHARBUF)
+ && CB_Equals_Str(meta, "bar", 3),
+ "Round trip metadata through file");
+
+ DECREF(got);
+ DECREF(segment);
+ DECREF(folder);
+}
+
+void
+TestSeg_run_tests() {
+ TestBatch *batch = TestBatch_new(21);
+
+ TestBatch_Plan(batch);
+ test_fields(batch);
+ test_metadata_storage(batch);
+ test_seg_name_and_num(batch);
+ test_count(batch);
+ test_Compare_To(batch);
+ test_Write_File_and_Read_File(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Index/TestSegment.cfh b/core/Lucy/Test/Index/TestSegment.cfh
new file mode 100644
index 0000000..8befd8c
--- /dev/null
+++ b/core/Lucy/Test/Index/TestSegment.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestSegment cnick TestSeg {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Index/TestSnapshot.c b/core/Lucy/Test/Index/TestSnapshot.c
new file mode 100644
index 0000000..9cd5eb4
--- /dev/null
+++ b/core/Lucy/Test/Index/TestSnapshot.c
@@ -0,0 +1,107 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Index/TestSnapshot.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Store/RAMFolder.h"
+
+static void
+test_Add_and_Delete(TestBatch *batch) {
+ Snapshot *snapshot = Snapshot_new();
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+ CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
+
+ Snapshot_Add_Entry(snapshot, foo);
+ Snapshot_Add_Entry(snapshot, foo); // redundant
+ VArray *entries = Snapshot_List(snapshot);
+ TEST_INT_EQ(batch, Snapshot_Num_Entries(snapshot), 1,
+ "One entry added");
+ TEST_TRUE(batch, CB_Equals(foo, VA_Fetch(entries, 0)), "correct entry");
+ DECREF(entries);
+
+ Snapshot_Add_Entry(snapshot, bar);
+ TEST_INT_EQ(batch, Snapshot_Num_Entries(snapshot), 2,
+ "second entry added");
+ Snapshot_Delete_Entry(snapshot, foo);
+ TEST_INT_EQ(batch, Snapshot_Num_Entries(snapshot), 1, "Delete_Entry");
+
+ DECREF(snapshot);
+}
+
+static void
+test_path_handling(TestBatch *batch) {
+ Snapshot *snapshot = Snapshot_new();
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ CharBuf *snap = (CharBuf*)ZCB_WRAP_STR("snap", 4);
+ CharBuf *crackle = (CharBuf*)ZCB_WRAP_STR("crackle", 7);
+
+ Snapshot_Write_File(snapshot, folder, snap);
+ TEST_TRUE(batch, CB_Equals(snap, (Obj*)Snapshot_Get_Path(snapshot)),
+ "Write_File() sets path as a side effect");
+
+ Folder_Rename(folder, snap, crackle);
+ Snapshot_Read_File(snapshot, folder, crackle);
+ TEST_TRUE(batch, CB_Equals(crackle, (Obj*)Snapshot_Get_Path(snapshot)),
+ "Read_File() sets path as a side effect");
+
+ Snapshot_Set_Path(snapshot, snap);
+ TEST_TRUE(batch, CB_Equals(snap, (Obj*)Snapshot_Get_Path(snapshot)),
+ "Set_Path()");
+
+ DECREF(folder);
+ DECREF(snapshot);
+}
+
+static void
+test_Read_File_and_Write_File(TestBatch *batch) {
+ Snapshot *snapshot = Snapshot_new();
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ CharBuf *snap = (CharBuf*)ZCB_WRAP_STR("snap", 4);
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+
+ Snapshot_Add_Entry(snapshot, foo);
+ Snapshot_Write_File(snapshot, folder, snap);
+
+ Snapshot *dupe = Snapshot_new();
+ Snapshot *read_retval = Snapshot_Read_File(dupe, folder, snap);
+ TEST_TRUE(batch, dupe == read_retval, "Read_File() returns the object");
+
+ VArray *orig_list = Snapshot_List(snapshot);
+ VArray *dupe_list = Snapshot_List(dupe);
+ TEST_TRUE(batch, VA_Equals(orig_list, (Obj*)dupe_list),
+ "Round trip through Write_File() and Read_File()");
+
+ DECREF(orig_list);
+ DECREF(dupe_list);
+ DECREF(dupe);
+ DECREF(snapshot);
+ DECREF(folder);
+}
+
+void
+TestSnapshot_run_tests() {
+ TestBatch *batch = TestBatch_new(9);
+ TestBatch_Plan(batch);
+ test_Add_and_Delete(batch);
+ test_path_handling(batch);
+ test_Read_File_and_Write_File(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Index/TestSnapshot.cfh b/core/Lucy/Test/Index/TestSnapshot.cfh
new file mode 100644
index 0000000..b577a24
--- /dev/null
+++ b/core/Lucy/Test/Index/TestSnapshot.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Index::TestSnapshot {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestBitVector.c b/core/Lucy/Test/Object/TestBitVector.c
new file mode 100644
index 0000000..3fed17c
--- /dev/null
+++ b/core/Lucy/Test/Object/TestBitVector.c
@@ -0,0 +1,460 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTBITVECTOR
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestBitVector.h"
+
+static void
+test_Set_and_Get(TestBatch *batch) {
+ unsigned i, max;
+ const uint32_t three = 3;
+ const uint32_t seventeen = 17;
+ BitVector *bit_vec = BitVec_new(8);
+
+ BitVec_Set(bit_vec, three);
+ TEST_TRUE(batch, BitVec_Get_Capacity(bit_vec) < seventeen,
+ "set below cap");
+ BitVec_Set(bit_vec, seventeen);
+ TEST_TRUE(batch, BitVec_Get_Capacity(bit_vec) > seventeen,
+ "set above cap causes BitVector to grow");
+
+ for (i = 0, max = BitVec_Get_Capacity(bit_vec); i < max; i++) {
+ if (i == three || i == seventeen) {
+ TEST_TRUE(batch, BitVec_Get(bit_vec, i), "set/get %d", i);
+ }
+ else {
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i), "get %d", i);
+ }
+ }
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i), "out of range get");
+
+ DECREF(bit_vec);
+}
+
+static void
+test_Flip(TestBatch *batch) {
+ BitVector *bit_vec = BitVec_new(0);
+ int i;
+
+ for (i = 0; i <= 20; i++) { BitVec_Flip(bit_vec, i); }
+ for (i = 0; i <= 20; i++) {
+ TEST_TRUE(batch, BitVec_Get(bit_vec, i), "flip on %d", i);
+ }
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i), "no flip %d", i);
+ for (i = 0; i <= 20; i++) { BitVec_Flip(bit_vec, i); }
+ for (i = 0; i <= 20; i++) {
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i), "flip off %d", i);
+ }
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i), "still no flip %d", i);
+
+ DECREF(bit_vec);
+}
+
+static void
+test_Flip_Block_ascending(TestBatch *batch) {
+ BitVector *bit_vec = BitVec_new(0);
+ int i;
+
+ for (i = 0; i <= 20; i++) {
+ BitVec_Flip_Block(bit_vec, i, 21 - i);
+ }
+
+ for (i = 0; i <= 20; i++) {
+ if (i % 2 == 0) {
+ TEST_TRUE(batch, BitVec_Get(bit_vec, i),
+ "Flip_Block ascending %d", i);
+ }
+ else {
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i),
+ "Flip_Block ascending %d", i);
+ }
+ }
+
+ DECREF(bit_vec);
+}
+
+static void
+test_Flip_Block_descending(TestBatch *batch) {
+ BitVector *bit_vec = BitVec_new(0);
+ int i;
+
+ for (i = 19; i >= 0; i--) {
+ BitVec_Flip_Block(bit_vec, 1, i);
+ }
+
+ for (i = 0; i <= 20; i++) {
+ if (i % 2) {
+ TEST_TRUE(batch, BitVec_Get(bit_vec, i),
+ "Flip_Block descending %d", i);
+ }
+ else {
+ TEST_FALSE(batch, BitVec_Get(bit_vec, i),
+ "Flip_Block descending %d", i);
+ }
+ }
+
+ DECREF(bit_vec);
+}
+
+static void
+test_Flip_Block_bulk(TestBatch *batch) {
+ int32_t offset;
+
+ for (offset = 0; offset <= 17; offset++) {
+ int32_t len;
+ for (len = 0; len <= 17; len++) {
+ int i;
+ int upper = offset + len - 1;
+ BitVector *bit_vec = BitVec_new(0);
+
+ BitVec_Flip_Block(bit_vec, offset, len);
+ for (i = 0; i <= 17; i++) {
+ if (i >= offset && i <= upper) {
+ if (!BitVec_Get(bit_vec, i)) { break; }
+ }
+ else {
+ if (BitVec_Get(bit_vec, i)) { break; }
+ }
+ }
+ TEST_INT_EQ(batch, i, 18, "Flip_Block(%d, %d)", offset, len);
+
+ DECREF(bit_vec);
+ }
+ }
+}
+
+static void
+test_Mimic(TestBatch *batch) {
+ int foo;
+
+ for (foo = 0; foo <= 17; foo++) {
+ int bar;
+ for (bar = 0; bar <= 17; bar++) {
+ int i;
+ BitVector *foo_vec = BitVec_new(0);
+ BitVector *bar_vec = BitVec_new(0);
+
+ BitVec_Set(foo_vec, foo);
+ BitVec_Set(bar_vec, bar);
+ BitVec_Mimic(foo_vec, (Obj*)bar_vec);
+
+ for (i = 0; i <= 17; i++) {
+ if (BitVec_Get(foo_vec, i) && i != bar) { break; }
+ }
+ TEST_INT_EQ(batch, i, 18, "Mimic(%d, %d)", foo, bar);
+
+ DECREF(foo_vec);
+ DECREF(bar_vec);
+ }
+ }
+}
+
+static BitVector*
+S_create_set(int set_num) {
+ int i;
+ int nums_1[] = { 1, 2, 3, 10, 20, 30, 0 };
+ int nums_2[] = { 2, 3, 4, 5, 6, 7, 8, 9, 10,
+ 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 0
+ };
+ int *nums = set_num == 1 ? nums_1 : nums_2;
+ BitVector *bit_vec = BitVec_new(31);
+ for (i = 0; nums[i] != 0; i++) {
+ BitVec_Set(bit_vec, nums[i]);
+ }
+ return bit_vec;
+}
+
+#define OP_OR 1
+#define OP_XOR 2
+#define OP_AND 3
+#define OP_AND_NOT 4
+static int
+S_verify_logical_op(BitVector *bit_vec, BitVector *set_1, BitVector *set_2,
+ int op) {
+ int i;
+
+ for (i = 0; i < 50; i++) {
+ bool_t wanted;
+
+ switch (op) {
+ case OP_OR:
+ wanted = BitVec_Get(set_1, i) || BitVec_Get(set_2, i);
+ break;
+ case OP_XOR:
+ wanted = (!BitVec_Get(set_1, i)) != (!BitVec_Get(set_2, i));
+ break;
+ case OP_AND:
+ wanted = BitVec_Get(set_1, i) && BitVec_Get(set_2, i);
+ break;
+ case OP_AND_NOT:
+ wanted = BitVec_Get(set_1, i) && (!BitVec_Get(set_2, i));
+ break;
+ default:
+ wanted = false;
+ THROW(ERR, "unknown op: %d", op);
+ }
+
+ if (BitVec_Get(bit_vec, i) != wanted) { break; }
+ }
+
+ return i;
+}
+
+static void
+test_Or(TestBatch *batch) {
+ BitVector *smaller = S_create_set(1);
+ BitVector *larger = S_create_set(2);
+ BitVector *set_1 = S_create_set(1);
+ BitVector *set_2 = S_create_set(2);
+
+ BitVec_Or(smaller, set_2);
+ TEST_INT_EQ(batch, S_verify_logical_op(smaller, set_1, set_2, OP_OR),
+ 50, "OR with self smaller than other");
+ BitVec_Or(larger, set_1);
+ TEST_INT_EQ(batch, S_verify_logical_op(larger, set_1, set_2, OP_OR),
+ 50, "OR with other smaller than self");
+
+ DECREF(smaller);
+ DECREF(larger);
+ DECREF(set_1);
+ DECREF(set_2);
+}
+
+static void
+test_Xor(TestBatch *batch) {
+ BitVector *smaller = S_create_set(1);
+ BitVector *larger = S_create_set(2);
+ BitVector *set_1 = S_create_set(1);
+ BitVector *set_2 = S_create_set(2);
+
+ BitVec_Xor(smaller, set_2);
+ TEST_INT_EQ(batch, S_verify_logical_op(smaller, set_1, set_2, OP_XOR),
+ 50, "XOR with self smaller than other");
+ BitVec_Xor(larger, set_1);
+ TEST_INT_EQ(batch, S_verify_logical_op(larger, set_1, set_2, OP_XOR),
+ 50, "XOR with other smaller than self");
+
+ DECREF(smaller);
+ DECREF(larger);
+ DECREF(set_1);
+ DECREF(set_2);
+}
+
+static void
+test_And(TestBatch *batch) {
+ BitVector *smaller = S_create_set(1);
+ BitVector *larger = S_create_set(2);
+ BitVector *set_1 = S_create_set(1);
+ BitVector *set_2 = S_create_set(2);
+
+ BitVec_And(smaller, set_2);
+ TEST_INT_EQ(batch, S_verify_logical_op(smaller, set_1, set_2, OP_AND),
+ 50, "AND with self smaller than other");
+ BitVec_And(larger, set_1);
+ TEST_INT_EQ(batch, S_verify_logical_op(larger, set_1, set_2, OP_AND),
+ 50, "AND with other smaller than self");
+
+ DECREF(smaller);
+ DECREF(larger);
+ DECREF(set_1);
+ DECREF(set_2);
+}
+
+static void
+test_And_Not(TestBatch *batch) {
+ BitVector *smaller = S_create_set(1);
+ BitVector *larger = S_create_set(2);
+ BitVector *set_1 = S_create_set(1);
+ BitVector *set_2 = S_create_set(2);
+
+ BitVec_And_Not(smaller, set_2);
+ TEST_INT_EQ(batch,
+ S_verify_logical_op(smaller, set_1, set_2, OP_AND_NOT),
+ 50, "AND_NOT with self smaller than other");
+ BitVec_And_Not(larger, set_1);
+ TEST_INT_EQ(batch,
+ S_verify_logical_op(larger, set_2, set_1, OP_AND_NOT),
+ 50, "AND_NOT with other smaller than self");
+
+ DECREF(smaller);
+ DECREF(larger);
+ DECREF(set_1);
+ DECREF(set_2);
+}
+
+static void
+test_Count(TestBatch *batch) {
+ int i;
+ int shuffled[64];
+ BitVector *bit_vec = BitVec_new(64);
+
+ for (i = 0; i < 64; i++) { shuffled[i] = i; }
+ for (i = 0; i < 64; i++) {
+ int shuffle_pos = rand() % 64;
+ int temp = shuffled[shuffle_pos];
+ shuffled[shuffle_pos] = shuffled[i];
+ shuffled[i] = temp;
+ }
+ for (i = 0; i < 64; i++) {
+ BitVec_Set(bit_vec, shuffled[i]);
+ if (BitVec_Count(bit_vec) != (uint32_t)(i + 1)) { break; }
+ }
+ TEST_INT_EQ(batch, i, 64, "Count() returns the right number of bits");
+
+ DECREF(bit_vec);
+}
+
+static void
+test_Next_Hit(TestBatch *batch) {
+ int i;
+
+ for (i = 24; i <= 33; i++) {
+ int probe;
+ BitVector *bit_vec = BitVec_new(64);
+ BitVec_Set(bit_vec, i);
+ TEST_INT_EQ(batch, BitVec_Next_Hit(bit_vec, 0), i,
+ "Next_Hit for 0 is %d", i);
+ TEST_INT_EQ(batch, BitVec_Next_Hit(bit_vec, 0), i,
+ "Next_Hit for 1 is %d", i);
+ for (probe = 15; probe <= i; probe++) {
+ TEST_INT_EQ(batch, BitVec_Next_Hit(bit_vec, probe), i,
+ "Next_Hit for %d is %d", probe, i);
+ }
+ for (probe = i + 1; probe <= i + 9; probe++) {
+ TEST_INT_EQ(batch, BitVec_Next_Hit(bit_vec, probe), -1,
+ "no Next_Hit for %d when max is %d", probe, i);
+ }
+ DECREF(bit_vec);
+ }
+}
+
+static void
+test_Clear_All(TestBatch *batch) {
+ BitVector *bit_vec = BitVec_new(64);
+ BitVec_Flip_Block(bit_vec, 0, 63);
+ BitVec_Clear_All(bit_vec);
+ TEST_INT_EQ(batch, BitVec_Next_Hit(bit_vec, 0), -1, "Clear_All");
+ DECREF(bit_vec);
+}
+
+static void
+test_Clone(TestBatch *batch) {
+ int i;
+ BitVector *self = BitVec_new(30);
+ BitVector *twin;
+
+ BitVec_Set(self, 2);
+ BitVec_Set(self, 3);
+ BitVec_Set(self, 10);
+ BitVec_Set(self, 20);
+
+ twin = BitVec_Clone(self);
+ for (i = 0; i < 50; i++) {
+ if (BitVec_Get(self, i) != BitVec_Get(twin, i)) { break; }
+ }
+ TEST_INT_EQ(batch, i, 50, "Clone");
+ TEST_INT_EQ(batch, BitVec_Count(twin), 4, "clone Count");
+
+ DECREF(self);
+ DECREF(twin);
+}
+
+static int
+S_compare_u64s(void *context, const void *va, const void *vb) {
+ uint64_t a = *(uint64_t*)va;
+ uint64_t b = *(uint64_t*)vb;
+ UNUSED_VAR(context);
+ return a == b ? 0 : a < b ? -1 : 1;
+}
+
+static void
+test_To_Array(TestBatch *batch) {
+ uint64_t *source_ints = TestUtils_random_u64s(NULL, 20, 0, 200);
+ BitVector *bit_vec = BitVec_new(0);
+ I32Array *array;
+ long num_unique = 0;
+ long i;
+
+ // Unique the random ints.
+ Sort_quicksort(source_ints, 20, sizeof(uint64_t),
+ S_compare_u64s, NULL);
+ for (i = 0; i < 19; i++) {
+ if (source_ints[i] != source_ints[i + 1]) {
+ source_ints[num_unique] = source_ints[i];
+ num_unique++;
+ }
+ }
+
+ // Set bits.
+ for (i = 0; i < num_unique; i++) {
+ BitVec_Set(bit_vec, (uint32_t)source_ints[i]);
+ }
+
+ // Create the array and compare it to the source.
+ array = BitVec_To_Array(bit_vec);
+ for (i = 0; i < num_unique; i++) {
+ if (I32Arr_Get(array, i) != (int32_t)source_ints[i]) { break; }
+ }
+ TEST_INT_EQ(batch, i, num_unique, "To_Array (%ld == %ld)", i,
+ num_unique);
+
+ DECREF(array);
+ DECREF(bit_vec);
+ FREEMEM(source_ints);
+}
+
+
+// Valgrind only - detect off-by-one error.
+static void
+test_off_by_one_error() {
+ int cap;
+ for (cap = 5; cap <= 24; cap++) {
+ BitVector *bit_vec = BitVec_new(cap);
+ BitVec_Set(bit_vec, cap - 2);
+ DECREF(bit_vec);
+ }
+}
+
+void
+TestBitVector_run_tests() {
+ TestBatch *batch = TestBatch_new(1029);
+
+ TestBatch_Plan(batch);
+ test_Set_and_Get(batch);
+ test_Flip(batch);
+ test_Flip_Block_ascending(batch);
+ test_Flip_Block_descending(batch);
+ test_Flip_Block_bulk(batch);
+ test_Mimic(batch);
+ test_Or(batch);
+ test_Xor(batch);
+ test_And(batch);
+ test_And_Not(batch);
+ test_Count(batch);
+ test_Next_Hit(batch);
+ test_Clear_All(batch);
+ test_Clone(batch);
+ test_To_Array(batch);
+ test_off_by_one_error();
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestBitVector.cfh b/core/Lucy/Test/Object/TestBitVector.cfh
new file mode 100644
index 0000000..75a1222
--- /dev/null
+++ b/core/Lucy/Test/Object/TestBitVector.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestBitVector {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestByteBuf.c b/core/Lucy/Test/Object/TestByteBuf.c
new file mode 100644
index 0000000..c627262
--- /dev/null
+++ b/core/Lucy/Test/Object/TestByteBuf.c
@@ -0,0 +1,160 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTBYTEBUF
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestByteBuf.h"
+
+static void
+test_Equals(TestBatch *batch) {
+ ByteBuf *wanted = BB_new_bytes("foo", 4); // Include terminating NULL.
+ ByteBuf *got = BB_new_bytes("foo", 4);
+
+ TEST_TRUE(batch, BB_Equals(wanted, (Obj*)got), "Equals");
+ TEST_INT_EQ(batch, BB_Hash_Sum(got), BB_Hash_Sum(wanted), "Hash_Sum");
+
+ TEST_TRUE(batch, BB_Equals_Bytes(got, "foo", 4), "Equals_Bytes");
+ TEST_FALSE(batch, BB_Equals_Bytes(got, "foo", 3),
+ "Equals_Bytes spoiled by different size");
+ TEST_FALSE(batch, BB_Equals_Bytes(got, "bar", 4),
+ "Equals_Bytes spoiled by different content");
+
+ BB_Set_Size(got, 3);
+ TEST_FALSE(batch, BB_Equals(wanted, (Obj*)got),
+ "Different size spoils Equals");
+ TEST_FALSE(batch, BB_Hash_Sum(got) == BB_Hash_Sum(wanted),
+ "Different size spoils Hash_Sum (probably -- at least this one)");
+
+ BB_Mimic_Bytes(got, "bar", 4);
+ TEST_INT_EQ(batch, BB_Get_Size(wanted), BB_Get_Size(got),
+ "same length");
+ TEST_FALSE(batch, BB_Equals(wanted, (Obj*)got),
+ "Different content spoils Equals");
+
+ DECREF(got);
+ DECREF(wanted);
+}
+
+static void
+test_Grow(TestBatch *batch) {
+ ByteBuf *bb = BB_new(1);
+ TEST_INT_EQ(batch, BB_Get_Capacity(bb), 8,
+ "Allocate in 8-byte increments");
+ BB_Grow(bb, 9);
+ TEST_INT_EQ(batch, BB_Get_Capacity(bb), 16,
+ "Grow in 8-byte increments");
+ DECREF(bb);
+}
+
+static void
+test_Clone(TestBatch *batch) {
+ ByteBuf *bb = BB_new_bytes("foo", 3);
+ ByteBuf *twin = BB_Clone(bb);
+ TEST_TRUE(batch, BB_Equals(bb, (Obj*)twin), "Clone");
+ DECREF(bb);
+ DECREF(twin);
+}
+
+static void
+test_compare(TestBatch *batch) {
+ ByteBuf *a = BB_new_bytes("foo\0a", 5);
+ ByteBuf *b = BB_new_bytes("foo\0b", 5);
+
+ BB_Set_Size(a, 4);
+ BB_Set_Size(b, 4);
+ TEST_INT_EQ(batch, BB_compare(&a, &b), 0,
+ "BB_compare returns 0 for equal ByteBufs");
+
+ BB_Set_Size(a, 3);
+ TEST_TRUE(batch, BB_compare(&a, &b) < 0, "shorter ByteBuf sorts first");
+
+ BB_Set_Size(a, 5);
+ BB_Set_Size(b, 5);
+ TEST_TRUE(batch, BB_compare(&a, &b) < 0,
+ "NULL doesn't interfere with BB_compare");
+
+ DECREF(a);
+ DECREF(b);
+}
+
+static void
+test_Mimic(TestBatch *batch) {
+ ByteBuf *a = BB_new_bytes("foo", 3);
+ ByteBuf *b = BB_new(0);
+
+ BB_Mimic(b, (Obj*)a);
+ TEST_TRUE(batch, BB_Equals(a, (Obj*)b), "Mimic");
+
+ BB_Mimic_Bytes(a, "bar", 4);
+ TEST_TRUE(batch, strcmp(BB_Get_Buf(a), "bar") == 0,
+ "Mimic_Bytes content");
+ TEST_INT_EQ(batch, BB_Get_Size(a), 4, "Mimic_Bytes size");
+
+ BB_Mimic(b, (Obj*)a);
+ TEST_TRUE(batch, BB_Equals(a, (Obj*)b), "Mimic");
+
+ DECREF(a);
+ DECREF(b);
+}
+
+static void
+test_Cat(TestBatch *batch) {
+ ByteBuf *wanted = BB_new_bytes("foobar", 6);
+ ByteBuf *got = BB_new_bytes("foo", 3);
+ ByteBuf *scratch = BB_new_bytes("bar", 3);
+
+ BB_Cat(got, scratch);
+ TEST_TRUE(batch, BB_Equals(wanted, (Obj*)got), "Cat");
+
+ BB_Mimic_Bytes(wanted, "foobarbaz", 9);
+ BB_Cat_Bytes(got, "baz", 3);
+ TEST_TRUE(batch, BB_Equals(wanted, (Obj*)got), "Cat_Bytes");
+
+ DECREF(scratch);
+ DECREF(got);
+ DECREF(wanted);
+}
+
+static void
+test_serialization(TestBatch *batch) {
+ ByteBuf *wanted = BB_new_bytes("foobar", 6);
+ ByteBuf *got = (ByteBuf*)TestUtils_freeze_thaw((Obj*)wanted);
+ TEST_TRUE(batch, got && BB_Equals(wanted, (Obj*)got),
+ "Serialization round trip");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+void
+TestBB_run_tests() {
+ TestBatch *batch = TestBatch_new(22);
+ TestBatch_Plan(batch);
+
+ test_Equals(batch);
+ test_Grow(batch);
+ test_Clone(batch);
+ test_compare(batch);
+ test_Mimic(batch);
+ test_Cat(batch);
+ test_serialization(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestByteBuf.cfh b/core/Lucy/Test/Object/TestByteBuf.cfh
new file mode 100644
index 0000000..42ab933
--- /dev/null
+++ b/core/Lucy/Test/Object/TestByteBuf.cfh
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+parcel Lucy;
+
+class Lucy::Test::Object::TestByteBuf cnick TestBB
+ inherits Lucy::Object::Obj {
+
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestCharBuf.c b/core/Lucy/Test/Object/TestCharBuf.c
new file mode 100644
index 0000000..02c6c75
--- /dev/null
+++ b/core/Lucy/Test/Object/TestCharBuf.c
@@ -0,0 +1,429 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTCHARBUF
+#include "Lucy/Util/ToolSet.h"
+#include <stdarg.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestCharBuf.h"
+
+static char smiley[] = { (char)0xE2, (char)0x98, (char)0xBA, 0 };
+static uint32_t smiley_len = 3;
+
+static CharBuf*
+S_get_cb(char *string) {
+ return CB_new_from_utf8(string, strlen(string));
+}
+
+static void
+test_Cat(TestBatch *batch) {
+ CharBuf *wanted = CB_newf("a%s", smiley);
+ CharBuf *got = S_get_cb("");
+
+ CB_Cat(got, wanted);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Cat");
+ DECREF(got);
+
+ got = S_get_cb("a");
+ CB_Cat_Char(got, 0x263A);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Cat_Char");
+ DECREF(got);
+
+ got = S_get_cb("a");
+ CB_Cat_Str(got, smiley, smiley_len);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Cat_Str");
+ DECREF(got);
+
+ got = S_get_cb("a");
+ CB_Cat_Trusted_Str(got, smiley, smiley_len);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Cat_Trusted_Str");
+ DECREF(got);
+
+ DECREF(wanted);
+}
+
+static void
+test_Mimic_and_Clone(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo");
+ CharBuf *got = S_get_cb("bar");
+
+ CB_Mimic(got, (Obj*)wanted);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Mimic");
+ DECREF(got);
+
+ got = S_get_cb("bar");
+ CB_Mimic_Str(got, "foo", 3);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Mimic_Str");
+ DECREF(got);
+
+ got = CB_Clone(wanted);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Clone");
+ DECREF(got);
+
+ DECREF(wanted);
+}
+
+static void
+test_Find(TestBatch *batch) {
+ CharBuf *string = CB_new(10);
+ CharBuf *substring = S_get_cb("foo");
+
+ TEST_TRUE(batch, CB_Find(string, substring) == -1, "Not in empty string");
+ CB_setf(string, "foo");
+ TEST_TRUE(batch, CB_Find(string, substring) == 0, "Find complete string");
+ CB_setf(string, "afoo");
+ TEST_TRUE(batch, CB_Find(string, substring) == 1, "Find after first");
+ CB_Set_Size(string, 3);
+ TEST_TRUE(batch, CB_Find(string, substring) == -1, "Don't overrun");
+ CB_setf(string, "afood");
+ TEST_TRUE(batch, CB_Find(string, substring) == 1, "Find in middle");
+
+ DECREF(substring);
+ DECREF(string);
+}
+
+static void
+test_Code_Point_At_and_From(TestBatch *batch) {
+ uint32_t code_points[] = { 'a', 0x263A, 0x263A, 'b', 0x263A, 'c' };
+ uint32_t num_code_points = sizeof(code_points) / sizeof(uint32_t);
+ CharBuf *string = CB_newf("a%s%sb%sc", smiley, smiley, smiley);
+ uint32_t i;
+
+ for (i = 0; i < num_code_points; i++) {
+ uint32_t from = num_code_points - i - 1;
+ TEST_INT_EQ(batch, CB_Code_Point_At(string, i), code_points[i],
+ "Code_Point_At %ld", (long)i);
+ TEST_INT_EQ(batch, CB_Code_Point_At(string, from),
+ code_points[from], "Code_Point_From %ld", (long)from);
+ }
+
+ DECREF(string);
+}
+
+static void
+test_SubString(TestBatch *batch) {
+ CharBuf *string = CB_newf("a%s%sb%sc", smiley, smiley, smiley);
+ CharBuf *wanted = CB_newf("%sb%s", smiley, smiley);
+ CharBuf *got = CB_SubString(string, 2, 3);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "SubString");
+ DECREF(wanted);
+ DECREF(got);
+ DECREF(string);
+}
+
+static void
+test_Nip_and_Chop(TestBatch *batch) {
+ CharBuf *wanted;
+ CharBuf *got;
+
+ wanted = CB_newf("%sb%sc", smiley, smiley);
+ got = CB_newf("a%s%sb%sc", smiley, smiley, smiley);
+ CB_Nip(got, 2);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Nip");
+ DECREF(wanted);
+ DECREF(got);
+
+ wanted = CB_newf("a%s%s", smiley, smiley);
+ got = CB_newf("a%s%sb%sc", smiley, smiley, smiley);
+ CB_Chop(got, 3);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Chop");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+
+static void
+test_Truncate(TestBatch *batch) {
+ CharBuf *wanted = CB_newf("a%s", smiley, smiley);
+ CharBuf *got = CB_newf("a%s%sb%sc", smiley, smiley, smiley);
+ CB_Truncate(got, 2);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "Truncate");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_Trim(TestBatch *batch) {
+ uint32_t spaces[] = {
+ ' ', '\t', '\r', '\n', 0x000B, 0x000C, 0x000D, 0x0085,
+ 0x00A0, 0x1680, 0x180E, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004,
+ 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x2028, 0x2029,
+ 0x202F, 0x205F, 0x3000
+ };
+ uint32_t num_spaces = sizeof(spaces) / sizeof(uint32_t);
+ uint32_t i;
+ CharBuf *got = CB_new(0);
+
+ // Surround a smiley with lots of whitespace.
+ for (i = 0; i < num_spaces; i++) { CB_Cat_Char(got, spaces[i]); }
+ CB_Cat_Char(got, 0x263A);
+ for (i = 0; i < num_spaces; i++) { CB_Cat_Char(got, spaces[i]); }
+
+ TEST_TRUE(batch, CB_Trim_Top(got), "Trim_Top returns true on success");
+ TEST_FALSE(batch, CB_Trim_Top(got),
+ "Trim_Top returns false on failure");
+ TEST_TRUE(batch, CB_Trim_Tail(got), "Trim_Tail returns true on success");
+ TEST_FALSE(batch, CB_Trim_Tail(got),
+ "Trim_Tail returns false on failure");
+ TEST_TRUE(batch, CB_Equals_Str(got, smiley, smiley_len),
+ "Trim_Top and Trim_Tail worked");
+
+ // Build the spacey smiley again.
+ CB_Truncate(got, 0);
+ for (i = 0; i < num_spaces; i++) { CB_Cat_Char(got, spaces[i]); }
+ CB_Cat_Char(got, 0x263A);
+ for (i = 0; i < num_spaces; i++) { CB_Cat_Char(got, spaces[i]); }
+
+ TEST_TRUE(batch, CB_Trim(got), "Trim returns true on success");
+ TEST_FALSE(batch, CB_Trim(got), "Trim returns false on failure");
+ TEST_TRUE(batch, CB_Equals_Str(got, smiley, smiley_len),
+ "Trim worked");
+
+ DECREF(got);
+}
+
+static void
+test_To_F64(TestBatch *batch) {
+ CharBuf *charbuf = S_get_cb("1.5");
+ double difference = 1.5 - CB_To_F64(charbuf);
+ if (difference < 0) { difference = 0 - difference; }
+ TEST_TRUE(batch, difference < 0.001, "To_F64");
+
+ CB_setf(charbuf, "-1.5");
+ difference = 1.5 + CB_To_F64(charbuf);
+ if (difference < 0) { difference = 0 - difference; }
+ TEST_TRUE(batch, difference < 0.001, "To_F64 negative");
+
+ CB_setf(charbuf, "1.59");
+ double value_full = CB_To_F64(charbuf);
+ CB_Set_Size(charbuf, 3);
+ double value_short = CB_To_F64(charbuf);
+ TEST_TRUE(batch, value_short < value_full,
+ "TO_F64 doesn't run past end of string");
+
+ DECREF(charbuf);
+}
+
+static void
+test_To_I64(TestBatch *batch) {
+ CharBuf *charbuf = S_get_cb("10");
+ TEST_TRUE(batch, CB_To_I64(charbuf) == 10, "To_I64");
+ CB_setf(charbuf, "-10");
+ TEST_TRUE(batch, CB_To_I64(charbuf) == -10, "To_I64 negative");
+ DECREF(charbuf);
+}
+
+
+static void
+test_vcatf_s(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar bizzle baz");
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %s baz", "bizzle");
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%s");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_null_string(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar [NULL] baz");
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %s baz", NULL);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%s NULL");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_cb(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar ZEKE baz");
+ CharBuf *catworthy = S_get_cb("ZEKE");
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %o baz", catworthy);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%o CharBuf");
+ DECREF(catworthy);
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_obj(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("ooga 20 booga");
+ Integer32 *i32 = Int32_new(20);
+ CharBuf *got = S_get_cb("ooga");
+ CB_catf(got, " %o booga", i32);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%o Obj");
+ DECREF(i32);
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_null_obj(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar [NULL] baz");
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %o baz", NULL);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%o NULL");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_i8(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar -3 baz");
+ int8_t num = -3;
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %i8 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%i8");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_i32(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar -100000 baz");
+ int32_t num = -100000;
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %i32 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%i32");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_i64(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar -5000000000 baz");
+ int64_t num = I64_C(-5000000000);
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %i64 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%i64");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_u8(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar 3 baz");
+ uint8_t num = 3;
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %u8 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%u8");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_u32(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar 100000 baz");
+ uint32_t num = 100000;
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %u32 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%u32");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_u64(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo bar 5000000000 baz");
+ uint64_t num = U64_C(5000000000);
+ CharBuf *got = S_get_cb("foo ");
+ CB_catf(got, "bar %u64 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%u64");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_f64(TestBatch *batch) {
+ CharBuf *wanted;
+ char buf[64];
+ float num = 1.3f;
+ CharBuf *got = S_get_cb("foo ");
+ sprintf(buf, "foo bar %g baz", num);
+ wanted = CB_new_from_trusted_utf8(buf, strlen(buf));
+ CB_catf(got, "bar %f64 baz", num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%f64");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_vcatf_x32(TestBatch *batch) {
+ CharBuf *wanted;
+ char buf[64];
+ unsigned long num = I32_MAX;
+ CharBuf *got = S_get_cb("foo ");
+#if (SIZEOF_LONG == 4)
+ sprintf(buf, "foo bar %.8lx baz", num);
+#elif (SIZEOF_INT == 4)
+ sprintf(buf, "foo bar %.8x baz", (unsigned)num);
+#endif
+ wanted = CB_new_from_trusted_utf8(buf, strlen(buf));
+ CB_catf(got, "bar %x32 baz", (uint32_t)num);
+ TEST_TRUE(batch, CB_Equals(wanted, (Obj*)got), "%%x32");
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_serialization(TestBatch *batch) {
+ CharBuf *wanted = S_get_cb("foo");
+ CharBuf *got = (CharBuf*)TestUtils_freeze_thaw((Obj*)wanted);
+ TEST_TRUE(batch, got && CB_Equals(wanted, (Obj*)got),
+ "Round trip through FREEZE/THAW");
+ DECREF(got);
+ DECREF(wanted);
+}
+
+void
+TestCB_run_tests() {
+ TestBatch *batch = TestBatch_new(55);
+ TestBatch_Plan(batch);
+
+ test_vcatf_s(batch);
+ test_vcatf_null_string(batch);
+ test_vcatf_cb(batch);
+ test_vcatf_obj(batch);
+ test_vcatf_null_obj(batch);
+ test_vcatf_i8(batch);
+ test_vcatf_i32(batch);
+ test_vcatf_i64(batch);
+ test_vcatf_u8(batch);
+ test_vcatf_u32(batch);
+ test_vcatf_u64(batch);
+ test_vcatf_f64(batch);
+ test_vcatf_x32(batch);
+ test_Cat(batch);
+ test_Mimic_and_Clone(batch);
+ test_Code_Point_At_and_From(batch);
+ test_Find(batch);
+ test_SubString(batch);
+ test_Nip_and_Chop(batch);
+ test_Truncate(batch);
+ test_Trim(batch);
+ test_To_F64(batch);
+ test_To_I64(batch);
+ test_serialization(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestCharBuf.cfh b/core/Lucy/Test/Object/TestCharBuf.cfh
new file mode 100644
index 0000000..7865df7
--- /dev/null
+++ b/core/Lucy/Test/Object/TestCharBuf.cfh
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+parcel Lucy;
+
+class Lucy::Test::Object::TestCharBuf cnick TestCB
+ inherits Lucy::Object::Obj {
+
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestHash.c b/core/Lucy/Test/Object/TestHash.c
new file mode 100644
index 0000000..687f19e
--- /dev/null
+++ b/core/Lucy/Test/Object/TestHash.c
@@ -0,0 +1,308 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+#include <stdlib.h>
+#include <time.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestHash.h"
+#include "Lucy/Object/Hash.h"
+
+static CharBuf*
+S_random_string() {
+ uint32_t len = rand() % 1200;
+ CharBuf *string = CB_new(len * 3);
+
+ while (len--) {
+ uint8_t bytes = (rand() % 3) + 1;
+ uint32_t code_point = 0;
+ switch (bytes & 0x3) {
+ case 1:
+ code_point = rand() % 0x80;
+ break;
+ case 2:
+ code_point = (rand() % (0x0800 - 0x0080)) + 0x0080;
+ break;
+ case 3:
+ code_point = (rand() % (0x10000 - 0x0800)) + 0x0800;
+ break;
+ }
+ CB_Cat_Char(string, code_point);
+ }
+
+ return string;
+}
+
+static void
+test_Equals(TestBatch *batch) {
+ Hash *hash = Hash_new(0);
+ Hash *other = Hash_new(0);
+ ZombieCharBuf *stuff = ZCB_WRAP_STR("stuff", 5);
+
+ TEST_TRUE(batch, Hash_Equals(hash, (Obj*)other),
+ "Empty hashes are equal");
+
+ Hash_Store_Str(hash, "foo", 3, INCREF(&EMPTY));
+ TEST_FALSE(batch, Hash_Equals(hash, (Obj*)other),
+ "Add one pair and Equals returns false");
+
+ Hash_Store_Str(other, "foo", 3, INCREF(&EMPTY));
+ TEST_TRUE(batch, Hash_Equals(hash, (Obj*)other),
+ "Add a matching pair and Equals returns true");
+
+ Hash_Store_Str(other, "foo", 3, INCREF(stuff));
+ TEST_FALSE(batch, Hash_Equals(hash, (Obj*)other),
+ "Non-matching value spoils Equals");
+
+ DECREF(hash);
+ DECREF(other);
+}
+
+static void
+test_Store_and_Fetch(TestBatch *batch) {
+ Hash *hash = Hash_new(100);
+ Hash *dupe = Hash_new(100);
+ const uint32_t starting_cap = Hash_Get_Capacity(hash);
+ VArray *expected = VA_new(100);
+ VArray *got = VA_new(100);
+ ZombieCharBuf *twenty = ZCB_WRAP_STR("20", 2);
+ ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2);
+ ZombieCharBuf *foo = ZCB_WRAP_STR("foo", 3);
+ int32_t i;
+
+ for (i = 0; i < 100; i++) {
+ CharBuf *cb = CB_newf("%i32", i);
+ Hash_Store(hash, (Obj*)cb, (Obj*)cb);
+ Hash_Store(dupe, (Obj*)cb, INCREF(cb));
+ VA_Push(expected, INCREF(cb));
+ }
+ TEST_TRUE(batch, Hash_Equals(hash, (Obj*)dupe), "Equals");
+
+ TEST_INT_EQ(batch, Hash_Get_Capacity(hash), starting_cap,
+ "Initial capacity sufficient (no rebuilds)");
+
+ for (i = 0; i < 100; i++) {
+ Obj *key = VA_Fetch(expected, i);
+ Obj *elem = Hash_Fetch(hash, key);
+ VA_Push(got, (Obj*)INCREF(elem));
+ }
+
+ TEST_TRUE(batch, VA_Equals(got, (Obj*)expected),
+ "basic Store and Fetch");
+ TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
+ "size incremented properly by Hash_Store");
+
+ TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)foo) == NULL,
+ "Fetch against non-existent key returns NULL");
+
+ Hash_Store(hash, (Obj*)forty, INCREF(foo));
+ TEST_TRUE(batch, ZCB_Equals(foo, Hash_Fetch(hash, (Obj*)forty)),
+ "Hash_Store replaces existing value");
+ TEST_FALSE(batch, Hash_Equals(hash, (Obj*)dupe),
+ "replacement value spoils equals");
+ TEST_INT_EQ(batch, Hash_Get_Size(hash), 100,
+ "size unaffected after value replaced");
+
+ TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == (Obj*)foo,
+ "Delete returns value");
+ DECREF(foo);
+ TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
+ "size decremented by successful Delete");
+ TEST_TRUE(batch, Hash_Delete(hash, (Obj*)forty) == NULL,
+ "Delete returns NULL when key not found");
+ TEST_INT_EQ(batch, Hash_Get_Size(hash), 99,
+ "size not decremented by unsuccessful Delete");
+ DECREF(Hash_Delete(dupe, (Obj*)forty));
+ TEST_TRUE(batch, VA_Equals(got, (Obj*)expected), "Equals after Delete");
+
+ Hash_Clear(hash);
+ TEST_TRUE(batch, Hash_Fetch(hash, (Obj*)twenty) == NULL, "Clear");
+ TEST_TRUE(batch, Hash_Get_Size(hash) == 0, "size is 0 after Clear");
+
+ DECREF(hash);
+ DECREF(dupe);
+ DECREF(got);
+ DECREF(expected);
+}
+
+static void
+test_Keys_Values_Iter(TestBatch *batch) {
+ uint32_t i;
+ Hash *hash = Hash_new(0); // trigger multiple rebuilds.
+ VArray *expected = VA_new(100);
+ VArray *keys;
+ VArray *values;
+
+ for (i = 0; i < 500; i++) {
+ CharBuf *cb = CB_newf("%u32", i);
+ Hash_Store(hash, (Obj*)cb, (Obj*)cb);
+ VA_Push(expected, INCREF(cb));
+ }
+
+ VA_Sort(expected, NULL, NULL);
+
+ keys = Hash_Keys(hash);
+ values = Hash_Values(hash);
+ VA_Sort(keys, NULL, NULL);
+ VA_Sort(values, NULL, NULL);
+ TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys");
+ TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values");
+ VA_Clear(keys);
+ VA_Clear(values);
+
+ {
+ Obj *key;
+ Obj *value;
+ Hash_Iterate(hash);
+ while (Hash_Next(hash, &key, &value)) {
+ VA_Push(keys, INCREF(key));
+ VA_Push(values, INCREF(value));
+ }
+ }
+
+ VA_Sort(keys, NULL, NULL);
+ VA_Sort(values, NULL, NULL);
+ TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "Keys from Iter");
+ TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "Values from Iter");
+
+ {
+ ZombieCharBuf *forty = ZCB_WRAP_STR("40", 2);
+ ZombieCharBuf *nope = ZCB_WRAP_STR("nope", 4);
+ Obj *key = Hash_Find_Key(hash, (Obj*)forty, ZCB_Hash_Sum(forty));
+ TEST_TRUE(batch, Obj_Equals(key, (Obj*)forty), "Find_Key");
+ key = Hash_Find_Key(hash, (Obj*)nope, ZCB_Hash_Sum(nope)),
+ TEST_TRUE(batch, key == NULL,
+ "Find_Key returns NULL for non-existent key");
+ }
+
+ DECREF(hash);
+ DECREF(expected);
+ DECREF(keys);
+ DECREF(values);
+}
+
+static void
+test_Dump_and_Load(TestBatch *batch) {
+ Hash *hash = Hash_new(0);
+ Obj *dump;
+ Hash *loaded;
+
+ Hash_Store_Str(hash, "foo", 3,
+ (Obj*)CB_new_from_trusted_utf8("foo", 3));
+ dump = (Obj*)Hash_Dump(hash);
+ loaded = (Hash*)Obj_Load(dump, dump);
+ TEST_TRUE(batch, Hash_Equals(hash, (Obj*)loaded),
+ "Dump => Load round trip");
+ DECREF(dump);
+ DECREF(loaded);
+
+ /* TODO: Fix Hash_Load().
+
+ Hash_Store_Str(hash, "_class", 6,
+ (Obj*)CB_new_from_trusted_utf8("not_a_class", 11));
+ dump = (Obj*)Hash_Dump(hash);
+ loaded = (Hash*)Obj_Load(dump, dump);
+
+ TEST_TRUE(batch, Hash_Equals(hash, (Obj*)loaded),
+ "Load still works with _class if it's not a real class");
+ DECREF(dump);
+ DECREF(loaded);
+
+ */
+
+ DECREF(hash);
+}
+
+static void
+test_serialization(TestBatch *batch) {
+ Hash *wanted = Hash_new(0);
+ Hash *got;
+ uint32_t i;
+
+ for (i = 0; i < 10; i++) {
+ CharBuf *cb = S_random_string();
+ Integer32 *num = Int32_new(i);
+ Hash_Store(wanted, (Obj*)cb, (Obj*)num);
+ Hash_Store(wanted, (Obj*)num, (Obj*)cb);
+ }
+
+ got = (Hash*)TestUtils_freeze_thaw((Obj*)wanted);
+ TEST_TRUE(batch, got && Hash_Equals(wanted, (Obj*)got),
+ "Round trip through serialization.");
+
+ DECREF(got);
+ DECREF(wanted);
+}
+
+static void
+test_stress(TestBatch *batch) {
+ uint32_t i;
+ Hash *hash = Hash_new(0); // trigger multiple rebuilds.
+ VArray *expected = VA_new(1000);
+ VArray *keys;
+ VArray *values;
+
+ for (i = 0; i < 1000; i++) {
+ CharBuf *cb = S_random_string();
+ while (Hash_Fetch(hash, (Obj*)cb)) {
+ DECREF(cb);
+ cb = S_random_string();
+ }
+ Hash_Store(hash, (Obj*)cb, (Obj*)cb);
+ VA_Push(expected, INCREF(cb));
+ }
+
+ VA_Sort(expected, NULL, NULL);
+
+ // Overwrite for good measure.
+ for (i = 0; i < 1000; i++) {
+ CharBuf *cb = (CharBuf*)VA_Fetch(expected, i);
+ Hash_Store(hash, (Obj*)cb, INCREF(cb));
+ }
+
+ keys = Hash_Keys(hash);
+ values = Hash_Values(hash);
+ VA_Sort(keys, NULL, NULL);
+ VA_Sort(values, NULL, NULL);
+ TEST_TRUE(batch, VA_Equals(keys, (Obj*)expected), "stress Keys");
+ TEST_TRUE(batch, VA_Equals(values, (Obj*)expected), "stress Values");
+
+ DECREF(keys);
+ DECREF(values);
+ DECREF(expected);
+ DECREF(hash);
+}
+
+void
+TestHash_run_tests() {
+ TestBatch *batch = TestBatch_new(29);
+
+ srand((unsigned int)time((time_t*)NULL));
+
+ TestBatch_Plan(batch);
+ test_Equals(batch);
+ test_Store_and_Fetch(batch);
+ test_Keys_Values_Iter(batch);
+ test_Dump_and_Load(batch);
+ test_serialization(batch);
+ test_stress(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestHash.cfh b/core/Lucy/Test/Object/TestHash.cfh
new file mode 100644
index 0000000..c3cf509
--- /dev/null
+++ b/core/Lucy/Test/Object/TestHash.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestHash {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestI32Array.c b/core/Lucy/Test/Object/TestI32Array.c
new file mode 100644
index 0000000..367d2f3
--- /dev/null
+++ b/core/Lucy/Test/Object/TestI32Array.c
@@ -0,0 +1,70 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTI32ARRAY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Object/TestI32Array.h"
+
+static int32_t source_ints[] = { -1, 0, I32_MIN, I32_MAX, 1 };
+static size_t num_ints = sizeof(source_ints) / sizeof(int32_t);
+
+static void
+test_all(TestBatch *batch) {
+ I32Array *i32_array = I32Arr_new(source_ints, num_ints);
+ int32_t *ints_copy = (int32_t*)malloc(num_ints * sizeof(int32_t));
+ I32Array *stolen = I32Arr_new_steal(ints_copy, num_ints);
+ size_t num_matched;
+
+ memcpy(ints_copy, source_ints, num_ints * sizeof(int32_t));
+
+ TEST_TRUE(batch, I32Arr_Get_Size(i32_array) == num_ints,
+ "Get_Size");
+ TEST_TRUE(batch, I32Arr_Get_Size(stolen) == num_ints,
+ "Get_Size for stolen");
+
+ for (num_matched = 0; num_matched < num_ints; num_matched++) {
+ if (source_ints[num_matched] != I32Arr_Get(i32_array, num_matched)) {
+ break;
+ }
+ }
+ TEST_INT_EQ(batch, num_matched, num_ints,
+ "Matched all source ints with Get()");
+
+ for (num_matched = 0; num_matched < num_ints; num_matched++) {
+ if (source_ints[num_matched] != I32Arr_Get(stolen, num_matched)) {
+ break;
+ }
+ }
+ TEST_INT_EQ(batch, num_matched, num_ints,
+ "Matched all source ints in stolen I32Array with Get()");
+
+ DECREF(i32_array);
+ DECREF(stolen);
+}
+
+void
+TestI32Arr_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+
+ TestBatch_Plan(batch);
+ test_all(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestI32Array.cfh b/core/Lucy/Test/Object/TestI32Array.cfh
new file mode 100644
index 0000000..2d97741
--- /dev/null
+++ b/core/Lucy/Test/Object/TestI32Array.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestI32Array cnick TestI32Arr {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestLockFreeRegistry.c b/core/Lucy/Test/Object/TestLockFreeRegistry.c
new file mode 100644
index 0000000..6a556bc
--- /dev/null
+++ b/core/Lucy/Test/Object/TestLockFreeRegistry.c
@@ -0,0 +1,78 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestLockFreeRegistry.h"
+#include "Lucy/Object/LockFreeRegistry.h"
+
+StupidHashCharBuf*
+StupidHashCharBuf_new(char *text) {
+ return (StupidHashCharBuf*)CB_new_from_utf8(text, strlen(text));
+}
+
+int32_t
+StupidHashCharBuf_hash_sum(StupidHashCharBuf *self) {
+ UNUSED_VAR(self);
+ return 1;
+}
+
+static void
+test_all(TestBatch *batch) {
+ LockFreeRegistry *registry = LFReg_new(10);
+ StupidHashCharBuf *foo = StupidHashCharBuf_new("foo");
+ StupidHashCharBuf *bar = StupidHashCharBuf_new("bar");
+ StupidHashCharBuf *baz = StupidHashCharBuf_new("baz");
+ StupidHashCharBuf *foo_dupe = StupidHashCharBuf_new("foo");
+
+ TEST_TRUE(batch, LFReg_Register(registry, (Obj*)foo, (Obj*)foo),
+ "Register() returns true on success");
+ TEST_FALSE(batch,
+ LFReg_Register(registry, (Obj*)foo_dupe, (Obj*)foo_dupe),
+ "Can't Register() keys that test equal");
+
+ TEST_TRUE(batch, LFReg_Register(registry, (Obj*)bar, (Obj*)bar),
+ "Register() key with the same Hash_Sum but that isn't Equal");
+
+ TEST_TRUE(batch, LFReg_Fetch(registry, (Obj*)foo_dupe) == (Obj*)foo,
+ "Fetch()");
+ TEST_TRUE(batch, LFReg_Fetch(registry, (Obj*)bar) == (Obj*)bar,
+ "Fetch() again");
+ TEST_TRUE(batch, LFReg_Fetch(registry, (Obj*)baz) == NULL,
+ "Fetch() non-existent key returns NULL");
+
+ DECREF(foo_dupe);
+ DECREF(baz);
+ DECREF(bar);
+ DECREF(foo);
+ DECREF(registry);
+}
+
+void
+TestLFReg_run_tests() {
+ TestBatch *batch = TestBatch_new(6);
+
+ TestBatch_Plan(batch);
+ test_all(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestLockFreeRegistry.cfh b/core/Lucy/Test/Object/TestLockFreeRegistry.cfh
new file mode 100644
index 0000000..9fd8f07
--- /dev/null
+++ b/core/Lucy/Test/Object/TestLockFreeRegistry.cfh
@@ -0,0 +1,35 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestLockFreeRegistry cnick TestLFReg {
+ inert void
+ run_tests();
+}
+
+/** Private test-only class for stressing LockFreeRegistry.
+ */
+class Lucy::Test::Object::StupidHashCharBuf inherits Lucy::Object::CharBuf {
+ inert incremented StupidHashCharBuf*
+ new(char *text);
+
+ /** Always returns 1, guaranteeing collisions. */
+ public int32_t
+ Hash_Sum(StupidHashCharBuf *self);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestNum.c b/core/Lucy/Test/Object/TestNum.c
new file mode 100644
index 0000000..3e380f8
--- /dev/null
+++ b/core/Lucy/Test/Object/TestNum.c
@@ -0,0 +1,270 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTNUM
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestNum.h"
+
+static void
+test_To_String(TestBatch *batch) {
+ Float32 *f32 = Float32_new(1.33f);
+ Float64 *f64 = Float64_new(1.33);
+ Integer32 *i32 = Int32_new(I32_MAX);
+ Integer64 *i64 = Int64_new(I64_MAX);
+ CharBuf *f32_string = Float32_To_String(f32);
+ CharBuf *f64_string = Float64_To_String(f64);
+ CharBuf *i32_string = Int32_To_String(i32);
+ CharBuf *i64_string = Int64_To_String(i64);
+
+ TEST_TRUE(batch, CB_Starts_With_Str(f32_string, "1.3", 3),
+ "Float32_To_String");
+ TEST_TRUE(batch, CB_Starts_With_Str(f64_string, "1.3", 3),
+ "Float64_To_String");
+ TEST_TRUE(batch, CB_Equals_Str(i32_string, "2147483647", 10),
+ "Int32_To_String");
+ TEST_TRUE(batch, CB_Equals_Str(i64_string, "9223372036854775807", 19),
+ "Int64_To_String");
+
+ DECREF(i64_string);
+ DECREF(i32_string);
+ DECREF(f64_string);
+ DECREF(f32_string);
+ DECREF(i64);
+ DECREF(i32);
+ DECREF(f64);
+ DECREF(f32);
+}
+
+static void
+test_accessors(TestBatch *batch) {
+ Float32 *f32 = Float32_new(1.0);
+ Float64 *f64 = Float64_new(1.0);
+ Integer32 *i32 = Int32_new(1);
+ Integer64 *i64 = Int64_new(1);
+ float wanted32 = 1.33f;
+ double wanted64 = 1.33;
+ float got32;
+ double got64;
+
+ Float32_Set_Value(f32, 1.33f);
+ TEST_FLOAT_EQ(batch, Float32_Get_Value(f32), 1.33f,
+ "F32 Set_Value Get_Value");
+
+ Float64_Set_Value(f64, 1.33);
+ got64 = Float64_Get_Value(f64);
+ TEST_TRUE(batch, *(int64_t*)&got64 == *(int64_t*)&wanted64,
+ "F64 Set_Value Get_Value");
+
+ TEST_TRUE(batch, Float32_To_I64(f32) == 1, "Float32_To_I64");
+ TEST_TRUE(batch, Float64_To_I64(f64) == 1, "Float64_To_I64");
+
+ got32 = (float)Float32_To_F64(f32);
+ TEST_TRUE(batch, *(int32_t*)&got32 == *(int32_t*)&wanted32,
+ "Float32_To_F64");
+
+ got64 = Float64_To_F64(f64);
+ TEST_TRUE(batch, *(int64_t*)&got64 == *(int64_t*)&wanted64,
+ "Float64_To_F64");
+
+ Int32_Set_Value(i32, I32_MIN);
+ TEST_INT_EQ(batch, Int32_Get_Value(i32), I32_MIN,
+ "I32 Set_Value Get_Value");
+
+ Int64_Set_Value(i64, I64_MIN);
+ TEST_TRUE(batch, Int64_Get_Value(i64) == I64_MIN,
+ "I64 Set_Value Get_Value");
+
+ Int32_Set_Value(i32, -1);
+ Int64_Set_Value(i64, -1);
+ TEST_TRUE(batch, Int32_To_F64(i32) == -1, "Int32_To_F64");
+ TEST_TRUE(batch, Int64_To_F64(i64) == -1, "Int64_To_F64");
+
+ DECREF(i64);
+ DECREF(i32);
+ DECREF(f64);
+ DECREF(f32);
+}
+
+static void
+test_Equals_and_Compare_To(TestBatch *batch) {
+ Float32 *f32 = Float32_new(1.0);
+ Float64 *f64 = Float64_new(1.0);
+ Integer32 *i32 = Int32_new(I32_MAX);
+ Integer64 *i64 = Int64_new(I64_MAX);
+
+ TEST_TRUE(batch, Float32_Compare_To(f32, (Obj*)f64) == 0,
+ "F32_Compare_To equal");
+ TEST_TRUE(batch, Float32_Equals(f32, (Obj*)f64),
+ "F32_Equals equal");
+
+ Float64_Set_Value(f64, 2.0);
+ TEST_TRUE(batch, Float32_Compare_To(f32, (Obj*)f64) < 0,
+ "F32_Compare_To less than");
+ TEST_FALSE(batch, Float32_Equals(f32, (Obj*)f64),
+ "F32_Equals less than");
+
+ Float64_Set_Value(f64, 0.0);
+ TEST_TRUE(batch, Float32_Compare_To(f32, (Obj*)f64) > 0,
+ "F32_Compare_To greater than");
+ TEST_FALSE(batch, Float32_Equals(f32, (Obj*)f64),
+ "F32_Equals greater than");
+
+ Float64_Set_Value(f64, 1.0);
+ Float32_Set_Value(f32, 1.0);
+ TEST_TRUE(batch, Float64_Compare_To(f64, (Obj*)f32) == 0,
+ "F64_Compare_To equal");
+ TEST_TRUE(batch, Float64_Equals(f64, (Obj*)f32),
+ "F64_Equals equal");
+
+ Float32_Set_Value(f32, 2.0);
+ TEST_TRUE(batch, Float64_Compare_To(f64, (Obj*)f32) < 0,
+ "F64_Compare_To less than");
+ TEST_FALSE(batch, Float64_Equals(f64, (Obj*)f32),
+ "F64_Equals less than");
+
+ Float32_Set_Value(f32, 0.0);
+ TEST_TRUE(batch, Float64_Compare_To(f64, (Obj*)f32) > 0,
+ "F64_Compare_To greater than");
+ TEST_FALSE(batch, Float64_Equals(f64, (Obj*)f32),
+ "F64_Equals greater than");
+
+ Float64_Set_Value(f64, I64_MAX * 2.0);
+ TEST_TRUE(batch, Float64_Compare_To(f64, (Obj*)i64) > 0,
+ "Float64 comparison to Integer64");
+ TEST_TRUE(batch, Int64_Compare_To(i64, (Obj*)f64) < 0,
+ "Integer64 comparison to Float64");
+
+ Float32_Set_Value(f32, I32_MAX * 2.0f);
+ TEST_TRUE(batch, Float32_Compare_To(f32, (Obj*)i32) > 0,
+ "Float32 comparison to Integer32");
+ TEST_TRUE(batch, Int32_Compare_To(i32, (Obj*)f32) < 0,
+ "Integer32 comparison to Float32");
+
+ DECREF(i64);
+ DECREF(i32);
+ DECREF(f64);
+ DECREF(f32);
+}
+
+static void
+test_Clone(TestBatch *batch) {
+ Float32 *f32 = Float32_new(1.33f);
+ Float64 *f64 = Float64_new(1.33);
+ Integer32 *i32 = Int32_new(I32_MAX);
+ Integer64 *i64 = Int64_new(I64_MAX);
+ Float32 *f32_dupe = Float32_Clone(f32);
+ Float64 *f64_dupe = Float64_Clone(f64);
+ Integer32 *i32_dupe = Int32_Clone(i32);
+ Integer64 *i64_dupe = Int64_Clone(i64);
+ TEST_TRUE(batch, Float32_Equals(f32, (Obj*)f32_dupe),
+ "Float32 Clone");
+ TEST_TRUE(batch, Float64_Equals(f64, (Obj*)f64_dupe),
+ "Float64 Clone");
+ TEST_TRUE(batch, Int32_Equals(i32, (Obj*)i32_dupe),
+ "Integer32 Clone");
+ TEST_TRUE(batch, Int64_Equals(i64, (Obj*)i64_dupe),
+ "Integer64 Clone");
+ DECREF(i64_dupe);
+ DECREF(i32_dupe);
+ DECREF(f64_dupe);
+ DECREF(f32_dupe);
+ DECREF(i64);
+ DECREF(i32);
+ DECREF(f64);
+ DECREF(f32);
+}
+
+static void
+test_Mimic(TestBatch *batch) {
+ Float32 *f32 = Float32_new(1.33f);
+ Float64 *f64 = Float64_new(1.33);
+ Integer32 *i32 = Int32_new(I32_MAX);
+ Integer64 *i64 = Int64_new(I64_MAX);
+ Float32 *f32_dupe = Float32_new(0.0f);
+ Float64 *f64_dupe = Float64_new(0.0);
+ Integer32 *i32_dupe = Int32_new(0);
+ Integer64 *i64_dupe = Int64_new(0);
+ Float32_Mimic(f32_dupe, (Obj*)f32);
+ Float64_Mimic(f64_dupe, (Obj*)f64);
+ Int32_Mimic(i32_dupe, (Obj*)i32);
+ Int64_Mimic(i64_dupe, (Obj*)i64);
+ TEST_TRUE(batch, Float32_Equals(f32, (Obj*)f32_dupe),
+ "Float32 Mimic");
+ TEST_TRUE(batch, Float64_Equals(f64, (Obj*)f64_dupe),
+ "Float64 Mimic");
+ TEST_TRUE(batch, Int32_Equals(i32, (Obj*)i32_dupe),
+ "Integer32 Mimic");
+ TEST_TRUE(batch, Int64_Equals(i64, (Obj*)i64_dupe),
+ "Integer64 Mimic");
+ DECREF(i64_dupe);
+ DECREF(i32_dupe);
+ DECREF(f64_dupe);
+ DECREF(f32_dupe);
+ DECREF(i64);
+ DECREF(i32);
+ DECREF(f64);
+ DECREF(f32);
+}
+
+static void
+test_serialization(TestBatch *batch) {
+ Float32 *f32 = Float32_new(1.33f);
+ Float64 *f64 = Float64_new(1.33);
+ Integer32 *i32 = Int32_new(-1);
+ Integer64 *i64 = Int64_new(-1);
+ Float32 *f32_thaw = (Float32*)TestUtils_freeze_thaw((Obj*)f32);
+ Float64 *f64_thaw = (Float64*)TestUtils_freeze_thaw((Obj*)f64);
+ Integer32 *i32_thaw = (Integer32*)TestUtils_freeze_thaw((Obj*)i32);
+ Integer64 *i64_thaw = (Integer64*)TestUtils_freeze_thaw((Obj*)i64);
+
+ TEST_TRUE(batch, Float32_Equals(f32, (Obj*)f32_thaw),
+ "Float32 freeze/thaw");
+ TEST_TRUE(batch, Float64_Equals(f64, (Obj*)f64_thaw),
+ "Float64 freeze/thaw");
+ TEST_TRUE(batch, Int32_Equals(i32, (Obj*)i32_thaw),
+ "Integer32 freeze/thaw");
+ TEST_TRUE(batch, Int64_Equals(i64, (Obj*)i64_thaw),
+ "Integer64 freeze/thaw");
+
+ DECREF(i64_thaw);
+ DECREF(i32_thaw);
+ DECREF(f64_thaw);
+ DECREF(f32_thaw);
+ DECREF(i64);
+ DECREF(i32);
+ DECREF(f64);
+ DECREF(f32);
+}
+
+void
+TestNum_run_tests() {
+ TestBatch *batch = TestBatch_new(42);
+ TestBatch_Plan(batch);
+
+ test_To_String(batch);
+ test_accessors(batch);
+ test_Equals_and_Compare_To(batch);
+ test_Clone(batch);
+ test_Mimic(batch);
+ test_serialization(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestNum.cfh b/core/Lucy/Test/Object/TestNum.cfh
new file mode 100644
index 0000000..00bd522
--- /dev/null
+++ b/core/Lucy/Test/Object/TestNum.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestNum {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestObj.c b/core/Lucy/Test/Object/TestObj.c
new file mode 100644
index 0000000..f44ad21
--- /dev/null
+++ b/core/Lucy/Test/Object/TestObj.c
@@ -0,0 +1,133 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTOBJ
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Object/TestObj.h"
+
+static Obj*
+S_new_testobj() {
+ ZombieCharBuf *klass = ZCB_WRAP_STR("TestObj", 7);
+ Obj *obj;
+ VTable *vtable = VTable_fetch_vtable((CharBuf*)klass);
+ if (!vtable) {
+ vtable = VTable_singleton((CharBuf*)klass, OBJ);
+ }
+ obj = VTable_Make_Obj(vtable);
+ return Obj_init(obj);
+}
+
+static void
+test_refcounts(TestBatch *batch) {
+ Obj *obj = S_new_testobj();
+
+ TEST_INT_EQ(batch, Obj_Get_RefCount(obj), 1,
+ "Correct starting refcount");
+
+ Obj_Inc_RefCount(obj);
+ TEST_INT_EQ(batch, Obj_Get_RefCount(obj), 2, "Inc_RefCount");
+
+ Obj_Dec_RefCount(obj);
+ TEST_INT_EQ(batch, Obj_Get_RefCount(obj), 1, "Dec_RefCount");
+
+ DECREF(obj);
+}
+
+static void
+test_To_String(TestBatch *batch) {
+ Obj *testobj = S_new_testobj();
+ CharBuf *string = Obj_To_String(testobj);
+ ZombieCharBuf *temp = ZCB_WRAP(string);
+ while (ZCB_Get_Size(temp)) {
+ if (ZCB_Starts_With_Str(temp, "TestObj", 7)) { break; }
+ ZCB_Nip_One(temp);
+ }
+ TEST_TRUE(batch, ZCB_Starts_With_Str(temp, "TestObj", 7), "To_String");
+ DECREF(string);
+ DECREF(testobj);
+}
+
+static void
+test_Dump(TestBatch *batch) {
+ Obj *testobj = S_new_testobj();
+ CharBuf *string = Obj_To_String(testobj);
+ Obj *dump = Obj_Dump(testobj);
+ TEST_TRUE(batch, Obj_Equals(dump, (Obj*)string),
+ "Default Dump returns To_String");
+ DECREF(dump);
+ DECREF(string);
+ DECREF(testobj);
+}
+
+static void
+test_Equals(TestBatch *batch) {
+ Obj *testobj = S_new_testobj();
+ Obj *other = S_new_testobj();
+
+ TEST_TRUE(batch, Obj_Equals(testobj, testobj),
+ "Equals is true for the same object");
+ TEST_FALSE(batch, Obj_Equals(testobj, other),
+ "Distinct objects are not equal");
+
+ DECREF(testobj);
+ DECREF(other);
+}
+
+static void
+test_Hash_Sum(TestBatch *batch) {
+ Obj *testobj = S_new_testobj();
+ int64_t address64 = PTR_TO_I64(testobj);
+ int32_t address32 = (int32_t)address64;
+ TEST_TRUE(batch, (Obj_Hash_Sum(testobj) == address32),
+ "Hash_Sum uses memory address");
+ DECREF(testobj);
+}
+
+static void
+test_Is_A(TestBatch *batch) {
+ CharBuf *charbuf = CB_new(0);
+ VTable *bb_vtable = CB_Get_VTable(charbuf);
+ CharBuf *klass = CB_Get_Class_Name(charbuf);
+
+ TEST_TRUE(batch, CB_Is_A(charbuf, CHARBUF), "CharBuf Is_A CharBuf.");
+ TEST_TRUE(batch, CB_Is_A(charbuf, OBJ), "CharBuf Is_A Obj.");
+ TEST_TRUE(batch, bb_vtable == CHARBUF, "Get_VTable");
+ TEST_TRUE(batch, CB_Equals(VTable_Get_Name(CHARBUF), (Obj*)klass),
+ "Get_Class_Name");
+
+ DECREF(charbuf);
+}
+
+
+void
+TestObj_run_tests() {
+ TestBatch *batch = TestBatch_new(12);
+
+ TestBatch_Plan(batch);
+
+ test_refcounts(batch);
+ test_To_String(batch);
+ test_Dump(batch);
+ test_Equals(batch);
+ test_Hash_Sum(batch);
+ test_Is_A(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestObj.cfh b/core/Lucy/Test/Object/TestObj.cfh
new file mode 100644
index 0000000..a8c97a8
--- /dev/null
+++ b/core/Lucy/Test/Object/TestObj.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestObj {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Object/TestVArray.c b/core/Lucy/Test/Object/TestVArray.c
new file mode 100644
index 0000000..9374760
--- /dev/null
+++ b/core/Lucy/Test/Object/TestVArray.c
@@ -0,0 +1,292 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTVARRAY
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Object/TestVArray.h"
+
+static CharBuf*
+S_new_cb(const char *text) {
+ return CB_new_from_utf8(text, strlen(text));
+}
+
+static void
+test_Equals(TestBatch *batch) {
+ VArray *array = VA_new(0);
+ VArray *other = VA_new(0);
+ ZombieCharBuf *stuff = ZCB_WRAP_STR("stuff", 5);
+
+ TEST_TRUE(batch, VA_Equals(array, (Obj*)other),
+ "Empty arrays are equal");
+
+ VA_Push(array, INCREF(&EMPTY));
+ TEST_FALSE(batch, VA_Equals(array, (Obj*)other),
+ "Add one elem and Equals returns false");
+
+ VA_Push(other, INCREF(&EMPTY));
+ TEST_TRUE(batch, VA_Equals(array, (Obj*)other),
+ "Add a matching elem and Equals returns true");
+
+ VA_Store(array, 2, INCREF(&EMPTY));
+ TEST_FALSE(batch, VA_Equals(array, (Obj*)other),
+ "Add elem after a NULL and Equals returns false");
+
+ VA_Store(other, 2, INCREF(&EMPTY));
+ TEST_TRUE(batch, VA_Equals(array, (Obj*)other),
+ "Empty elems don't spoil Equals");
+
+ VA_Store(other, 2, INCREF(stuff));
+ TEST_FALSE(batch, VA_Equals(array, (Obj*)other),
+ "Non-matching value spoils Equals");
+
+ VA_Excise(array, 1, 2); // removes empty elems
+ VA_Delete(other, 1); // leaves NULL in place of deleted elem
+ VA_Delete(other, 2);
+ TEST_FALSE(batch, VA_Equals(array, (Obj*)other),
+ "Empty trailing elements spoil Equals");
+
+ DECREF(array);
+ DECREF(other);
+}
+
+static void
+test_Store_Fetch(TestBatch *batch) {
+ VArray *array = VA_new(0);
+ CharBuf *elem;
+
+ TEST_TRUE(batch, VA_Fetch(array, 2) == NULL, "Fetch beyond end");
+
+ VA_Store(array, 2, (Obj*)CB_newf("foo"));
+ elem = (CharBuf*)CERTIFY(VA_Fetch(array, 2), CHARBUF);
+ TEST_INT_EQ(batch, 3, VA_Get_Size(array), "Store updates size");
+ TEST_TRUE(batch, CB_Equals_Str(elem, "foo", 3), "Store");
+
+ INCREF(elem);
+ TEST_INT_EQ(batch, 2, CB_Get_RefCount(elem),
+ "start with refcount of 2");
+ VA_Store(array, 2, (Obj*)CB_newf("bar"));
+ TEST_INT_EQ(batch, 1, CB_Get_RefCount(elem),
+ "Displacing elem via Store updates refcount");
+ DECREF(elem);
+ elem = (CharBuf*)CERTIFY(VA_Fetch(array, 2), CHARBUF);
+ TEST_TRUE(batch, CB_Equals_Str(elem, "bar", 3), "Store displacement");
+
+ DECREF(array);
+}
+
+static void
+test_Push_Pop_Shift_Unshift(TestBatch *batch) {
+ VArray *array = VA_new(0);
+ CharBuf *elem;
+
+ TEST_INT_EQ(batch, VA_Get_Size(array), 0, "size starts at 0");
+ VA_Push(array, (Obj*)CB_newf("a"));
+ VA_Push(array, (Obj*)CB_newf("b"));
+ VA_Push(array, (Obj*)CB_newf("c"));
+
+ TEST_INT_EQ(batch, VA_Get_Size(array), 3, "size after Push");
+ TEST_TRUE(batch, NULL != CERTIFY(VA_Fetch(array, 2), CHARBUF), "Push");
+
+ elem = (CharBuf*)CERTIFY(VA_Shift(array), CHARBUF);
+ TEST_TRUE(batch, CB_Equals_Str(elem, "a", 1), "Shift");
+ TEST_INT_EQ(batch, VA_Get_Size(array), 2, "size after Shift");
+ DECREF(elem);
+
+ elem = (CharBuf*)CERTIFY(VA_Pop(array), CHARBUF);
+ TEST_TRUE(batch, CB_Equals_Str(elem, "c", 1), "Pop");
+ TEST_INT_EQ(batch, VA_Get_Size(array), 1, "size after Pop");
+ DECREF(elem);
+
+ VA_Unshift(array, (Obj*)CB_newf("foo"));
+ elem = (CharBuf*)CERTIFY(VA_Fetch(array, 0), CHARBUF);
+ TEST_TRUE(batch, CB_Equals_Str(elem, "foo", 3), "Unshift");
+ TEST_INT_EQ(batch, VA_Get_Size(array), 2, "size after Shift");
+
+ DECREF(array);
+}
+
+static void
+test_Delete(TestBatch *batch) {
+ VArray *wanted = VA_new(5);
+ VArray *got = VA_new(5);
+ uint32_t i;
+
+ for (i = 0; i < 5; i++) { VA_Push(got, (Obj*)CB_newf("%u32", i)); }
+ VA_Store(wanted, 0, (Obj*)CB_newf("0", i));
+ VA_Store(wanted, 1, (Obj*)CB_newf("1", i));
+ VA_Store(wanted, 4, (Obj*)CB_newf("4", i));
+ DECREF(VA_Delete(got, 2));
+ DECREF(VA_Delete(got, 3));
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got), "Delete");
+
+ DECREF(wanted);
+ DECREF(got);
+}
+
+static void
+test_Resize(TestBatch *batch) {
+ VArray *array = VA_new(3);
+ uint32_t i;
+
+ for (i = 0; i < 2; i++) { VA_Push(array, (Obj*)CB_newf("%u32", i)); }
+ TEST_INT_EQ(batch, VA_Get_Capacity(array), 3, "Start with capacity 3");
+
+ VA_Resize(array, 4);
+ TEST_INT_EQ(batch, VA_Get_Size(array), 4, "Resize up");
+ TEST_INT_EQ(batch, VA_Get_Capacity(array), 4,
+ "Resize changes capacity");
+
+ VA_Resize(array, 2);
+ TEST_INT_EQ(batch, VA_Get_Size(array), 2, "Resize down");
+ TEST_TRUE(batch, VA_Fetch(array, 2) == NULL, "Resize down zaps elem");
+
+ DECREF(array);
+}
+
+static void
+test_Excise(TestBatch *batch) {
+ VArray *wanted = VA_new(5);
+ VArray *got = VA_new(5);
+
+ for (uint32_t i = 0; i < 5; i++) {
+ VA_Push(wanted, (Obj*)CB_newf("%u32", i));
+ VA_Push(got, (Obj*)CB_newf("%u32", i));
+ }
+
+ VA_Excise(got, 7, 1);
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got),
+ "Excise outside of range is no-op");
+
+ VA_Excise(got, 2, 2);
+ DECREF(VA_Delete(wanted, 2));
+ DECREF(VA_Delete(wanted, 3));
+ VA_Store(wanted, 2, VA_Delete(wanted, 4));
+ VA_Resize(wanted, 3);
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got),
+ "Excise multiple elems");
+
+ VA_Excise(got, 2, 2);
+ VA_Resize(wanted, 2);
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got),
+ "Splicing too many elems truncates");
+
+ VA_Excise(got, 0, 1);
+ VA_Store(wanted, 0, VA_Delete(wanted, 1));
+ VA_Resize(wanted, 1);
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got),
+ "Excise first elem");
+
+ DECREF(got);
+ DECREF(wanted);
+}
+
+static void
+test_Push_VArray(TestBatch *batch) {
+ VArray *wanted = VA_new(0);
+ VArray *got = VA_new(0);
+ VArray *scratch = VA_new(0);
+ uint32_t i;
+
+ for (i = 0; i < 4; i++) { VA_Push(wanted, (Obj*)CB_newf("%u32", i)); }
+ for (i = 0; i < 2; i++) { VA_Push(got, (Obj*)CB_newf("%u32", i)); }
+ for (i = 2; i < 4; i++) { VA_Push(scratch, (Obj*)CB_newf("%u32", i)); }
+
+ VA_Push_VArray(got, scratch);
+ TEST_TRUE(batch, VA_Equals(wanted, (Obj*)got), "Push_VArray");
+
+ DECREF(wanted);
+ DECREF(got);
+ DECREF(scratch);
+}
+
+static void
+test_Clone_and_Shallow_Copy(TestBatch *batch) {
+ VArray *array = VA_new(0);
+ VArray *twin;
+ uint32_t i;
+
+ for (i = 0; i < 10; i++) {
+ VA_Push(array, (Obj*)CB_newf("%u32", i));
+ }
+ twin = VA_Shallow_Copy(array);
+ TEST_TRUE(batch, VA_Equals(array, (Obj*)twin), "Shallow_Copy");
+ TEST_TRUE(batch, VA_Fetch(array, 1) == VA_Fetch(twin, 1),
+ "Shallow_Copy doesn't clone elements");
+ DECREF(twin);
+
+ twin = VA_Clone(array);
+ TEST_TRUE(batch, VA_Equals(array, (Obj*)twin), "Clone");
+ TEST_TRUE(batch, VA_Fetch(array, 1) != VA_Fetch(twin, 1),
+ "Clone performs deep clone");
+
+ DECREF(array);
+ DECREF(twin);
+}
+
+static void
+test_Dump_and_Load(TestBatch *batch) {
+ VArray *array = VA_new(0);
+ Obj *dump;
+ VArray *loaded;
+
+ VA_Push(array, (Obj*)S_new_cb("foo"));
+ dump = (Obj*)VA_Dump(array);
+ loaded = (VArray*)Obj_Load(dump, dump);
+ TEST_TRUE(batch, VA_Equals(array, (Obj*)loaded),
+ "Dump => Load round trip");
+
+ DECREF(array);
+ DECREF(dump);
+ DECREF(loaded);
+}
+
+static void
+test_serialization(TestBatch *batch) {
+ VArray *array = VA_new(0);
+ VArray *dupe;
+ VA_Store(array, 1, (Obj*)CB_newf("foo"));
+ VA_Store(array, 3, (Obj*)CB_newf("bar"));
+ dupe = (VArray*)TestUtils_freeze_thaw((Obj*)array);
+ TEST_TRUE(batch, dupe && VA_Equals(array, (Obj*)dupe),
+ "Round trip through FREEZE/THAW");
+ DECREF(dupe);
+ DECREF(array);
+}
+
+void
+TestVArray_run_tests() {
+ TestBatch *batch = TestBatch_new(39);
+
+ TestBatch_Plan(batch);
+
+ test_Equals(batch);
+ test_Store_Fetch(batch);
+ test_Push_Pop_Shift_Unshift(batch);
+ test_Delete(batch);
+ test_Resize(batch);
+ test_Excise(batch);
+ test_Push_VArray(batch);
+ test_Clone_and_Shallow_Copy(batch);
+ test_Dump_and_Load(batch);
+ test_serialization(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Object/TestVArray.cfh b/core/Lucy/Test/Object/TestVArray.cfh
new file mode 100644
index 0000000..597d472
--- /dev/null
+++ b/core/Lucy/Test/Object/TestVArray.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Object::TestVArray {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestArchitecture.c b/core/Lucy/Test/Plan/TestArchitecture.c
new file mode 100644
index 0000000..cdc64e0
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestArchitecture.c
@@ -0,0 +1,49 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTARCHITECTURE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Plan/TestArchitecture.h"
+#include "Lucy/Plan/Architecture.h"
+
+TestArchitecture*
+TestArch_new() {
+ TestArchitecture *self
+ = (TestArchitecture*)VTable_Make_Obj(TESTARCHITECTURE);
+ return TestArch_init(self);
+}
+
+TestArchitecture*
+TestArch_init(TestArchitecture *self) {
+ Arch_init((Architecture*)self);
+ return self;
+}
+
+int32_t
+TestArch_index_interval(TestArchitecture *self) {
+ UNUSED_VAR(self);
+ return 5;
+}
+
+int32_t
+TestArch_skip_interval(TestArchitecture *self) {
+ UNUSED_VAR(self);
+ return 3;
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestArchitecture.cfh b/core/Lucy/Test/Plan/TestArchitecture.cfh
new file mode 100644
index 0000000..c80c00c
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestArchitecture.cfh
@@ -0,0 +1,39 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Returns absurdly low values for Index_Interval() and Skip_Interval().
+ */
+
+class Lucy::Test::Plan::TestArchitecture cnick TestArch
+ inherits Lucy::Plan::Architecture {
+
+ inert incremented TestArchitecture*
+ new();
+
+ inert TestArchitecture*
+ init(TestArchitecture *self);
+
+ public int32_t
+ Index_Interval(TestArchitecture *self);
+
+ public int32_t
+ Skip_Interval(TestArchitecture *self);
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestBlobType.c b/core/Lucy/Test/Plan/TestBlobType.c
new file mode 100644
index 0000000..22991fb
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestBlobType.c
@@ -0,0 +1,54 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTBLOBTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Plan/TestBlobType.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Plan/BlobType.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ BlobType *type = BlobType_new(true);
+ Obj *dump = (Obj*)BlobType_Dump(type);
+ Obj *clone = Obj_Load(dump, dump);
+ Obj *another_dump = (Obj*)BlobType_Dump_For_Schema(type);
+ BlobType *another_clone = BlobType_load(NULL, another_dump);
+
+ TEST_TRUE(batch, BlobType_Equals(type, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_TRUE(batch, BlobType_Equals(type, (Obj*)another_clone),
+ "Dump_For_Schema => Load round trip");
+
+ DECREF(type);
+ DECREF(dump);
+ DECREF(clone);
+ DECREF(another_dump);
+ DECREF(another_clone);
+}
+
+void
+TestBlobType_run_tests() {
+ TestBatch *batch = TestBatch_new(2);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestBlobType.cfh b/core/Lucy/Test/Plan/TestBlobType.cfh
new file mode 100644
index 0000000..c1255ae
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestBlobType.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Plan::TestBlobType {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestFieldType.c b/core/Lucy/Test/Plan/TestFieldType.c
new file mode 100644
index 0000000..bb2e41a
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestFieldType.c
@@ -0,0 +1,104 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTFIELDTYPE
+#define C_LUCY_DUMMYFIELDTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Plan/TestFieldType.h"
+#include "Lucy/Test/TestUtils.h"
+
+DummyFieldType*
+DummyFieldType_new() {
+ DummyFieldType *self = (DummyFieldType*)VTable_Make_Obj(DUMMYFIELDTYPE);
+ return (DummyFieldType*)FType_init((FieldType*)self);
+}
+
+static FieldType*
+S_alt_field_type() {
+ ZombieCharBuf *name = ZCB_WRAP_STR("DummyFieldType2", 15);
+ VTable *vtable = VTable_singleton((CharBuf*)name, DUMMYFIELDTYPE);
+ FieldType *self = (FieldType*)VTable_Make_Obj(vtable);
+ return FType_init(self);
+}
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ FieldType *type = (FieldType*)DummyFieldType_new();
+ FieldType *other = (FieldType*)DummyFieldType_new();
+ FieldType *class_differs = S_alt_field_type();
+ FieldType *boost_differs = (FieldType*)DummyFieldType_new();
+ FieldType *indexed = (FieldType*)DummyFieldType_new();
+ FieldType *stored = (FieldType*)DummyFieldType_new();
+
+ FType_Set_Boost(other, 1.0);
+ FType_Set_Indexed(indexed, false);
+ FType_Set_Stored(stored, false);
+
+ FType_Set_Boost(boost_differs, 1.5);
+ FType_Set_Indexed(indexed, true);
+ FType_Set_Stored(stored, true);
+
+ TEST_TRUE(batch, FType_Equals(type, (Obj*)other),
+ "Equals() true with identical stats");
+ TEST_FALSE(batch, FType_Equals(type, (Obj*)class_differs),
+ "Equals() false with subclass");
+ TEST_FALSE(batch, FType_Equals(type, (Obj*)class_differs),
+ "Equals() false with super class");
+ TEST_FALSE(batch, FType_Equals(type, (Obj*)boost_differs),
+ "Equals() false with different boost");
+ TEST_FALSE(batch, FType_Equals(type, (Obj*)indexed),
+ "Equals() false with indexed => true");
+ TEST_FALSE(batch, FType_Equals(type, (Obj*)stored),
+ "Equals() false with stored => true");
+
+ DECREF(stored);
+ DECREF(indexed);
+ DECREF(boost_differs);
+ DECREF(other);
+ DECREF(type);
+}
+
+static void
+test_Compare_Values(TestBatch *batch) {
+ FieldType *type = (FieldType*)DummyFieldType_new();
+ ZombieCharBuf *a = ZCB_WRAP_STR("a", 1);
+ ZombieCharBuf *b = ZCB_WRAP_STR("b", 1);
+
+ TEST_TRUE(batch,
+ FType_Compare_Values(type, (Obj*)a, (Obj*)b) < 0,
+ "a less than b");
+ TEST_TRUE(batch,
+ FType_Compare_Values(type, (Obj*)b, (Obj*)a) > 0,
+ "b greater than a");
+ TEST_TRUE(batch,
+ FType_Compare_Values(type, (Obj*)b, (Obj*)b) == 0,
+ "b equals b");
+
+ DECREF(type);
+}
+
+void
+TestFType_run_tests() {
+ TestBatch *batch = TestBatch_new(9);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ test_Compare_Values(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestFieldType.cfh b/core/Lucy/Test/Plan/TestFieldType.cfh
new file mode 100644
index 0000000..597201e
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestFieldType.cfh
@@ -0,0 +1,28 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Plan::TestFieldType cnick TestFType {
+ inert void
+ run_tests();
+}
+
+class Lucy::Test::Plan::DummyFieldType inherits Lucy::Plan::FieldType {
+ inert incremented DummyFieldType*
+ new();
+}
+
diff --git a/core/Lucy/Test/Plan/TestFullTextType.c b/core/Lucy/Test/Plan/TestFullTextType.c
new file mode 100644
index 0000000..1be5a09
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestFullTextType.c
@@ -0,0 +1,109 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTFULLTEXTTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Plan/TestFullTextType.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Analysis/CaseFolder.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ RegexTokenizer *tokenizer = RegexTokenizer_new(NULL);
+ CaseFolder *case_folder = CaseFolder_new();
+ FullTextType *type = FullTextType_new((Analyzer*)tokenizer);
+ FullTextType *other = FullTextType_new((Analyzer*)case_folder);
+ FullTextType *boost_differs = FullTextType_new((Analyzer*)tokenizer);
+ FullTextType *not_indexed = FullTextType_new((Analyzer*)tokenizer);
+ FullTextType *not_stored = FullTextType_new((Analyzer*)tokenizer);
+ FullTextType *highlightable = FullTextType_new((Analyzer*)tokenizer);
+ Obj *dump = (Obj*)FullTextType_Dump(type);
+ Obj *clone = Obj_Load(dump, dump);
+ Obj *another_dump = (Obj*)FullTextType_Dump_For_Schema(type);
+
+ FullTextType_Set_Boost(boost_differs, 1.5);
+ FullTextType_Set_Indexed(not_indexed, false);
+ FullTextType_Set_Stored(not_stored, false);
+ FullTextType_Set_Highlightable(highlightable, true);
+
+ // (This step is normally performed by Schema_Load() internally.)
+ Hash_Store_Str((Hash*)another_dump, "analyzer", 8, INCREF(tokenizer));
+ FullTextType *another_clone = FullTextType_load(NULL, another_dump);
+
+ TEST_FALSE(batch, FullTextType_Equals(type, (Obj*)boost_differs),
+ "Equals() false with different boost");
+ TEST_FALSE(batch, FullTextType_Equals(type, (Obj*)other),
+ "Equals() false with different Analyzer");
+ TEST_FALSE(batch, FullTextType_Equals(type, (Obj*)not_indexed),
+ "Equals() false with indexed => false");
+ TEST_FALSE(batch, FullTextType_Equals(type, (Obj*)not_stored),
+ "Equals() false with stored => false");
+ TEST_FALSE(batch, FullTextType_Equals(type, (Obj*)highlightable),
+ "Equals() false with highlightable => true");
+ TEST_TRUE(batch, FullTextType_Equals(type, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_TRUE(batch, FullTextType_Equals(type, (Obj*)another_clone),
+ "Dump_For_Schema => Load round trip");
+
+ DECREF(another_clone);
+ DECREF(dump);
+ DECREF(clone);
+ DECREF(another_dump);
+ DECREF(highlightable);
+ DECREF(not_stored);
+ DECREF(not_indexed);
+ DECREF(boost_differs);
+ DECREF(other);
+ DECREF(type);
+ DECREF(case_folder);
+ DECREF(tokenizer);
+}
+
+static void
+test_Compare_Values(TestBatch *batch) {
+ RegexTokenizer *tokenizer = RegexTokenizer_new(NULL);
+ FullTextType *type = FullTextType_new((Analyzer*)tokenizer);
+ ZombieCharBuf *a = ZCB_WRAP_STR("a", 1);
+ ZombieCharBuf *b = ZCB_WRAP_STR("b", 1);
+
+ TEST_TRUE(batch,
+ FullTextType_Compare_Values(type, (Obj*)a, (Obj*)b) < 0,
+ "a less than b");
+ TEST_TRUE(batch,
+ FullTextType_Compare_Values(type, (Obj*)b, (Obj*)a) > 0,
+ "b greater than a");
+ TEST_TRUE(batch,
+ FullTextType_Compare_Values(type, (Obj*)b, (Obj*)b) == 0,
+ "b equals b");
+
+ DECREF(type);
+ DECREF(tokenizer);
+}
+
+void
+TestFullTextType_run_tests() {
+ TestBatch *batch = TestBatch_new(10);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ test_Compare_Values(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestFullTextType.cfh b/core/Lucy/Test/Plan/TestFullTextType.cfh
new file mode 100644
index 0000000..40810dd
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestFullTextType.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Plan::TestFullTextType {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestNumericType.c b/core/Lucy/Test/Plan/TestNumericType.c
new file mode 100644
index 0000000..18ff082
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestNumericType.c
@@ -0,0 +1,103 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTNUMERICTYPE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Plan/TestNumericType.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Plan/BlobType.h"
+#include "Lucy/Plan/NumericType.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ Int32Type *i32 = Int32Type_new();
+ Int64Type *i64 = Int64Type_new();
+ Float32Type *f32 = Float32Type_new();
+ Float64Type *f64 = Float64Type_new();
+
+ TEST_FALSE(batch, Int32Type_Equals(i32, (Obj*)i64),
+ "Int32Type_Equals() false for different type");
+ TEST_FALSE(batch, Int32Type_Equals(i32, NULL),
+ "Int32Type_Equals() false for NULL");
+
+ TEST_FALSE(batch, Int64Type_Equals(i64, (Obj*)i32),
+ "Int64Type_Equals() false for different type");
+ TEST_FALSE(batch, Int64Type_Equals(i64, NULL),
+ "Int64Type_Equals() false for NULL");
+
+ TEST_FALSE(batch, Float32Type_Equals(f32, (Obj*)f64),
+ "Float32Type_Equals() false for different type");
+ TEST_FALSE(batch, Float32Type_Equals(f32, NULL),
+ "Float32Type_Equals() false for NULL");
+
+ TEST_FALSE(batch, Float64Type_Equals(f64, (Obj*)f32),
+ "Float64Type_Equals() false for different type");
+ TEST_FALSE(batch, Float64Type_Equals(f64, NULL),
+ "Float64Type_Equals() false for NULL");
+
+ {
+ Obj *dump = (Obj*)Int32Type_Dump(i32);
+ Obj *other = Obj_Load(dump, dump);
+ TEST_TRUE(batch, Int32Type_Equals(i32, other),
+ "Dump => Load round trip for Int32Type");
+ DECREF(dump);
+ DECREF(other);
+ }
+
+ {
+ Obj *dump = (Obj*)Int64Type_Dump(i64);
+ Obj *other = Obj_Load(dump, dump);
+ TEST_TRUE(batch, Int64Type_Equals(i64, other),
+ "Dump => Load round trip for Int64Type");
+ DECREF(dump);
+ DECREF(other);
+ }
+
+ {
+ Obj *dump = (Obj*)Float32Type_Dump(f32);
+ Obj *other = Obj_Load(dump, dump);
+ TEST_TRUE(batch, Float32Type_Equals(f32, other),
+ "Dump => Load round trip for Float32Type");
+ DECREF(dump);
+ DECREF(other);
+ }
+
+ {
+ Obj *dump = (Obj*)Float64Type_Dump(f64);
+ Obj *other = Obj_Load(dump, dump);
+ TEST_TRUE(batch, Float64Type_Equals(f64, other),
+ "Dump => Load round trip for Float64Type");
+ DECREF(dump);
+ DECREF(other);
+ }
+
+ DECREF(i32);
+ DECREF(i64);
+ DECREF(f32);
+ DECREF(f64);
+}
+
+void
+TestNumericType_run_tests() {
+ TestBatch *batch = TestBatch_new(12);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Plan/TestNumericType.cfh b/core/Lucy/Test/Plan/TestNumericType.cfh
new file mode 100644
index 0000000..1650ce6
--- /dev/null
+++ b/core/Lucy/Test/Plan/TestNumericType.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Plan::TestNumericType {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestLeafQuery.c b/core/Lucy/Test/Search/TestLeafQuery.c
new file mode 100644
index 0000000..095a678
--- /dev/null
+++ b/core/Lucy/Test/Search/TestLeafQuery.c
@@ -0,0 +1,65 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTLEAFQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Search/TestLeafQuery.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Search/LeafQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ LeafQuery *query = TestUtils_make_leaf_query("content", "foo");
+ LeafQuery *field_differs = TestUtils_make_leaf_query("stuff", "foo");
+ LeafQuery *null_field = TestUtils_make_leaf_query(NULL, "foo");
+ LeafQuery *term_differs = TestUtils_make_leaf_query("content", "bar");
+ LeafQuery *boost_differs = TestUtils_make_leaf_query("content", "foo");
+ Obj *dump = (Obj*)LeafQuery_Dump(query);
+ LeafQuery *clone = (LeafQuery*)LeafQuery_Load(term_differs, dump);
+
+ TEST_FALSE(batch, LeafQuery_Equals(query, (Obj*)field_differs),
+ "Equals() false with different field");
+ TEST_FALSE(batch, LeafQuery_Equals(query, (Obj*)null_field),
+ "Equals() false with null field");
+ TEST_FALSE(batch, LeafQuery_Equals(query, (Obj*)term_differs),
+ "Equals() false with different term");
+ LeafQuery_Set_Boost(boost_differs, 0.5);
+ TEST_FALSE(batch, LeafQuery_Equals(query, (Obj*)boost_differs),
+ "Equals() false with different boost");
+ TEST_TRUE(batch, LeafQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(query);
+ DECREF(term_differs);
+ DECREF(field_differs);
+ DECREF(null_field);
+ DECREF(boost_differs);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+void
+TestLeafQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(5);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestLeafQuery.cfh b/core/Lucy/Test/Search/TestLeafQuery.cfh
new file mode 100644
index 0000000..5a16779
--- /dev/null
+++ b/core/Lucy/Test/Search/TestLeafQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestLeafQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestMatchAllQuery.c b/core/Lucy/Test/Search/TestMatchAllQuery.c
new file mode 100644
index 0000000..7ff1525
--- /dev/null
+++ b/core/Lucy/Test/Search/TestMatchAllQuery.c
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTMATCHALLQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestMatchAllQuery.h"
+#include "Lucy/Search/MatchAllQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ MatchAllQuery *query = MatchAllQuery_new();
+ Obj *dump = (Obj*)MatchAllQuery_Dump(query);
+ MatchAllQuery *clone = (MatchAllQuery*)MatchAllQuery_Load(query, dump);
+
+ TEST_TRUE(batch, MatchAllQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_FALSE(batch, MatchAllQuery_Equals(query, (Obj*)&EMPTY), "Equals");
+
+ DECREF(query);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+
+void
+TestMatchAllQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(2);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestMatchAllQuery.cfh b/core/Lucy/Test/Search/TestMatchAllQuery.cfh
new file mode 100644
index 0000000..e56f29b
--- /dev/null
+++ b/core/Lucy/Test/Search/TestMatchAllQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestMatchAllQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestNOTQuery.c b/core/Lucy/Test/Search/TestNOTQuery.c
new file mode 100644
index 0000000..9a345fc
--- /dev/null
+++ b/core/Lucy/Test/Search/TestNOTQuery.c
@@ -0,0 +1,64 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTNOTQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestNOTQuery.h"
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ Query *a_leaf = (Query*)TestUtils_make_leaf_query(NULL, "a");
+ Query *b_leaf = (Query*)TestUtils_make_leaf_query(NULL, "b");
+ NOTQuery *query = NOTQuery_new(a_leaf);
+ NOTQuery *kids_differ = NOTQuery_new(b_leaf);
+ NOTQuery *boost_differs = NOTQuery_new(a_leaf);
+ Obj *dump = (Obj*)NOTQuery_Dump(query);
+ NOTQuery *clone = (NOTQuery*)Obj_Load(dump, dump);
+
+ TEST_FALSE(batch, NOTQuery_Equals(query, (Obj*)kids_differ),
+ "Different kids spoil Equals");
+ TEST_TRUE(batch, NOTQuery_Equals(query, (Obj*)boost_differs),
+ "Equals with identical boosts");
+ NOTQuery_Set_Boost(boost_differs, 1.5);
+ TEST_FALSE(batch, NOTQuery_Equals(query, (Obj*)boost_differs),
+ "Different boost spoils Equals");
+ TEST_TRUE(batch, NOTQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(a_leaf);
+ DECREF(b_leaf);
+ DECREF(query);
+ DECREF(kids_differ);
+ DECREF(boost_differs);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+void
+TestNOTQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestNOTQuery.cfh b/core/Lucy/Test/Search/TestNOTQuery.cfh
new file mode 100644
index 0000000..3bfa4cd
--- /dev/null
+++ b/core/Lucy/Test/Search/TestNOTQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestNOTQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestNoMatchQuery.c b/core/Lucy/Test/Search/TestNoMatchQuery.c
new file mode 100644
index 0000000..910c157
--- /dev/null
+++ b/core/Lucy/Test/Search/TestNoMatchQuery.c
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTNOMATCHQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestNoMatchQuery.h"
+#include "Lucy/Search/NoMatchQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ NoMatchQuery *query = NoMatchQuery_new();
+ Obj *dump = (Obj*)NoMatchQuery_Dump(query);
+ NoMatchQuery *clone = (NoMatchQuery*)NoMatchQuery_Load(query, dump);
+
+ TEST_TRUE(batch, NoMatchQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+ TEST_FALSE(batch, NoMatchQuery_Equals(query, (Obj*)&EMPTY), "Equals");
+
+ DECREF(query);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+
+void
+TestNoMatchQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(2);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestNoMatchQuery.cfh b/core/Lucy/Test/Search/TestNoMatchQuery.cfh
new file mode 100644
index 0000000..2b4f1c9
--- /dev/null
+++ b/core/Lucy/Test/Search/TestNoMatchQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestNoMatchQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestPhraseQuery.c b/core/Lucy/Test/Search/TestPhraseQuery.c
new file mode 100644
index 0000000..5685c4a
--- /dev/null
+++ b/core/Lucy/Test/Search/TestPhraseQuery.c
@@ -0,0 +1,47 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTPHRASEQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestPhraseQuery.h"
+#include "Lucy/Search/PhraseQuery.h"
+
+static void
+test_Dump_And_Load(TestBatch *batch) {
+ PhraseQuery *query
+ = TestUtils_make_phrase_query("content", "a", "b", "c", NULL);
+ Obj *dump = (Obj*)PhraseQuery_Dump(query);
+ PhraseQuery *twin = (PhraseQuery*)Obj_Load(dump, dump);
+ TEST_TRUE(batch, PhraseQuery_Equals(query, (Obj*)twin),
+ "Dump => Load round trip");
+ DECREF(query);
+ DECREF(dump);
+ DECREF(twin);
+}
+
+void
+TestPhraseQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(1);
+ TestBatch_Plan(batch);
+ test_Dump_And_Load(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestPhraseQuery.cfh b/core/Lucy/Test/Search/TestPhraseQuery.cfh
new file mode 100644
index 0000000..ca1617a
--- /dev/null
+++ b/core/Lucy/Test/Search/TestPhraseQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestPhraseQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestPolyQuery.c b/core/Lucy/Test/Search/TestPolyQuery.c
new file mode 100644
index 0000000..89d73be
--- /dev/null
+++ b/core/Lucy/Test/Search/TestPolyQuery.c
@@ -0,0 +1,84 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTPOLYQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestPolyQuery.h"
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Search/ORQuery.h"
+#include "Lucy/Search/PolyQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch, uint32_t boolop) {
+ LeafQuery *a_leaf = TestUtils_make_leaf_query(NULL, "a");
+ LeafQuery *b_leaf = TestUtils_make_leaf_query(NULL, "b");
+ LeafQuery *c_leaf = TestUtils_make_leaf_query(NULL, "c");
+ PolyQuery *query
+ = (PolyQuery*)TestUtils_make_poly_query(boolop, INCREF(a_leaf),
+ INCREF(b_leaf), NULL);
+ PolyQuery *kids_differ
+ = (PolyQuery*)TestUtils_make_poly_query(boolop, INCREF(a_leaf),
+ INCREF(b_leaf),
+ INCREF(c_leaf),
+ NULL);
+ PolyQuery *boost_differs
+ = (PolyQuery*)TestUtils_make_poly_query(boolop, INCREF(a_leaf),
+ INCREF(b_leaf), NULL);
+ Obj *dump = (Obj*)PolyQuery_Dump(query);
+ PolyQuery *clone = (PolyQuery*)Obj_Load(dump, dump);
+
+ TEST_FALSE(batch, PolyQuery_Equals(query, (Obj*)kids_differ),
+ "Different kids spoil Equals");
+ TEST_TRUE(batch, PolyQuery_Equals(query, (Obj*)boost_differs),
+ "Equals with identical boosts");
+ PolyQuery_Set_Boost(boost_differs, 1.5);
+ TEST_FALSE(batch, PolyQuery_Equals(query, (Obj*)boost_differs),
+ "Different boost spoils Equals");
+ TEST_TRUE(batch, PolyQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(a_leaf);
+ DECREF(b_leaf);
+ DECREF(c_leaf);
+ DECREF(query);
+ DECREF(kids_differ);
+ DECREF(boost_differs);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+void
+TestANDQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch, BOOLOP_AND);
+ DECREF(batch);
+}
+
+void
+TestORQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch, BOOLOP_OR);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestPolyQuery.cfh b/core/Lucy/Test/Search/TestPolyQuery.cfh
new file mode 100644
index 0000000..710f523
--- /dev/null
+++ b/core/Lucy/Test/Search/TestPolyQuery.cfh
@@ -0,0 +1,31 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestPolyQuery { }
+
+inert class Lucy::Test::Search::TestANDQuery {
+ inert void
+ run_tests();
+}
+
+inert class Lucy::Test::Search::TestORQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestQueryParser.c b/core/Lucy/Test/Search/TestQueryParser.c
new file mode 100644
index 0000000..ef5ec10
--- /dev/null
+++ b/core/Lucy/Test/Search/TestQueryParser.c
@@ -0,0 +1,78 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTQUERYPARSER
+#include "Lucy/Util/ToolSet.h"
+#include <stdarg.h>
+#include <string.h>
+
+#include "Lucy/Test/Search/TestQueryParser.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Search/PhraseQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Search/ORQuery.h"
+
+TestQueryParser*
+TestQP_new(const char *query_string, Query *tree, Query *expanded,
+ uint32_t num_hits) {
+ TestQueryParser *self
+ = (TestQueryParser*)VTable_Make_Obj(TESTQUERYPARSER);
+ return TestQP_init(self, query_string, tree, expanded, num_hits);
+}
+
+TestQueryParser*
+TestQP_init(TestQueryParser *self, const char *query_string, Query *tree,
+ Query *expanded, uint32_t num_hits) {
+ self->query_string = query_string ? TestUtils_get_cb(query_string) : NULL;
+ self->tree = tree ? tree : NULL;
+ self->expanded = expanded ? expanded : NULL;
+ self->num_hits = num_hits;
+ return self;
+}
+
+void
+TestQP_destroy(TestQueryParser *self) {
+ DECREF(self->query_string);
+ DECREF(self->tree);
+ DECREF(self->expanded);
+ SUPER_DESTROY(self, TESTQUERYPARSER);
+}
+
+CharBuf*
+TestQP_get_query_string(TestQueryParser *self) {
+ return self->query_string;
+}
+
+Query*
+TestQP_get_tree(TestQueryParser *self) {
+ return self->tree;
+}
+
+Query*
+TestQP_get_expanded(TestQueryParser *self) {
+ return self->expanded;
+}
+
+uint32_t
+TestQP_get_num_hits(TestQueryParser *self) {
+ return self->num_hits;
+}
+
+
+
diff --git a/core/Lucy/Test/Search/TestQueryParser.cfh b/core/Lucy/Test/Search/TestQueryParser.cfh
new file mode 100644
index 0000000..2780326
--- /dev/null
+++ b/core/Lucy/Test/Search/TestQueryParser.cfh
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Test case object for QueryParser unit tests.
+ */
+
+class Lucy::Test::Search::TestQueryParser cnick TestQP
+ inherits Lucy::Object::Obj {
+
+ CharBuf *query_string;
+ Query *tree;
+ Query *expanded;
+ uint32_t num_hits;
+
+ /** Note that unlike most Clownfish constructors, this routine will consume one
+ * reference count each for <code>tree</code>, and <code>expanded</code>.
+ */
+ inert incremented TestQueryParser*
+ new(const char *query_string = NULL, Query *tree = NULL,
+ Query *expanded = NULL, uint32_t num_hits);
+
+ inert TestQueryParser*
+ init(TestQueryParser *self, const char *query_string = NULL,
+ Query *tree = NULL, Query *expanded = NULL, uint32_t num_hits);
+
+ nullable CharBuf*
+ Get_Query_String(TestQueryParser *self);
+
+ nullable Query*
+ Get_Tree(TestQueryParser *self);
+
+ nullable Query*
+ Get_Expanded(TestQueryParser *self);
+
+ uint32_t
+ Get_Num_Hits(TestQueryParser *self);
+
+ public void
+ Destroy(TestQueryParser *self);
+}
+
+__C__
+
+__END_C__
+
+
diff --git a/core/Lucy/Test/Search/TestQueryParserLogic.c b/core/Lucy/Test/Search/TestQueryParserLogic.c
new file mode 100644
index 0000000..fa17b19
--- /dev/null
+++ b/core/Lucy/Test/Search/TestQueryParserLogic.c
@@ -0,0 +1,685 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTQUERYPARSERLOGIC
+#define C_LUCY_TESTQUERYPARSER
+#include "Lucy/Util/ToolSet.h"
+#include <stdarg.h>
+#include <string.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Search/TestQueryParserLogic.h"
+#include "Lucy/Test/Search/TestQueryParser.h"
+#include "Lucy/Test/TestSchema.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Index/Indexer.h"
+#include "Lucy/Search/Hits.h"
+#include "Lucy/Search/IndexSearcher.h"
+#include "Lucy/Search/QueryParser.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Search/PhraseQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Search/MatchAllQuery.h"
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Search/NoMatchQuery.h"
+#include "Lucy/Search/ORQuery.h"
+#include "Lucy/Search/RequiredOptionalQuery.h"
+#include "Lucy/Store/RAMFolder.h"
+
+#define make_leaf_query (Query*)lucy_TestUtils_make_leaf_query
+#define make_not_query (Query*)lucy_TestUtils_make_not_query
+#define make_poly_query (Query*)lucy_TestUtils_make_poly_query
+
+static TestQueryParser*
+logical_test_empty_phrase(uint32_t boolop) {
+ Query *tree = make_leaf_query(NULL, "\"\"");
+ UNUSED_VAR(boolop);
+ return TestQP_new("\"\"", tree, NULL, 0);
+}
+
+static TestQueryParser*
+logical_test_empty_parens(uint32_t boolop) {
+ Query *tree = make_poly_query(boolop, NULL);
+ return TestQP_new("()", tree, NULL, 0);
+}
+
+static TestQueryParser*
+logical_test_nested_empty_parens(uint32_t boolop) {
+ Query *inner = make_poly_query(boolop, NULL);
+ Query *tree = make_poly_query(boolop, inner, NULL);
+ return TestQP_new("(())", tree, NULL, 0);
+}
+
+static TestQueryParser*
+logical_test_nested_empty_phrase(uint32_t boolop) {
+ Query *leaf = make_leaf_query(NULL, "\"\"");
+ Query *tree = make_poly_query(boolop, leaf, NULL);
+ return TestQP_new("(\"\")", tree, NULL, 0);
+}
+
+static TestQueryParser*
+logical_test_simple_term(uint32_t boolop) {
+ Query *tree = make_leaf_query(NULL, "b");
+ UNUSED_VAR(boolop);
+ return TestQP_new("b", tree, NULL, 3);
+}
+
+static TestQueryParser*
+logical_test_one_nested_term(uint32_t boolop) {
+ Query *leaf = make_leaf_query(NULL, "a");
+ Query *tree = make_poly_query(boolop, leaf, NULL);
+ return TestQP_new("(a)", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_one_term_phrase(uint32_t boolop) {
+ Query *tree = make_leaf_query(NULL, "\"a\"");
+ UNUSED_VAR(boolop);
+ return TestQP_new("\"a\"", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_two_terms(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *tree = make_poly_query(boolop, a_leaf, b_leaf, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 3;
+ return TestQP_new("a b", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_two_terms_nested(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *tree = make_poly_query(boolop, a_leaf, b_leaf, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 3;
+ return TestQP_new("(a b)", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_one_term_one_single_term_phrase(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "\"b\"");
+ Query *tree = make_poly_query(boolop, a_leaf, b_leaf, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 3;
+ return TestQP_new("a \"b\"", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_two_terms_one_nested(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *b_tree = make_poly_query(boolop, b_leaf, NULL);
+ Query *tree = make_poly_query(boolop, a_leaf, b_tree, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 3;
+ return TestQP_new("a (b)", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_one_term_one_nested_single_term_phrase(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "\"b\"");
+ Query *b_tree = make_poly_query(boolop, b_leaf, NULL);
+ Query *tree = make_poly_query(boolop, a_leaf, b_tree, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 3;
+ return TestQP_new("a (\"b\")", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_phrase(uint32_t boolop) {
+ Query *tree = make_leaf_query(NULL, "\"a b\"");
+ UNUSED_VAR(boolop);
+ return TestQP_new("\"a b\"", tree, NULL, 3);
+}
+
+static TestQueryParser*
+logical_test_nested_phrase(uint32_t boolop) {
+ Query *leaf = make_leaf_query(NULL, "\"a b\"");
+ Query *tree = make_poly_query(boolop, leaf, NULL);
+ return TestQP_new("(\"a b\")", tree, NULL, 3);
+}
+
+static TestQueryParser*
+logical_test_three_terms(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *c_leaf = make_leaf_query(NULL, "c");
+ Query *tree = make_poly_query(boolop, a_leaf, b_leaf,
+ c_leaf, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 2;
+ return TestQP_new("a b c", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_three_terms_two_nested(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *c_leaf = make_leaf_query(NULL, "c");
+ Query *inner_tree = make_poly_query(boolop, b_leaf, c_leaf, NULL);
+ Query *tree = make_poly_query(boolop, a_leaf, inner_tree, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 2;
+ return TestQP_new("a (b c)", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_one_term_one_phrase(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *bc_leaf = make_leaf_query(NULL, "\"b c\"");
+ Query *tree = make_poly_query(boolop, a_leaf, bc_leaf, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 2;
+ return TestQP_new("a \"b c\"", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_one_term_one_nested_phrase(uint32_t boolop) {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *bc_leaf = make_leaf_query(NULL, "\"b c\"");
+ Query *inner_tree = make_poly_query(boolop, bc_leaf, NULL);
+ Query *tree = make_poly_query(boolop, a_leaf, inner_tree, NULL);
+ uint32_t num_hits = boolop == BOOLOP_OR ? 4 : 2;
+ return TestQP_new("a (\"b c\")", tree, NULL, num_hits);
+}
+
+static TestQueryParser*
+logical_test_long_phrase(uint32_t boolop) {
+ Query *tree = make_leaf_query(NULL, "\"a b c\"");
+ UNUSED_VAR(boolop);
+ return TestQP_new("\"a b c\"", tree, NULL, 2);
+}
+
+static TestQueryParser*
+logical_test_pure_negation(uint32_t boolop) {
+ Query *leaf = make_leaf_query(NULL, "x");
+ Query *tree = make_not_query(leaf);
+ UNUSED_VAR(boolop);
+ return TestQP_new("-x", tree, NULL, 0);
+}
+
+static TestQueryParser*
+logical_test_double_negative(uint32_t boolop) {
+ Query *tree = make_leaf_query(NULL, "a");
+ UNUSED_VAR(boolop);
+ return TestQP_new("--a", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_triple_negative(uint32_t boolop) {
+ Query *leaf = make_leaf_query(NULL, "a");
+ Query *tree = make_not_query(leaf);
+ UNUSED_VAR(boolop);
+ return TestQP_new("---a", tree, NULL, 0);
+}
+
+// Technically, this should produce an acceptably small result set, but it's
+// too difficult to prune -- so QParser_Prune just lops it because it's a
+// top-level NOTQuery.
+static TestQueryParser*
+logical_test_nested_negations(uint32_t boolop) {
+ Query *query = make_leaf_query(NULL, "a");
+ query = make_poly_query(boolop, query, NULL);
+ query = make_not_query(query);
+ query = make_poly_query(BOOLOP_AND, query, NULL);
+ query = make_not_query(query);
+ return TestQP_new("-(-(a))", query, NULL, 0);
+}
+
+static TestQueryParser*
+logical_test_two_terms_one_required(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *tree;
+ if (boolop == BOOLOP_AND) {
+ tree = make_poly_query(boolop, a_query, b_query, NULL);
+ }
+ else {
+ tree = (Query*)ReqOptQuery_new(b_query, a_query);
+ DECREF(b_query);
+ DECREF(a_query);
+ }
+ return TestQP_new("a +b", tree, NULL, 3);
+}
+
+static TestQueryParser*
+logical_test_intersection(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *tree = make_poly_query(BOOLOP_AND, a_query, b_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a AND b", tree, NULL, 3);
+}
+
+static TestQueryParser*
+logical_test_three_way_intersection(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *tree = make_poly_query(BOOLOP_AND, a_query, b_query,
+ c_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a AND b AND c", tree, NULL, 2);
+}
+
+static TestQueryParser*
+logical_test_union(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *tree = make_poly_query(BOOLOP_OR, a_query, b_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a OR b", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_three_way_union(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *tree = make_poly_query(BOOLOP_OR, a_query, b_query, c_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a OR b OR c", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_a_or_plus_b(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *tree = make_poly_query(BOOLOP_OR, a_query, b_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a OR +b", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_and_not(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *not_b = make_not_query(b_query);
+ Query *tree = make_poly_query(BOOLOP_AND, a_query, not_b, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a AND NOT b", tree, NULL, 1);
+}
+
+static TestQueryParser*
+logical_test_nested_or(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *nested = make_poly_query(BOOLOP_OR, b_query, c_query, NULL);
+ Query *tree = make_poly_query(boolop, a_query, nested, NULL);
+ return TestQP_new("a (b OR c)", tree, NULL, boolop == BOOLOP_OR ? 4 : 3);
+}
+
+static TestQueryParser*
+logical_test_and_nested_or(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *nested = make_poly_query(BOOLOP_OR, b_query, c_query, NULL);
+ Query *tree = make_poly_query(BOOLOP_AND, a_query, nested, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a AND (b OR c)", tree, NULL, 3);
+}
+
+static TestQueryParser*
+logical_test_or_nested_or(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *nested = make_poly_query(BOOLOP_OR, b_query, c_query, NULL);
+ Query *tree = make_poly_query(BOOLOP_OR, a_query, nested, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a OR (b OR c)", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_and_not_nested_or(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *nested = make_poly_query(BOOLOP_OR, b_query, c_query, NULL);
+ Query *not_nested = make_not_query(nested);
+ Query *tree = make_poly_query(BOOLOP_AND, a_query,
+ not_nested, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a AND NOT (b OR c)", tree, NULL, 1);
+}
+
+static TestQueryParser*
+logical_test_required_phrase_negated_term(uint32_t boolop) {
+ Query *bc_query = make_leaf_query(NULL, "\"b c\"");
+ Query *d_query = make_leaf_query(NULL, "d");
+ Query *not_d = make_not_query(d_query);
+ Query *tree = make_poly_query(BOOLOP_AND, bc_query, not_d, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("+\"b c\" -d", tree, NULL, 1);
+}
+
+static TestQueryParser*
+logical_test_required_term_optional_phrase(uint32_t boolop) {
+ Query *ab_query = make_leaf_query(NULL, "\"a b\"");
+ Query *d_query = make_leaf_query(NULL, "d");
+ Query *tree;
+ if (boolop == BOOLOP_AND) {
+ tree = make_poly_query(BOOLOP_AND, ab_query, d_query, NULL);
+ }
+ else {
+ tree = (Query*)ReqOptQuery_new(d_query, ab_query);
+ DECREF(d_query);
+ DECREF(ab_query);
+ }
+ UNUSED_VAR(boolop);
+ return TestQP_new("\"a b\" +d", tree, NULL, 1);
+}
+
+static TestQueryParser*
+logical_test_nested_nest(uint32_t boolop) {
+ Query *a_query = make_leaf_query(NULL, "a");
+ Query *b_query = make_leaf_query(NULL, "b");
+ Query *c_query = make_leaf_query(NULL, "c");
+ Query *d_query = make_leaf_query(NULL, "d");
+ Query *innermost = make_poly_query(BOOLOP_AND, c_query, d_query, NULL);
+ Query *inner = make_poly_query(BOOLOP_OR, b_query, innermost, NULL);
+ Query *not_inner = make_not_query(inner);
+ Query *tree = make_poly_query(BOOLOP_AND, a_query, not_inner, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("a AND NOT (b OR (c AND d))", tree, NULL, 1);
+}
+
+static TestQueryParser*
+logical_test_field_bool_group(uint32_t boolop) {
+ Query *b_query = make_leaf_query("content", "b");
+ Query *c_query = make_leaf_query("content", "c");
+ Query *tree = make_poly_query(boolop, b_query, c_query, NULL);
+ return TestQP_new("content:(b c)", tree, NULL,
+ boolop == BOOLOP_OR ? 3 : 2);
+}
+
+static TestQueryParser*
+logical_test_field_multi_OR(uint32_t boolop) {
+ Query *a_query = make_leaf_query("content", "a");
+ Query *b_query = make_leaf_query("content", "b");
+ Query *c_query = make_leaf_query("content", "c");
+ Query *tree = make_poly_query(BOOLOP_OR, a_query, b_query, c_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("content:(a OR b OR c)", tree, NULL, 4);
+}
+
+static TestQueryParser*
+logical_test_field_multi_AND(uint32_t boolop) {
+ Query *a_query = make_leaf_query("content", "a");
+ Query *b_query = make_leaf_query("content", "b");
+ Query *c_query = make_leaf_query("content", "c");
+ Query *tree = make_poly_query(BOOLOP_AND, a_query, b_query,
+ c_query, NULL);
+ UNUSED_VAR(boolop);
+ return TestQP_new("content:(a AND b AND c)", tree, NULL, 2);
+}
+
+static TestQueryParser*
+logical_test_field_phrase(uint32_t boolop) {
+ Query *tree = make_leaf_query("content", "\"b c\"");
+ UNUSED_VAR(boolop);
+ return TestQP_new("content:\"b c\"", tree, NULL, 2);
+}
+
+static TestQueryParser*
+prune_test_null_querystring() {
+ Query *pruned = (Query*)NoMatchQuery_new();
+ return TestQP_new(NULL, NULL, pruned, 0);
+}
+
+static TestQueryParser*
+prune_test_matchall() {
+ Query *tree = (Query*)MatchAllQuery_new();
+ Query *pruned = (Query*)NoMatchQuery_new();
+ return TestQP_new(NULL, tree, pruned, 0);
+}
+
+static TestQueryParser*
+prune_test_nomatch() {
+ Query *tree = (Query*)NoMatchQuery_new();
+ Query *pruned = (Query*)NoMatchQuery_new();
+ return TestQP_new(NULL, tree, pruned, 0);
+}
+
+static TestQueryParser*
+prune_test_optional_not() {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *not_b = make_not_query(b_leaf);
+ Query *tree = make_poly_query(BOOLOP_OR, (Query*)INCREF(a_leaf),
+ not_b, NULL);
+ Query *nomatch = (Query*)NoMatchQuery_new();
+ Query *pruned = make_poly_query(BOOLOP_OR, a_leaf, nomatch, NULL);
+ return TestQP_new(NULL, tree, pruned, 4);
+}
+
+static TestQueryParser*
+prune_test_reqopt_optional_not() {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *not_b = make_not_query(b_leaf);
+ Query *tree = (Query*)ReqOptQuery_new(a_leaf, not_b);
+ Query *nomatch = (Query*)NoMatchQuery_new();
+ Query *pruned = (Query*)ReqOptQuery_new(a_leaf, nomatch);
+ DECREF(nomatch);
+ DECREF(not_b);
+ DECREF(a_leaf);
+ return TestQP_new(NULL, tree, pruned, 4);
+}
+
+static TestQueryParser*
+prune_test_reqopt_required_not() {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *not_a = make_not_query(a_leaf);
+ Query *tree = (Query*)ReqOptQuery_new(not_a, b_leaf);
+ Query *nomatch = (Query*)NoMatchQuery_new();
+ Query *pruned = (Query*)ReqOptQuery_new(nomatch, b_leaf);
+ DECREF(nomatch);
+ DECREF(not_a);
+ DECREF(b_leaf);
+ return TestQP_new(NULL, tree, pruned, 0);
+}
+
+static TestQueryParser*
+prune_test_not_and_not() {
+ Query *a_leaf = make_leaf_query(NULL, "a");
+ Query *b_leaf = make_leaf_query(NULL, "b");
+ Query *not_a = make_not_query(a_leaf);
+ Query *not_b = make_not_query(b_leaf);
+ Query *tree = make_poly_query(BOOLOP_AND, not_a, not_b, NULL);
+ Query *pruned = make_poly_query(BOOLOP_AND, NULL);
+ return TestQP_new(NULL, tree, pruned, 0);
+}
+
+/***************************************************************************/
+
+typedef TestQueryParser*
+(*lucy_TestQPLogic_logical_test_t)(uint32_t boolop_sym);
+
+static lucy_TestQPLogic_logical_test_t logical_test_funcs[] = {
+ logical_test_empty_phrase,
+ logical_test_empty_parens,
+ logical_test_nested_empty_parens,
+ logical_test_nested_empty_phrase,
+ logical_test_simple_term,
+ logical_test_one_nested_term,
+ logical_test_one_term_phrase,
+ logical_test_two_terms,
+ logical_test_two_terms_nested,
+ logical_test_one_term_one_single_term_phrase,
+ logical_test_two_terms_one_nested,
+ logical_test_one_term_one_nested_phrase,
+ logical_test_phrase,
+ logical_test_nested_phrase,
+ logical_test_three_terms,
+ logical_test_three_terms_two_nested,
+ logical_test_one_term_one_phrase,
+ logical_test_one_term_one_nested_single_term_phrase,
+ logical_test_long_phrase,
+ logical_test_pure_negation,
+ logical_test_double_negative,
+ logical_test_triple_negative,
+ logical_test_nested_negations,
+ logical_test_two_terms_one_required,
+ logical_test_intersection,
+ logical_test_three_way_intersection,
+ logical_test_union,
+ logical_test_three_way_union,
+ logical_test_a_or_plus_b,
+ logical_test_and_not,
+ logical_test_nested_or,
+ logical_test_and_nested_or,
+ logical_test_or_nested_or,
+ logical_test_and_not_nested_or,
+ logical_test_required_phrase_negated_term,
+ logical_test_required_term_optional_phrase,
+ logical_test_nested_nest,
+ logical_test_field_phrase,
+ logical_test_field_bool_group,
+ logical_test_field_multi_OR,
+ logical_test_field_multi_AND,
+ NULL
+};
+
+typedef TestQueryParser*
+(*lucy_TestQPLogic_prune_test_t)();
+
+static lucy_TestQPLogic_prune_test_t prune_test_funcs[] = {
+ prune_test_null_querystring,
+ prune_test_matchall,
+ prune_test_nomatch,
+ prune_test_optional_not,
+ prune_test_reqopt_optional_not,
+ prune_test_reqopt_required_not,
+ prune_test_not_and_not,
+ NULL
+};
+
+static Folder*
+S_create_index() {
+ Schema *schema = (Schema*)TestSchema_new();
+ RAMFolder *folder = RAMFolder_new(NULL);
+ VArray *doc_set = TestUtils_doc_set();
+ Indexer *indexer = Indexer_new(schema, (Obj*)folder, NULL, 0);
+ uint32_t i, max;
+
+ CharBuf *field = (CharBuf*)ZCB_WRAP_STR("content", 7);
+ for (i = 0, max = VA_Get_Size(doc_set); i < max; i++) {
+ Doc *doc = Doc_new(NULL, 0);
+ Doc_Store(doc, field, VA_Fetch(doc_set, i));
+ Indexer_Add_Doc(indexer, doc, 1.0f);
+ DECREF(doc);
+ }
+
+ Indexer_Commit(indexer);
+
+ DECREF(doc_set);
+ DECREF(indexer);
+ DECREF(schema);
+
+ return (Folder*)folder;
+}
+
+void
+TestQPLogic_run_tests() {
+ uint32_t i;
+ TestBatch *batch = TestBatch_new(178);
+ Folder *folder = S_create_index();
+ IndexSearcher *searcher = IxSearcher_new((Obj*)folder);
+ QueryParser *or_parser = QParser_new(IxSearcher_Get_Schema(searcher),
+ NULL, NULL, NULL);
+ ZombieCharBuf *AND = ZCB_WRAP_STR("AND", 3);
+ QueryParser *and_parser = QParser_new(IxSearcher_Get_Schema(searcher),
+ NULL, (CharBuf*)AND, NULL);
+ QParser_Set_Heed_Colons(or_parser, true);
+ QParser_Set_Heed_Colons(and_parser, true);
+
+ TestBatch_Plan(batch);
+
+ // Run logical tests with default boolop of OR.
+ for (i = 0; logical_test_funcs[i] != NULL; i++) {
+ lucy_TestQPLogic_logical_test_t test_func = logical_test_funcs[i];
+ TestQueryParser *test_case = test_func(BOOLOP_OR);
+ Query *tree = QParser_Tree(or_parser, test_case->query_string);
+ Query *parsed = QParser_Parse(or_parser, test_case->query_string);
+ Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
+
+ TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
+ "tree() OR %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
+ "hits: OR %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ DECREF(hits);
+ DECREF(parsed);
+ DECREF(tree);
+ DECREF(test_case);
+ }
+
+ // Run logical tests with default boolop of AND.
+ for (i = 0; logical_test_funcs[i] != NULL; i++) {
+ lucy_TestQPLogic_logical_test_t test_func = logical_test_funcs[i];
+ TestQueryParser *test_case = test_func(BOOLOP_AND);
+ Query *tree = QParser_Tree(and_parser, test_case->query_string);
+ Query *parsed = QParser_Parse(and_parser, test_case->query_string);
+ Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
+
+ TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
+ "tree() AND %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
+ "hits: AND %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ DECREF(hits);
+ DECREF(parsed);
+ DECREF(tree);
+ DECREF(test_case);
+ }
+
+ // Run tests for QParser_Prune().
+ for (i = 0; prune_test_funcs[i] != NULL; i++) {
+ lucy_TestQPLogic_prune_test_t test_func = prune_test_funcs[i];
+ TestQueryParser *test_case = test_func();
+ CharBuf *qstring = test_case->tree
+ ? Query_To_String(test_case->tree)
+ : CB_new_from_trusted_utf8("(NULL)", 6);
+ Query *tree = test_case->tree;
+ Query *wanted = test_case->expanded;
+ Query *pruned = QParser_Prune(or_parser, tree);
+ Query *expanded;
+ Hits *hits;
+
+ TEST_TRUE(batch, Query_Equals(pruned, (Obj*)wanted),
+ "prune() %s", (char*)CB_Get_Ptr8(qstring));
+ expanded = QParser_Expand(or_parser, pruned);
+ hits = IxSearcher_Hits(searcher, (Obj*)expanded, 0, 10, NULL);
+ TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
+ "hits: %s", (char*)CB_Get_Ptr8(qstring));
+
+ DECREF(hits);
+ DECREF(expanded);
+ DECREF(pruned);
+ DECREF(qstring);
+ DECREF(test_case);
+ }
+
+ DECREF(and_parser);
+ DECREF(or_parser);
+ DECREF(searcher);
+ DECREF(folder);
+ DECREF(batch);
+}
+
diff --git a/core/Lucy/Test/Search/TestQueryParserLogic.cfh b/core/Lucy/Test/Search/TestQueryParserLogic.cfh
new file mode 100644
index 0000000..8732646
--- /dev/null
+++ b/core/Lucy/Test/Search/TestQueryParserLogic.cfh
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Tests for logical structure of Query objects output by QueryParser.
+ */
+
+inert class Lucy::Test::Search::TestQueryParserLogic cnick TestQPLogic {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestQueryParserSyntax.c b/core/Lucy/Test/Search/TestQueryParserSyntax.c
new file mode 100644
index 0000000..dd2efd0
--- /dev/null
+++ b/core/Lucy/Test/Search/TestQueryParserSyntax.c
@@ -0,0 +1,344 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTQUERYPARSERSYNTAX
+#define C_LUCY_TESTQUERYPARSER
+#include "Lucy/Util/ToolSet.h"
+#include <stdarg.h>
+#include <string.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Search/TestQueryParserSyntax.h"
+#include "Lucy/Test/Search/TestQueryParser.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Search/Hits.h"
+#include "Lucy/Search/IndexSearcher.h"
+#include "Lucy/Search/QueryParser.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Search/PhraseQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Search/ORQuery.h"
+#include "Lucy/Store/Folder.h"
+
+#define make_term_query (Query*)lucy_TestUtils_make_term_query
+#define make_phrase_query (Query*)lucy_TestUtils_make_phrase_query
+#define make_leaf_query (Query*)lucy_TestUtils_make_leaf_query
+#define make_not_query (Query*)lucy_TestUtils_make_not_query
+#define make_poly_query (Query*)lucy_TestUtils_make_poly_query
+
+static TestQueryParser*
+leaf_test_simple_term() {
+ Query *tree = make_leaf_query(NULL, "a");
+ Query *plain_q = make_term_query("plain", "a");
+ Query *fancy_q = make_term_query("fancy", "a");
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("a", tree, expanded, 4);
+}
+
+static TestQueryParser*
+leaf_test_simple_phrase() {
+ Query *tree = make_leaf_query(NULL, "\"a b\"");
+ Query *plain_q = make_phrase_query("plain", "a", "b", NULL);
+ Query *fancy_q = make_phrase_query("fancy", "a", "b", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"a b\"", tree, expanded, 3);
+}
+
+static TestQueryParser*
+leaf_test_unclosed_quote() {
+ Query *tree = make_leaf_query(NULL, "\"a b");
+ Query *plain_q = make_phrase_query("plain", "a", "b", NULL);
+ Query *fancy_q = make_phrase_query("fancy", "a", "b", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"a b", tree, expanded, 3);
+}
+
+static TestQueryParser*
+leaf_test_escaped_quotes_inside() {
+ Query *tree = make_leaf_query(NULL, "\"\\\"a b\\\"\"");
+ Query *plain_q = make_phrase_query("plain", "\"a", "b\"", NULL);
+ Query *fancy_q = make_phrase_query("fancy", "a", "b", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"\\\"a b\\\"\"", tree, expanded, 3);
+}
+
+static TestQueryParser*
+leaf_test_escaped_quotes_outside() {
+ Query *tree = make_leaf_query(NULL, "\\\"a");
+ Query *plain_q = make_term_query("plain", "\"a");
+ Query *fancy_q = make_term_query("fancy", "a");
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\\\"a", tree, expanded, 4);
+}
+
+static TestQueryParser*
+leaf_test_single_term_phrase() {
+ Query *tree = make_leaf_query(NULL, "\"a\"");
+ Query *plain_q = make_phrase_query("plain", "a", NULL);
+ Query *fancy_q = make_phrase_query("fancy", "a", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"a\"", tree, expanded, 4);
+}
+
+static TestQueryParser*
+leaf_test_longer_phrase() {
+ Query *tree = make_leaf_query(NULL, "\"a b c\"");
+ Query *plain_q = make_phrase_query("plain", "a", "b", "c", NULL);
+ Query *fancy_q = make_phrase_query("fancy", "a", "b", "c", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"a b c\"", tree, expanded, 2);
+}
+
+static TestQueryParser*
+leaf_test_empty_phrase() {
+ Query *tree = make_leaf_query(NULL, "\"\"");
+ Query *plain_q = make_phrase_query("plain", NULL);
+ Query *fancy_q = make_phrase_query("fancy", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"\"", tree, expanded, 0);
+}
+
+static TestQueryParser*
+leaf_test_phrase_with_stopwords() {
+ Query *tree = make_leaf_query(NULL, "\"x a\"");
+ Query *plain_q = make_phrase_query("plain", "x", "a", NULL);
+ Query *fancy_q = make_phrase_query("fancy", "a", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("\"x a\"", tree, expanded, 4);
+}
+
+static TestQueryParser*
+leaf_test_different_tokenization() {
+ Query *tree = make_leaf_query(NULL, "a.b");
+ Query *plain_q = make_term_query("plain", "a.b");
+ Query *fancy_q = make_phrase_query("fancy", "a", "b", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new("a.b", tree, expanded, 3);
+}
+
+static TestQueryParser*
+leaf_test_http() {
+ char address[] = "http://www.foo.com/bar.html";
+ Query *tree = make_leaf_query(NULL, address);
+ Query *plain_q = make_term_query("plain", address);
+ Query *fancy_q = make_phrase_query("fancy", "http", "www", "foo",
+ "com", "bar", "html", NULL);
+ Query *expanded = make_poly_query(BOOLOP_OR, fancy_q, plain_q, NULL);
+ return TestQP_new(address, tree, expanded, 0);
+}
+
+static TestQueryParser*
+leaf_test_field() {
+ Query *tree = make_leaf_query("plain", "b");
+ Query *expanded = make_term_query("plain", "b");
+ return TestQP_new("plain:b", tree, expanded, 3);
+}
+
+static TestQueryParser*
+leaf_test_unrecognized_field() {
+ Query *tree = make_leaf_query("bogusfield", "b");
+ Query *expanded = make_term_query("bogusfield", "b");
+ return TestQP_new("bogusfield:b", tree, expanded, 0);
+}
+
+static TestQueryParser*
+leaf_test_unescape_colons() {
+ Query *tree = make_leaf_query("plain", "a\\:b");
+ Query *expanded = make_term_query("plain", "a:b");
+ return TestQP_new("plain:a\\:b", tree, expanded, 0);
+}
+
+static TestQueryParser*
+syntax_test_minus_plus() {
+ Query *leaf = make_leaf_query(NULL, "a");
+ Query *tree = make_not_query(leaf);
+ return TestQP_new("-+a", tree, NULL, 0);
+}
+
+static TestQueryParser*
+syntax_test_plus_minus() {
+ // Not a perfect result, but then it's not a good query string.
+ Query *leaf = make_leaf_query(NULL, "a");
+ Query *tree = make_not_query(leaf);
+ return TestQP_new("+-a", tree, NULL, 0);
+}
+
+static TestQueryParser*
+syntax_test_minus_minus() {
+ // Not a perfect result, but then it's not a good query string.
+ Query *tree = make_leaf_query(NULL, "a");
+ return TestQP_new("--a", tree, NULL, 4);
+}
+
+static TestQueryParser*
+syntax_test_not_minus() {
+ Query *tree = make_leaf_query(NULL, "a");
+ return TestQP_new("NOT -a", tree, NULL, 4);
+}
+
+static TestQueryParser*
+syntax_test_not_plus() {
+ // Not a perfect result, but then it's not a good query string.
+ Query *leaf = make_leaf_query(NULL, "a");
+ Query *tree = make_not_query(leaf);
+ return TestQP_new("NOT +a", tree, NULL, 0);
+}
+
+static TestQueryParser*
+syntax_test_padded_plus() {
+ Query *plus = make_leaf_query(NULL, "+");
+ Query *a = make_leaf_query(NULL, "a");
+ Query *tree = make_poly_query(BOOLOP_OR, plus, a, NULL);
+ return TestQP_new("+ a", tree, NULL, 4);
+}
+
+static TestQueryParser*
+syntax_test_padded_minus() {
+ Query *minus = make_leaf_query(NULL, "-");
+ Query *a = make_leaf_query(NULL, "a");
+ Query *tree = make_poly_query(BOOLOP_OR, minus, a, NULL);
+ return TestQP_new("- a", tree, NULL, 4);
+}
+
+static TestQueryParser*
+syntax_test_unclosed_parens() {
+ // Not a perfect result, but then it's not a good query string.
+ Query *inner = make_poly_query(BOOLOP_OR, NULL);
+ Query *tree = make_poly_query(BOOLOP_OR, inner, NULL);
+ return TestQP_new("((", tree, NULL, 0);
+}
+
+static TestQueryParser*
+syntax_test_escaped_quotes_outside() {
+ Query *tree = make_leaf_query(NULL, "\\\"a\\\"");
+ return TestQP_new("\\\"a\\\"", tree, NULL, 4);
+}
+
+static TestQueryParser*
+syntax_test_escaped_quotes_inside() {
+ Query *tree = make_leaf_query(NULL, "\"\\\"a\\\"\"");
+ return TestQP_new("\"\\\"a\\\"\"", tree, NULL, 4);
+}
+
+static TestQueryParser*
+syntax_test_identifier_field_name() {
+ // Field names must be identifiers, i.e. they cannot start with a number.
+ Query *tree = make_leaf_query(NULL, "10:30");
+ return TestQP_new("10:30", tree, NULL, 0);
+}
+
+static TestQueryParser*
+syntax_test_double_colon() {
+ Query *tree = make_leaf_query(NULL, "PHP::Interpreter");
+ return TestQP_new("PHP::Interpreter", tree, NULL, 0);
+}
+
+/***************************************************************************/
+
+typedef TestQueryParser*
+(*lucy_TestQPSyntax_test_t)();
+
+static lucy_TestQPSyntax_test_t leaf_test_funcs[] = {
+ leaf_test_simple_term,
+ leaf_test_simple_phrase,
+ leaf_test_unclosed_quote,
+ leaf_test_escaped_quotes_inside,
+ leaf_test_escaped_quotes_outside,
+ leaf_test_single_term_phrase,
+ leaf_test_longer_phrase,
+ leaf_test_empty_phrase,
+ leaf_test_different_tokenization,
+ leaf_test_phrase_with_stopwords,
+ leaf_test_http,
+ leaf_test_field,
+ leaf_test_unrecognized_field,
+ leaf_test_unescape_colons,
+ NULL
+};
+
+static lucy_TestQPSyntax_test_t syntax_test_funcs[] = {
+ syntax_test_minus_plus,
+ syntax_test_plus_minus,
+ syntax_test_minus_minus,
+ syntax_test_not_minus,
+ syntax_test_not_plus,
+ syntax_test_padded_plus,
+ syntax_test_padded_minus,
+ syntax_test_unclosed_parens,
+ syntax_test_escaped_quotes_outside,
+ syntax_test_escaped_quotes_inside,
+ syntax_test_identifier_field_name,
+ syntax_test_double_colon,
+ NULL
+};
+
+void
+TestQPSyntax_run_tests(Folder *index) {
+ uint32_t i;
+ TestBatch *batch = TestBatch_new(66);
+ IndexSearcher *searcher = IxSearcher_new((Obj*)index);
+ QueryParser *qparser = QParser_new(IxSearcher_Get_Schema(searcher),
+ NULL, NULL, NULL);
+ QParser_Set_Heed_Colons(qparser, true);
+
+ TestBatch_Plan(batch);
+
+ for (i = 0; leaf_test_funcs[i] != NULL; i++) {
+ lucy_TestQPSyntax_test_t test_func = leaf_test_funcs[i];
+ TestQueryParser *test_case = test_func();
+ Query *tree = QParser_Tree(qparser, test_case->query_string);
+ Query *expanded = QParser_Expand_Leaf(qparser, test_case->tree);
+ Query *parsed = QParser_Parse(qparser, test_case->query_string);
+ Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
+
+ TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
+ "tree() %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ TEST_TRUE(batch, Query_Equals(expanded, (Obj*)test_case->expanded),
+ "expand_leaf() %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
+ "hits: %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ DECREF(hits);
+ DECREF(parsed);
+ DECREF(expanded);
+ DECREF(tree);
+ DECREF(test_case);
+ }
+
+ for (i = 0; syntax_test_funcs[i] != NULL; i++) {
+ lucy_TestQPSyntax_test_t test_func = syntax_test_funcs[i];
+ TestQueryParser *test_case = test_func();
+ Query *tree = QParser_Tree(qparser, test_case->query_string);
+ Query *parsed = QParser_Parse(qparser, test_case->query_string);
+ Hits *hits = IxSearcher_Hits(searcher, (Obj*)parsed, 0, 10, NULL);
+
+ TEST_TRUE(batch, Query_Equals(tree, (Obj*)test_case->tree),
+ "tree() %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ TEST_INT_EQ(batch, Hits_Total_Hits(hits), test_case->num_hits,
+ "hits: %s", (char*)CB_Get_Ptr8(test_case->query_string));
+ DECREF(hits);
+ DECREF(parsed);
+ DECREF(tree);
+ DECREF(test_case);
+ }
+
+ DECREF(batch);
+ DECREF(searcher);
+ DECREF(qparser);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestQueryParserSyntax.cfh b/core/Lucy/Test/Search/TestQueryParserSyntax.cfh
new file mode 100644
index 0000000..f6a2a18
--- /dev/null
+++ b/core/Lucy/Test/Search/TestQueryParserSyntax.cfh
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Tests for logical structure of Query objects output by QueryParser.
+ */
+
+inert class Lucy::Test::Search::TestQueryParserSyntax cnick TestQPSyntax {
+ inert void
+ run_tests(Folder *index);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestRangeQuery.c b/core/Lucy/Test/Search/TestRangeQuery.c
new file mode 100644
index 0000000..3004b11
--- /dev/null
+++ b/core/Lucy/Test/Search/TestRangeQuery.c
@@ -0,0 +1,70 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTRANGEQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestRangeQuery.h"
+#include "Lucy/Search/RangeQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ RangeQuery *query
+ = TestUtils_make_range_query("content", "foo", "phooey", true, true);
+ RangeQuery *lo_term_differs
+ = TestUtils_make_range_query("content", "goo", "phooey", true, true);
+ RangeQuery *hi_term_differs
+ = TestUtils_make_range_query("content", "foo", "gooey", true, true);
+ RangeQuery *include_lower_differs
+ = TestUtils_make_range_query("content", "foo", "phooey", false, true);
+ RangeQuery *include_upper_differs
+ = TestUtils_make_range_query("content", "foo", "phooey", true, false);
+ Obj *dump = (Obj*)RangeQuery_Dump(query);
+ RangeQuery *clone = (RangeQuery*)RangeQuery_Load(lo_term_differs, dump);
+
+ TEST_FALSE(batch, RangeQuery_Equals(query, (Obj*)lo_term_differs),
+ "Equals() false with different lower term");
+ TEST_FALSE(batch, RangeQuery_Equals(query, (Obj*)hi_term_differs),
+ "Equals() false with different upper term");
+ TEST_FALSE(batch, RangeQuery_Equals(query, (Obj*)include_lower_differs),
+ "Equals() false with different include_lower");
+ TEST_FALSE(batch, RangeQuery_Equals(query, (Obj*)include_upper_differs),
+ "Equals() false with different include_upper");
+ TEST_TRUE(batch, RangeQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(query);
+ DECREF(lo_term_differs);
+ DECREF(hi_term_differs);
+ DECREF(include_lower_differs);
+ DECREF(include_upper_differs);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+
+void
+TestRangeQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(5);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestRangeQuery.cfh b/core/Lucy/Test/Search/TestRangeQuery.cfh
new file mode 100644
index 0000000..8cf247f
--- /dev/null
+++ b/core/Lucy/Test/Search/TestRangeQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestRangeQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestReqOptQuery.c b/core/Lucy/Test/Search/TestReqOptQuery.c
new file mode 100644
index 0000000..d5dd329
--- /dev/null
+++ b/core/Lucy/Test/Search/TestReqOptQuery.c
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTREQOPTQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Search/TestReqOptQuery.h"
+#include "Lucy/Search/RequiredOptionalQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ Query *a_leaf = (Query*)TestUtils_make_leaf_query(NULL, "a");
+ Query *b_leaf = (Query*)TestUtils_make_leaf_query(NULL, "b");
+ Query *c_leaf = (Query*)TestUtils_make_leaf_query(NULL, "c");
+ RequiredOptionalQuery *query = ReqOptQuery_new(a_leaf, b_leaf);
+ RequiredOptionalQuery *kids_differ = ReqOptQuery_new(a_leaf, c_leaf);
+ RequiredOptionalQuery *boost_differs = ReqOptQuery_new(a_leaf, b_leaf);
+ Obj *dump = (Obj*)ReqOptQuery_Dump(query);
+ RequiredOptionalQuery *clone
+ = (RequiredOptionalQuery*)Obj_Load(dump, dump);
+
+ TEST_FALSE(batch, ReqOptQuery_Equals(query, (Obj*)kids_differ),
+ "Different kids spoil Equals");
+ TEST_TRUE(batch, ReqOptQuery_Equals(query, (Obj*)boost_differs),
+ "Equals with identical boosts");
+ ReqOptQuery_Set_Boost(boost_differs, 1.5);
+ TEST_FALSE(batch, ReqOptQuery_Equals(query, (Obj*)boost_differs),
+ "Different boost spoils Equals");
+ TEST_TRUE(batch, ReqOptQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(a_leaf);
+ DECREF(b_leaf);
+ DECREF(c_leaf);
+ DECREF(query);
+ DECREF(kids_differ);
+ DECREF(boost_differs);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+void
+TestReqOptQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestReqOptQuery.cfh b/core/Lucy/Test/Search/TestReqOptQuery.cfh
new file mode 100644
index 0000000..b20f8d8
--- /dev/null
+++ b/core/Lucy/Test/Search/TestReqOptQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestReqOptQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestSeriesMatcher.c b/core/Lucy/Test/Search/TestSeriesMatcher.c
new file mode 100644
index 0000000..3f13b9f
--- /dev/null
+++ b/core/Lucy/Test/Search/TestSeriesMatcher.c
@@ -0,0 +1,135 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTSERIESMATCHER
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Search/TestSeriesMatcher.h"
+#include "Lucy/Search/BitVecMatcher.h"
+#include "Lucy/Search/SeriesMatcher.h"
+
+static SeriesMatcher*
+S_make_series_matcher(I32Array *doc_ids, I32Array *offsets, int32_t doc_max) {
+ int32_t num_doc_ids = I32Arr_Get_Size(doc_ids);
+ int32_t num_matchers = I32Arr_Get_Size(offsets);
+ VArray *matchers = VA_new(num_matchers);
+ int32_t tick = 0;
+ int32_t i;
+
+ // Divvy up doc_ids by segment into BitVectors.
+ for (i = 0; i < num_matchers; i++) {
+ int32_t offset = I32Arr_Get(offsets, i);
+ int32_t max = i == num_matchers - 1
+ ? doc_max + 1
+ : I32Arr_Get(offsets, i + 1);
+ BitVector *bit_vec = BitVec_new(max - offset);
+ while (tick < num_doc_ids) {
+ int32_t doc_id = I32Arr_Get(doc_ids, tick);
+ if (doc_id > max) { break; }
+ else { tick++; }
+ BitVec_Set(bit_vec, doc_id - offset);
+ }
+ VA_Push(matchers, (Obj*)BitVecMatcher_new(bit_vec));
+ DECREF(bit_vec);
+ }
+
+ {
+ SeriesMatcher *series_matcher = SeriesMatcher_new(matchers, offsets);
+ DECREF(matchers);
+ return series_matcher;
+ }
+}
+
+static I32Array*
+S_generate_match_list(int32_t first, int32_t max, int32_t doc_inc) {
+ int32_t count = (int32_t)ceil(((float)max - first) / doc_inc);
+ int32_t *doc_ids = (int32_t*)MALLOCATE(count * sizeof(int32_t));
+ int32_t doc_id = first;
+ int32_t i = 0;
+
+ for (; doc_id < max; doc_id += doc_inc, i++) {
+ doc_ids[i] = doc_id;
+ }
+ if (i != count) { THROW(ERR, "Screwed up somehow: %i32 %i32", i, count); }
+
+ return I32Arr_new_steal(doc_ids, count);
+}
+
+static void
+S_do_test_matrix(TestBatch *batch, int32_t doc_max, int32_t first_doc_id,
+ int32_t doc_inc, int32_t offset_inc) {
+ I32Array *doc_ids
+ = S_generate_match_list(first_doc_id, doc_max, doc_inc);
+ I32Array *offsets
+ = S_generate_match_list(0, doc_max, offset_inc);
+ SeriesMatcher *series_matcher
+ = S_make_series_matcher(doc_ids, offsets, doc_max);
+ uint32_t num_in_agreement = 0;
+ int32_t got;
+
+ while (0 != (got = SeriesMatcher_Next(series_matcher))) {
+ if (got != I32Arr_Get(doc_ids, num_in_agreement)) { break; }
+ num_in_agreement++;
+ }
+ TEST_INT_EQ(batch, num_in_agreement, I32Arr_Get_Size(doc_ids),
+ "doc_max=%d first_doc_id=%d doc_inc=%d offset_inc=%d",
+ doc_max, first_doc_id, doc_inc, offset_inc);
+
+ DECREF(doc_ids);
+ DECREF(offsets);
+ DECREF(series_matcher);
+}
+
+static void
+test_matrix(TestBatch *batch) {
+ int32_t doc_max_nums[] = { 10, 100, 1000, 0 };
+ int32_t first_doc_ids[] = { 1, 2, 10, 0 };
+ int32_t doc_inc_nums[] = { 20, 13, 9, 4, 2, 0 };
+ int32_t offset_inc_nums[] = { 7, 29, 71, 0 };
+ int32_t a, b, c, d;
+
+ for (a = 0; doc_max_nums[a] != 0; a++) {
+ for (b = 0; first_doc_ids[b] != 0; b++) {
+ for (c = 0; doc_inc_nums[c] != 0; c++) {
+ for (d = 0; offset_inc_nums[d] != 0; d++) {
+ int32_t doc_max = doc_max_nums[a];
+ int32_t first_doc_id = first_doc_ids[b];
+ int32_t doc_inc = doc_inc_nums[c];
+ int32_t offset_inc = offset_inc_nums[d];
+ if (first_doc_id > doc_max) {
+ continue;
+ }
+ else {
+ S_do_test_matrix(batch, doc_max, first_doc_id,
+ doc_inc, offset_inc);
+ }
+ }
+ }
+ }
+ }
+}
+
+void
+TestSeriesMatcher_run_tests() {
+ TestBatch *batch = TestBatch_new(135);
+ TestBatch_Plan(batch);
+ test_matrix(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestSeriesMatcher.cfh b/core/Lucy/Test/Search/TestSeriesMatcher.cfh
new file mode 100644
index 0000000..2b7729c
--- /dev/null
+++ b/core/Lucy/Test/Search/TestSeriesMatcher.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestSeriesMatcher {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Search/TestTermQuery.c b/core/Lucy/Test/Search/TestTermQuery.c
new file mode 100644
index 0000000..8fd3738
--- /dev/null
+++ b/core/Lucy/Test/Search/TestTermQuery.c
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTTERMQUERY
+#include "Lucy/Util/ToolSet.h"
+#include <math.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Search/TestTermQuery.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Search/TermQuery.h"
+
+static void
+test_Dump_Load_and_Equals(TestBatch *batch) {
+ TermQuery *query = TestUtils_make_term_query("content", "foo");
+ TermQuery *field_differs = TestUtils_make_term_query("stuff", "foo");
+ TermQuery *term_differs = TestUtils_make_term_query("content", "bar");
+ TermQuery *boost_differs = TestUtils_make_term_query("content", "foo");
+ Obj *dump = (Obj*)TermQuery_Dump(query);
+ TermQuery *clone = (TermQuery*)TermQuery_Load(term_differs, dump);
+
+ TEST_FALSE(batch, TermQuery_Equals(query, (Obj*)field_differs),
+ "Equals() false with different field");
+ TEST_FALSE(batch, TermQuery_Equals(query, (Obj*)term_differs),
+ "Equals() false with different term");
+ TermQuery_Set_Boost(boost_differs, 0.5);
+ TEST_FALSE(batch, TermQuery_Equals(query, (Obj*)boost_differs),
+ "Equals() false with different boost");
+ TEST_TRUE(batch, TermQuery_Equals(query, (Obj*)clone),
+ "Dump => Load round trip");
+
+ DECREF(query);
+ DECREF(term_differs);
+ DECREF(field_differs);
+ DECREF(boost_differs);
+ DECREF(dump);
+ DECREF(clone);
+}
+
+void
+TestTermQuery_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ TestBatch_Plan(batch);
+ test_Dump_Load_and_Equals(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Search/TestTermQuery.cfh b/core/Lucy/Test/Search/TestTermQuery.cfh
new file mode 100644
index 0000000..2b8a031
--- /dev/null
+++ b/core/Lucy/Test/Search/TestTermQuery.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Search::TestTermQuery {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/MockFileHandle.c b/core/Lucy/Test/Store/MockFileHandle.c
new file mode 100644
index 0000000..4551499
--- /dev/null
+++ b/core/Lucy/Test/Store/MockFileHandle.c
@@ -0,0 +1,64 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MOCKFILEHANDLE
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test/Store/MockFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+
+MockFileHandle*
+MockFileHandle_new(const CharBuf *path, int64_t length) {
+ MockFileHandle *self = (MockFileHandle*)VTable_Make_Obj(MOCKFILEHANDLE);
+ return MockFileHandle_init(self, path, length);
+}
+
+MockFileHandle*
+MockFileHandle_init(MockFileHandle *self, const CharBuf *path,
+ int64_t length) {
+ FH_do_open((FileHandle*)self, path, 0);
+ self->len = length;
+ return self;
+}
+
+bool_t
+MockFileHandle_window(MockFileHandle *self, FileWindow *window,
+ int64_t offset, int64_t len) {
+ UNUSED_VAR(self);
+ FileWindow_Set_Window(window, NULL, offset, len);
+ return true;
+}
+
+bool_t
+MockFileHandle_release_window(MockFileHandle *self, FileWindow *window) {
+ UNUSED_VAR(self);
+ FileWindow_Set_Window(window, NULL, 0, 0);
+ return true;
+}
+
+int64_t
+MockFileHandle_length(MockFileHandle *self) {
+ return self->len;
+}
+
+bool_t
+MockFileHandle_close(MockFileHandle *self) {
+ UNUSED_VAR(self);
+ return true;
+}
+
+
diff --git a/core/Lucy/Test/Store/MockFileHandle.cfh b/core/Lucy/Test/Store/MockFileHandle.cfh
new file mode 100644
index 0000000..980466e
--- /dev/null
+++ b/core/Lucy/Test/Store/MockFileHandle.cfh
@@ -0,0 +1,44 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Mock-object FileHandle for testing InStream/OutStream.
+ */
+class Lucy::Store::MockFileHandle inherits Lucy::Store::FileHandle {
+
+ int64_t len;
+
+ inert incremented MockFileHandle*
+ new(const CharBuf *path = NULL, int64_t length);
+
+ inert MockFileHandle*
+ init(MockFileHandle *self, const CharBuf *path = NULL, int64_t length);
+
+ bool_t
+ Window(MockFileHandle *self, FileWindow *window, int64_t offset, int64_t len);
+
+ bool_t
+ Release_Window(MockFileHandle *self, FileWindow *window);
+
+ int64_t
+ Length(MockFileHandle *self);
+
+ bool_t
+ Close(MockFileHandle *self);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestCompoundFileReader.c b/core/Lucy/Test/Store/TestCompoundFileReader.c
new file mode 100644
index 0000000..feebeb9
--- /dev/null
+++ b/core/Lucy/Test/Store/TestCompoundFileReader.c
@@ -0,0 +1,350 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ZOMBIECHARBUF
+#define C_LUCY_RAMFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestCompoundFileReader.h"
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Util/Json.h"
+
+static ZombieCharBuf cfmeta_file = ZCB_LITERAL("cfmeta.json");
+static ZombieCharBuf cf_file = ZCB_LITERAL("cf.dat");
+static ZombieCharBuf foo = ZCB_LITERAL("foo");
+static ZombieCharBuf bar = ZCB_LITERAL("bar");
+static ZombieCharBuf baz = ZCB_LITERAL("baz");
+static ZombieCharBuf stuff = ZCB_LITERAL("stuff");
+
+static Folder*
+S_folder_with_contents() {
+ ZombieCharBuf seg_1 = ZCB_LITERAL("seg_1");
+ RAMFolder *folder = RAMFolder_new((CharBuf*)&seg_1);
+ OutStream *foo_out = RAMFolder_Open_Out(folder, (CharBuf*)&foo);
+ OutStream *bar_out = RAMFolder_Open_Out(folder, (CharBuf*)&bar);
+ OutStream_Write_Bytes(foo_out, "foo", 3);
+ OutStream_Write_Bytes(bar_out, "bar", 3);
+ OutStream_Close(foo_out);
+ OutStream_Close(bar_out);
+ DECREF(foo_out);
+ DECREF(bar_out);
+ RAMFolder_Consolidate(folder, (CharBuf*)&EMPTY);
+ return (Folder*)folder;
+}
+
+static void
+test_open(TestBatch *batch) {
+ Folder *real_folder;
+ CompoundFileReader *cf_reader;
+ Hash *metadata;
+
+ Err_set_error(NULL);
+ real_folder = S_folder_with_contents();
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ TEST_TRUE(batch, cf_reader == NULL,
+ "Return NULL when cfmeta file missing");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when cfmeta file missing");
+ DECREF(real_folder);
+
+ Err_set_error(NULL);
+ real_folder = S_folder_with_contents();
+ Folder_Delete(real_folder, (CharBuf*)&cf_file);
+ cf_reader = CFReader_open(real_folder);
+ TEST_TRUE(batch, cf_reader == NULL,
+ "Return NULL when cf.dat file missing");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when cf.dat file missing");
+ DECREF(real_folder);
+
+ Err_set_error(NULL);
+ real_folder = S_folder_with_contents();
+ metadata = (Hash*)Json_slurp_json(real_folder, (CharBuf*)&cfmeta_file);
+ Hash_Store_Str(metadata, "format", 6, (Obj*)CB_newf("%i32", -1));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ TEST_TRUE(batch, cf_reader == NULL,
+ "Return NULL when format is invalid");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when format is invalid");
+
+ Err_set_error(NULL);
+ Hash_Store_Str(metadata, "format", 6, (Obj*)CB_newf("%i32", 1000));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ TEST_TRUE(batch, cf_reader == NULL,
+ "Return NULL when format is too recent");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when format too recent");
+
+ Err_set_error(NULL);
+ DECREF(Hash_Delete_Str(metadata, "format", 6));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ TEST_TRUE(batch, cf_reader == NULL,
+ "Return NULL when format key is missing");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when format key is missing");
+
+ Hash_Store_Str(metadata, "format", 6,
+ (Obj*)CB_newf("%i32", CFWriter_current_file_format));
+ DECREF(Hash_Delete_Str(metadata, "files", 5));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ TEST_TRUE(batch, cf_reader == NULL,
+ "Return NULL when files key is missing");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when files key is missing");
+
+ DECREF(metadata);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_MkDir_and_Find_Folder(TestBatch *batch) {
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+
+ TEST_FALSE(batch,
+ CFReader_Local_Is_Directory(cf_reader, (CharBuf*)&stuff),
+ "Local_Is_Directory returns false for non-existent entry");
+
+ TEST_TRUE(batch, CFReader_MkDir(cf_reader, (CharBuf*)&stuff),
+ "MkDir returns true");
+ TEST_TRUE(batch,
+ Folder_Find_Folder(real_folder, (CharBuf*)&stuff) != NULL,
+ "Local_MkDir pass-through");
+ TEST_TRUE(batch,
+ Folder_Find_Folder(real_folder, (CharBuf*)&stuff)
+ == CFReader_Find_Folder(cf_reader, (CharBuf*)&stuff),
+ "Local_Find_Folder pass-through");
+ TEST_TRUE(batch,
+ CFReader_Local_Is_Directory(cf_reader, (CharBuf*)&stuff),
+ "Local_Is_Directory pass through");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, CFReader_MkDir(cf_reader, (CharBuf*)&stuff),
+ "MkDir returns false when dir already exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "MkDir sets Err_error when dir already exists");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, CFReader_MkDir(cf_reader, (CharBuf*)&foo),
+ "MkDir returns false when virtual file exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "MkDir sets Err_error when virtual file exists");
+
+ TEST_TRUE(batch,
+ CFReader_Find_Folder(cf_reader, (CharBuf*)&foo) == NULL,
+ "Virtual file not reported as directory");
+ TEST_FALSE(batch, CFReader_Local_Is_Directory(cf_reader, (CharBuf*)&foo),
+ "Local_Is_Directory returns false for virtual file");
+
+ DECREF(real_folder);
+ DECREF(cf_reader);
+}
+
+static void
+test_Local_Delete_and_Exists(TestBatch *batch) {
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+
+ CFReader_MkDir(cf_reader, (CharBuf*)&stuff);
+ TEST_TRUE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&stuff),
+ "pass through for Local_Exists");
+ TEST_TRUE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&foo),
+ "Local_Exists returns true for virtual file");
+
+ TEST_TRUE(batch,
+ CFReader_Local_Exists(cf_reader, (CharBuf*)&cfmeta_file),
+ "cfmeta file exists");
+
+ TEST_TRUE(batch, CFReader_Local_Delete(cf_reader, (CharBuf*)&stuff),
+ "Local_Delete returns true when zapping real entity");
+ TEST_FALSE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&stuff),
+ "Local_Exists returns false after real entity zapped");
+
+ TEST_TRUE(batch, CFReader_Local_Delete(cf_reader, (CharBuf*)&foo),
+ "Local_Delete returns true when zapping virtual file");
+ TEST_FALSE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&foo),
+ "Local_Exists returns false after virtual file zapped");
+
+ TEST_TRUE(batch, CFReader_Local_Delete(cf_reader, (CharBuf*)&bar),
+ "Local_Delete returns true when zapping last virtual file");
+ TEST_FALSE(batch,
+ CFReader_Local_Exists(cf_reader, (CharBuf*)&cfmeta_file),
+ "cfmeta file deleted when last virtual file deleted");
+ TEST_FALSE(batch,
+ CFReader_Local_Exists(cf_reader, (CharBuf*)&cf_file),
+ "compound data file deleted when last virtual file deleted");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_Open_Dir(TestBatch *batch) {
+
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+ DirHandle *dh;
+ CharBuf *entry;
+ bool_t saw_foo = false;
+ bool_t saw_stuff = false;
+ bool_t stuff_was_dir = false;
+
+ CFReader_MkDir(cf_reader, (CharBuf*)&stuff);
+
+ dh = CFReader_Local_Open_Dir(cf_reader);
+ entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ if (CB_Equals(entry, (Obj*)&foo)) {
+ saw_foo = true;
+ }
+ else if (CB_Equals(entry, (Obj*)&stuff)) {
+ saw_stuff = true;
+ stuff_was_dir = DH_Entry_Is_Dir(dh);
+ }
+ }
+
+ TEST_TRUE(batch, saw_foo, "DirHandle iterated over virtual file");
+ TEST_TRUE(batch, saw_stuff, "DirHandle iterated over real directory");
+ TEST_TRUE(batch, stuff_was_dir,
+ "DirHandle knew that real entry was dir");
+
+ DECREF(dh);
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_Open_FileHandle(TestBatch *batch) {
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+ FileHandle *fh;
+
+ {
+ OutStream *outstream = CFReader_Open_Out(cf_reader, (CharBuf*)&baz);
+ OutStream_Write_Bytes(outstream, "baz", 3);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ }
+
+ fh = CFReader_Local_Open_FileHandle(cf_reader, (CharBuf*)&baz,
+ FH_READ_ONLY);
+ TEST_TRUE(batch, fh != NULL,
+ "Local_Open_FileHandle pass-through for real file");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = CFReader_Local_Open_FileHandle(cf_reader, (CharBuf*)&stuff,
+ FH_READ_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "Local_Open_FileHandle for non-existent file returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_Open_FileHandle for non-existent file sets Err_error");
+
+ Err_set_error(NULL);
+ fh = CFReader_Local_Open_FileHandle(cf_reader, (CharBuf*)&foo,
+ FH_READ_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "Local_Open_FileHandle for virtual file returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_Open_FileHandle for virtual file sets Err_error");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_Open_In(TestBatch *batch) {
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+ InStream *instream;
+
+ instream = CFReader_Local_Open_In(cf_reader, (CharBuf*)&foo);
+ TEST_TRUE(batch, instream != NULL,
+ "Local_Open_In for virtual file");
+ TEST_TRUE(batch,
+ CB_Starts_With(InStream_Get_Filename(instream), CFReader_Get_Path(cf_reader)),
+ "InStream's path includes directory");
+ DECREF(instream);
+
+ {
+ OutStream *outstream = CFReader_Open_Out(cf_reader, (CharBuf*)&baz);
+ OutStream_Write_Bytes(outstream, "baz", 3);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ instream = CFReader_Local_Open_In(cf_reader, (CharBuf*)&baz);
+ TEST_TRUE(batch, instream != NULL,
+ "Local_Open_In pass-through for real file");
+ DECREF(instream);
+ }
+
+ Err_set_error(NULL);
+ instream = CFReader_Local_Open_In(cf_reader, (CharBuf*)&stuff);
+ TEST_TRUE(batch, instream == NULL,
+ "Local_Open_In for non-existent file returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_Open_In for non-existent file sets Err_error");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Close(TestBatch *batch) {
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+
+ CFReader_Close(cf_reader);
+ PASS(batch, "Close completes without incident");
+
+ CFReader_Close(cf_reader);
+ PASS(batch, "Calling Close() multiple times is ok");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+void
+TestCFReader_run_tests() {
+ TestBatch *batch = TestBatch_new(48);
+
+ TestBatch_Plan(batch);
+ test_open(batch);
+ test_Local_MkDir_and_Find_Folder(batch);
+ test_Local_Delete_and_Exists(batch);
+ test_Local_Open_Dir(batch);
+ test_Local_Open_FileHandle(batch);
+ test_Local_Open_In(batch);
+ test_Close(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestCompoundFileReader.cfh b/core/Lucy/Test/Store/TestCompoundFileReader.cfh
new file mode 100644
index 0000000..713f7d7
--- /dev/null
+++ b/core/Lucy/Test/Store/TestCompoundFileReader.cfh
@@ -0,0 +1,25 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestCompoundFileReader
+ cnick TestCFReader {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestCompoundFileWriter.c b/core/Lucy/Test/Store/TestCompoundFileWriter.c
new file mode 100644
index 0000000..bf4fe1a
--- /dev/null
+++ b/core/Lucy/Test/Store/TestCompoundFileWriter.c
@@ -0,0 +1,134 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_CHARBUF
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestCompoundFileWriter.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Util/Json.h"
+
+static CharBuf cfmeta_file = ZCB_LITERAL("cfmeta.json");
+static CharBuf cfmeta_temp = ZCB_LITERAL("cfmeta.json.temp");
+static CharBuf cf_file = ZCB_LITERAL("cf.dat");
+static CharBuf foo = ZCB_LITERAL("foo");
+static CharBuf bar = ZCB_LITERAL("bar");
+static CharBuf seg_1 = ZCB_LITERAL("seg_1");
+
+static Folder*
+S_folder_with_contents() {
+ RAMFolder *folder = RAMFolder_new(&seg_1);
+ OutStream *foo_out = RAMFolder_Open_Out(folder, &foo);
+ OutStream *bar_out = RAMFolder_Open_Out(folder, &bar);
+ OutStream_Write_Bytes(foo_out, "foo", 3);
+ OutStream_Write_Bytes(bar_out, "bar", 3);
+ OutStream_Close(foo_out);
+ OutStream_Close(bar_out);
+ DECREF(foo_out);
+ DECREF(bar_out);
+ return (Folder*)folder;
+}
+
+static void
+test_Consolidate(TestBatch *batch) {
+ Folder *folder = S_folder_with_contents();
+ FileHandle *fh;
+
+ // Fake up detritus from failed consolidation.
+ fh = Folder_Open_FileHandle(folder, &cf_file,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ DECREF(fh);
+ fh = Folder_Open_FileHandle(folder, &cfmeta_temp,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ DECREF(fh);
+
+ {
+ CompoundFileWriter *cf_writer = CFWriter_new(folder);
+ CFWriter_Consolidate(cf_writer);
+ PASS(batch, "Consolidate completes despite leftover files");
+ DECREF(cf_writer);
+ }
+
+ TEST_TRUE(batch, Folder_Exists(folder, &cf_file),
+ "cf.dat file written");
+ TEST_TRUE(batch, Folder_Exists(folder, &cfmeta_file),
+ "cfmeta.json file written");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo),
+ "original file zapped");
+ TEST_FALSE(batch, Folder_Exists(folder, &cfmeta_temp),
+ "detritus from failed consolidation zapped");
+
+ DECREF(folder);
+}
+
+static void
+test_offsets(TestBatch *batch) {
+ Folder *folder = S_folder_with_contents();
+ CompoundFileWriter *cf_writer = CFWriter_new(folder);
+ Hash *cf_metadata;
+ Hash *files;
+
+ CFWriter_Consolidate(cf_writer);
+
+ cf_metadata = (Hash*)CERTIFY(
+ Json_slurp_json(folder, &cfmeta_file), HASH);
+ files = (Hash*)CERTIFY(
+ Hash_Fetch_Str(cf_metadata, "files", 5), HASH);
+ {
+ CharBuf *file;
+ Obj *filestats;
+ bool_t offsets_ok = true;
+
+ TEST_TRUE(batch, Hash_Get_Size(files) > 0, "Multiple files");
+
+ Hash_Iterate(files);
+ while (Hash_Next(files, (Obj**)&file, &filestats)) {
+ Hash *stats = (Hash*)CERTIFY(filestats, HASH);
+ Obj *offset = CERTIFY(Hash_Fetch_Str(stats, "offset", 6), OBJ);
+ int64_t offs = Obj_To_I64(offset);
+ if (offs % 8 != 0) {
+ offsets_ok = false;
+ FAIL(batch, "Offset %" I64P " for %s not a multiple of 8",
+ offset, CB_Get_Ptr8(file));
+ break;
+ }
+ }
+ if (offsets_ok) {
+ PASS(batch, "All offsets are multiples of 8");
+ }
+ }
+
+ DECREF(cf_metadata);
+ DECREF(cf_writer);
+ DECREF(folder);
+}
+
+void
+TestCFWriter_run_tests() {
+ TestBatch *batch = TestBatch_new(7);
+
+ TestBatch_Plan(batch);
+ test_Consolidate(batch);
+ test_offsets(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestCompoundFileWriter.cfh b/core/Lucy/Test/Store/TestCompoundFileWriter.cfh
new file mode 100644
index 0000000..b11462e
--- /dev/null
+++ b/core/Lucy/Test/Store/TestCompoundFileWriter.cfh
@@ -0,0 +1,25 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestCompoundFileWriter
+ cnick TestCFWriter {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFSDirHandle.c b/core/Lucy/Test/Store/TestFSDirHandle.c
new file mode 100644
index 0000000..033a2e6
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFSDirHandle.c
@@ -0,0 +1,100 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+// rmdir
+#ifdef CHY_HAS_DIRECT_H
+ #include <direct.h>
+#endif
+
+// rmdir
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestFSDirHandle.h"
+#include "Lucy/Store/FSDirHandle.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/OutStream.h"
+
+static void
+test_all(TestBatch *batch) {
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+ CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
+ CharBuf *foo_boffo = (CharBuf*)ZCB_WRAP_STR("foo/boffo", 9);
+ CharBuf *test_dir = (CharBuf*)ZCB_WRAP_STR("_fsdir_test", 11);
+ FSFolder *folder = FSFolder_new(test_dir);
+ bool_t saw_foo = false;
+ bool_t saw_boffo = false;
+ bool_t foo_was_dir = false;
+ bool_t boffo_was_dir = false;
+ int count = 0;
+
+ // Clean up after previous failed runs.
+ FSFolder_Delete(folder, foo_boffo);
+ FSFolder_Delete(folder, foo);
+ FSFolder_Delete(folder, boffo);
+ rmdir("_fsdir_test");
+
+ FSFolder_Initialize(folder);
+ FSFolder_MkDir(folder, foo);
+ OutStream *outstream = FSFolder_Open_Out(folder, boffo);
+ DECREF(outstream);
+ outstream = FSFolder_Open_Out(folder, foo_boffo);
+ DECREF(outstream);
+
+ FSDirHandle *dh = FSDH_open(test_dir);
+ CharBuf *entry = FSDH_Get_Entry(dh);
+ while (FSDH_Next(dh)) {
+ count++;
+ if (CB_Equals(entry, (Obj*)foo)) {
+ saw_foo = true;
+ foo_was_dir = FSDH_Entry_Is_Dir(dh);
+ }
+ else if (CB_Equals(entry, (Obj*)boffo)) {
+ saw_boffo = true;
+ boffo_was_dir = FSDH_Entry_Is_Dir(dh);
+ }
+ }
+ TEST_INT_EQ(batch, 2, count, "correct number of entries");
+ TEST_TRUE(batch, saw_foo, "Directory was iterated over");
+ TEST_TRUE(batch, foo_was_dir,
+ "Dir correctly identified by Entry_Is_Dir");
+ TEST_TRUE(batch, saw_boffo, "File was iterated over");
+ TEST_FALSE(batch, boffo_was_dir,
+ "File correctly identified by Entry_Is_Dir");
+
+ DECREF(dh);
+ FSFolder_Delete(folder, foo_boffo);
+ FSFolder_Delete(folder, foo);
+ FSFolder_Delete(folder, boffo);
+ DECREF(folder);
+ rmdir("_fsdir_test");
+}
+
+void
+TestFSDH_run_tests() {
+ TestBatch *batch = TestBatch_new(5);
+
+ TestBatch_Plan(batch);
+ test_all(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFSDirHandle.cfh b/core/Lucy/Test/Store/TestFSDirHandle.cfh
new file mode 100644
index 0000000..1ea21e0
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFSDirHandle.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestFSDirHandle cnick TestFSDH {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFSFileHandle.c b/core/Lucy/Test/Store/TestFSFileHandle.c
new file mode 100644
index 0000000..e3255d9
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFSFileHandle.c
@@ -0,0 +1,257 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h> // for remove()
+
+#define C_LUCY_CHARBUF
+#define C_LUCY_FSFILEHANDLE
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h> // close
+#elif defined(CHY_HAS_IO_H)
+ #include <io.h> // close
+#endif
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestFSFileHandle.h"
+#include "Lucy/Store/FSFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+
+static void
+test_open(TestBatch *batch) {
+
+ FSFileHandle *fh;
+ CharBuf *test_filename = (CharBuf*)ZCB_WRAP_STR("_fstest", 7);
+
+ remove((char*)CB_Get_Ptr8(test_filename));
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_READ_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "open() with FH_READ_ONLY on non-existent file returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "open() with FH_READ_ONLY on non-existent file sets error");
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "open() without FH_CREATE returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "open() without FH_CREATE sets error");
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_CREATE);
+ TEST_TRUE(batch, fh == NULL,
+ "open() without FH_WRITE_ONLY returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "open() without FH_WRITE_ONLY sets error");
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ TEST_TRUE(batch, fh && FSFH_Is_A(fh, FSFILEHANDLE), "open() succeeds");
+ TEST_TRUE(batch, Err_get_error() == NULL, "open() no errors");
+ FSFH_Write(fh, "foo", 3);
+ if (!FSFH_Close(fh)) { RETHROW(INCREF(Err_get_error())); }
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ TEST_TRUE(batch, fh == NULL, "FH_EXCLUSIVE blocks open()");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "FH_EXCLUSIVE blocks open(), sets error");
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh && FSFH_Is_A(fh, FSFILEHANDLE),
+ "open() for append");
+ TEST_TRUE(batch, Err_get_error() == NULL,
+ "open() for append -- no errors");
+ FSFH_Write(fh, "bar", 3);
+ if (!FSFH_Close(fh)) { RETHROW(INCREF(Err_get_error())); }
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_READ_ONLY);
+ TEST_TRUE(batch, fh && FSFH_Is_A(fh, FSFILEHANDLE), "open() read only");
+ TEST_TRUE(batch, Err_get_error() == NULL,
+ "open() read only -- no errors");
+ DECREF(fh);
+
+ remove((char*)CB_Get_Ptr8(test_filename));
+}
+
+static void
+test_Read_Write(TestBatch *batch) {
+ FSFileHandle *fh;
+ const char *foo = "foo";
+ const char *bar = "bar";
+ char buffer[12];
+ char *buf = buffer;
+ CharBuf *test_filename = (CharBuf*)ZCB_WRAP_STR("_fstest", 7);
+
+ remove((char*)CB_Get_Ptr8(test_filename));
+ fh = FSFH_open(test_filename,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+
+ TEST_TRUE(batch, FSFH_Length(fh) == I64_C(0), "Length initially 0");
+ TEST_TRUE(batch, FSFH_Write(fh, foo, 3), "Write returns success");
+ TEST_TRUE(batch, FSFH_Length(fh) == I64_C(3), "Length after Write");
+ TEST_TRUE(batch, FSFH_Write(fh, bar, 3), "Write returns success");
+ TEST_TRUE(batch, FSFH_Length(fh) == I64_C(6), "Length after 2 Writes");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, FSFH_Read(fh, buf, 0, 2),
+ "Reading from a write-only handle returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Reading from a write-only handle sets error");
+ if (!FSFH_Close(fh)) { RETHROW(INCREF(Err_get_error())); }
+ DECREF(fh);
+
+ // Reopen for reading.
+ Err_set_error(NULL);
+ fh = FSFH_open(test_filename, FH_READ_ONLY);
+
+ TEST_TRUE(batch, FSFH_Length(fh) == I64_C(6), "Length on Read");
+ TEST_TRUE(batch, FSFH_Read(fh, buf, 0, 6), "Read returns success");
+ TEST_TRUE(batch, strncmp(buf, "foobar", 6) == 0, "Read/Write");
+ TEST_TRUE(batch, FSFH_Read(fh, buf, 2, 3), "Read returns success");
+ TEST_TRUE(batch, strncmp(buf, "oba", 3) == 0, "Read with offset");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, FSFH_Read(fh, buf, -1, 4),
+ "Read() with a negative offset returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Read() with a negative offset sets error");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, FSFH_Read(fh, buf, 6, 1),
+ "Read() past EOF returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Read() past EOF sets error");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, FSFH_Write(fh, foo, 3),
+ "Writing to a read-only handle returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Writing to a read-only handle sets error");
+
+ DECREF(fh);
+ remove((char*)CB_Get_Ptr8(test_filename));
+}
+
+static void
+test_Close(TestBatch *batch) {
+ CharBuf *test_filename = (CharBuf*)ZCB_WRAP_STR("_fstest", 7);
+ FSFileHandle *fh;
+
+ remove((char*)CB_Get_Ptr8(test_filename));
+ fh = FSFH_open(test_filename,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ TEST_TRUE(batch, FSFH_Close(fh), "Close returns true for write-only");
+ DECREF(fh);
+
+ // Simulate an OS error when closing the file descriptor. This
+ // approximates what would happen if, say, we run out of disk space.
+ remove((char*)CB_Get_Ptr8(test_filename));
+ fh = FSFH_open(test_filename,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+#ifdef _MSC_VER
+ SKIP(batch, "LUCY-155");
+ SKIP(batch, "LUCY-155");
+#else
+ int saved_fd = fh->fd;
+ fh->fd = -1;
+ Err_set_error(NULL);
+ bool_t result = FSFH_Close(fh);
+ TEST_FALSE(batch, result, "Failed Close() returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed Close() sets Err_error");
+ fh->fd = saved_fd;
+#endif /* _MSC_VER */
+ DECREF(fh);
+
+ fh = FSFH_open(test_filename, FH_READ_ONLY);
+ TEST_TRUE(batch, FSFH_Close(fh), "Close returns true for read-only");
+
+ DECREF(fh);
+ remove((char*)CB_Get_Ptr8(test_filename));
+}
+
+static void
+test_Window(TestBatch *batch) {
+ CharBuf *test_filename = (CharBuf*)ZCB_WRAP_STR("_fstest", 7);
+ FSFileHandle *fh;
+ FileWindow *window = FileWindow_new();
+ uint32_t i;
+
+ remove((char*)CB_Get_Ptr8(test_filename));
+ fh = FSFH_open(test_filename,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ for (i = 0; i < 1024; i++) {
+ FSFH_Write(fh, "foo ", 4);
+ }
+ if (!FSFH_Close(fh)) { RETHROW(INCREF(Err_get_error())); }
+
+ // Reopen for reading.
+ DECREF(fh);
+ fh = FSFH_open(test_filename, FH_READ_ONLY);
+ if (!fh) { RETHROW(INCREF(Err_get_error())); }
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, FSFH_Window(fh, window, -1, 4),
+ "Window() with a negative offset returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Window() with a negative offset sets error");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, FSFH_Window(fh, window, 4000, 1000),
+ "Window() past EOF returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Window() past EOF sets error");
+
+ TEST_TRUE(batch, FSFH_Window(fh, window, 1021, 2),
+ "Window() returns true");
+ TEST_TRUE(batch,
+ strncmp(window->buf - window->offset + 1021, "oo", 2) == 0,
+ "Window()");
+
+ TEST_TRUE(batch, FSFH_Release_Window(fh, window),
+ "Release_Window() returns true");
+ TEST_TRUE(batch, window->buf == NULL, "Release_Window() resets buf");
+ TEST_TRUE(batch, window->offset == 0, "Release_Window() resets offset");
+ TEST_TRUE(batch, window->len == 0, "Release_Window() resets len");
+
+ DECREF(window);
+ DECREF(fh);
+ remove((char*)CB_Get_Ptr8(test_filename));
+}
+
+void
+TestFSFH_run_tests() {
+ TestBatch *batch = TestBatch_new(46);
+
+ TestBatch_Plan(batch);
+ test_open(batch);
+ test_Read_Write(batch);
+ test_Close(batch);
+ test_Window(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFSFileHandle.cfh b/core/Lucy/Test/Store/TestFSFileHandle.cfh
new file mode 100644
index 0000000..dffaeee
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFSFileHandle.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestFSFileHandle cnick TestFSFH {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFSFolder.c b/core/Lucy/Test/Store/TestFSFolder.c
new file mode 100644
index 0000000..b198e86
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFSFolder.c
@@ -0,0 +1,208 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+// mkdir, rmdir
+#ifdef CHY_HAS_DIRECT_H
+ #include <direct.h>
+#endif
+
+// rmdir
+#ifdef CHY_HAS_UNISTD_H
+ #include <unistd.h>
+#endif
+
+// mkdir, stat
+#ifdef CHY_HAS_SYS_STAT_H
+ #include <sys/stat.h>
+#endif
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestFSFolder.h"
+#include "Lucy/Test/Store/TestFolderCommon.h"
+#include "Lucy/Store/FSFolder.h"
+#include "Lucy/Store/OutStream.h"
+
+/* The tests involving symlinks have to be run with administrator privileges
+ * under Windows, so disable by default.
+ */
+#ifndef CHY_HAS_WINDOWS_H
+#define ENABLE_SYMLINK_TESTS
+// Create the symlinks needed by test_protect_symlinks().
+static bool_t
+S_create_test_symlinks(void);
+#endif /* CHY_HAS_WINDOWS_H */
+
+static Folder*
+S_set_up() {
+ rmdir("_fstest");
+ CharBuf *test_dir = (CharBuf*)ZCB_WRAP_STR("_fstest", 7);
+ FSFolder *folder = FSFolder_new(test_dir);
+ FSFolder_Initialize(folder);
+ if (!FSFolder_Check(folder)) {
+ RETHROW(INCREF(Err_get_error()));
+ }
+ return (Folder*)folder;
+}
+
+static void
+S_tear_down() {
+ struct stat stat_buf;
+ rmdir("_fstest");
+ if (stat("_fstest", &stat_buf) != -1) {
+ THROW(ERR, "Can't clean up directory _fstest");
+ }
+}
+
+static void
+test_Initialize_and_Check(TestBatch *batch) {
+ rmdir("_fstest");
+ CharBuf *test_dir = (CharBuf*)ZCB_WRAP_STR("_fstest", 7);
+ FSFolder *folder = FSFolder_new(test_dir);
+ TEST_FALSE(batch, FSFolder_Check(folder),
+ "Check() returns false when folder dir doesn't exist");
+ FSFolder_Initialize(folder);
+ PASS(batch, "Initialize() concludes without incident");
+ TEST_TRUE(batch, FSFolder_Check(folder),
+ "Initialize() created dir, and now Check() succeeds");
+ DECREF(folder);
+ S_tear_down();
+}
+
+static void
+test_protect_symlinks(TestBatch *batch) {
+#ifdef ENABLE_SYMLINK_TESTS
+ FSFolder *folder = (FSFolder*)S_set_up();
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+ CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
+ CharBuf *foo_boffo = (CharBuf*)ZCB_WRAP_STR("foo/boffo", 9);
+
+ FSFolder_MkDir(folder, foo);
+ FSFolder_MkDir(folder, bar);
+ OutStream *outstream = FSFolder_Open_Out(folder, foo_boffo);
+ DECREF(outstream);
+
+ if (!S_create_test_symlinks()) {
+ FAIL(batch, "symlink creation failed");
+ FAIL(batch, "symlink creation failed");
+ FAIL(batch, "symlink creation failed");
+ FAIL(batch, "symlink creation failed");
+ FAIL(batch, "symlink creation failed");
+ // Try to clean up anyway.
+ FSFolder_Delete_Tree(folder, foo);
+ FSFolder_Delete_Tree(folder, bar);
+ }
+ else {
+ VArray *list = FSFolder_List_R(folder, NULL);
+ bool_t saw_bazooka_boffo = false;
+ for (uint32_t i = 0, max = VA_Get_Size(list); i < max; i++) {
+ CharBuf *entry = (CharBuf*)VA_Fetch(list, i);
+ if (CB_Ends_With_Str(entry, "bazooka/boffo", 13)) {
+ saw_bazooka_boffo = true;
+ }
+ }
+ TEST_FALSE(batch, saw_bazooka_boffo,
+ "List_R() shouldn't follow symlinks");
+ DECREF(list);
+
+ TEST_TRUE(batch, FSFolder_Delete_Tree(folder, bar),
+ "Delete_Tree() returns true");
+ TEST_FALSE(batch, FSFolder_Exists(folder, bar),
+ "Tree is really gone");
+ TEST_TRUE(batch, FSFolder_Exists(folder, foo),
+ "Original folder sill there");
+ TEST_TRUE(batch, FSFolder_Exists(folder, foo_boffo),
+ "Delete_Tree() did not follow directory symlink");
+ FSFolder_Delete_Tree(folder, foo);
+ }
+ DECREF(folder);
+ S_tear_down();
+#else
+ SKIP(batch, "Tests requiring symlink() disabled");
+ SKIP(batch, "Tests requiring symlink() disabled");
+ SKIP(batch, "Tests requiring symlink() disabled");
+ SKIP(batch, "Tests requiring symlink() disabled");
+ SKIP(batch, "Tests requiring symlink() disabled");
+#endif // ENABLE_SYMLINK_TESTS
+}
+
+void
+test_disallow_updir(TestBatch *batch) {
+ FSFolder *outer_folder = (FSFolder*)S_set_up();
+
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+ CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
+ FSFolder_MkDir(outer_folder, foo);
+ FSFolder_MkDir(outer_folder, bar);
+
+ CharBuf *inner_path = (CharBuf*)ZCB_WRAP_STR("_fstest/foo", 11);
+ FSFolder *foo_folder = FSFolder_new(inner_path);
+ CharBuf *up_bar = (CharBuf*)ZCB_WRAP_STR("../bar", 6);
+ TEST_FALSE(batch, FSFolder_Exists(foo_folder, up_bar),
+ "up-dirs are inaccessible.");
+
+ DECREF(foo_folder);
+ FSFolder_Delete(outer_folder, foo);
+ FSFolder_Delete(outer_folder, bar);
+ DECREF(outer_folder);
+ S_tear_down();
+}
+
+void
+TestFSFolder_run_tests() {
+ uint32_t num_tests = TestFolderCommon_num_tests() + 9;
+ TestBatch *batch = TestBatch_new(num_tests);
+
+ TestBatch_Plan(batch);
+ test_Initialize_and_Check(batch);
+ TestFolderCommon_run_tests(batch, S_set_up, S_tear_down);
+ test_protect_symlinks(batch);
+ test_disallow_updir(batch);
+
+ DECREF(batch);
+}
+
+#ifdef ENABLE_SYMLINK_TESTS
+
+#ifdef CHY_HAS_WINDOWS_H
+#include "windows.h"
+#elif defined(CHY_HAS_UNISTD_H)
+#include <unistd.h>
+#else
+#error "Don't have either windows.h or unistd.h"
+#endif
+
+static bool_t
+S_create_test_symlinks(void) {
+#ifdef CHY_HAS_WINDOWS_H
+ if (!CreateSymbolicLink("_fstest\\bar\\banana", "_fstest\\foo\\boffo", 0)
+ || !CreateSymbolicLink("_fstest\\bar\\bazooka", "_fstest\\foo", 1)
+ ) {
+ return false;
+ }
+#else
+ if (symlink("_fstest/foo/boffo", "_fstest/bar/banana")
+ || symlink("_fstest/foo", "_fstest/bar/bazooka")
+ ) {
+ return false;
+ }
+#endif
+ return true;
+}
+
+#endif /* ENABLE_SYMLINK_TESTS */
+
diff --git a/core/Lucy/Test/Store/TestFSFolder.cfh b/core/Lucy/Test/Store/TestFSFolder.cfh
new file mode 100644
index 0000000..726e149
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFSFolder.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestFSFolder {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFileHandle.c b/core/Lucy/Test/Store/TestFileHandle.c
new file mode 100644
index 0000000..14aa1e3
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFileHandle.c
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTINSTREAM
+#define C_LUCY_INSTREAM
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestFileHandle.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+
+static void
+S_no_op_method(const void *vself) {
+ UNUSED_VAR(vself);
+}
+
+static FileHandle*
+S_new_filehandle() {
+ ZombieCharBuf *klass = ZCB_WRAP_STR("TestFileHandle", 14);
+ FileHandle *fh;
+ VTable *vtable = VTable_fetch_vtable((CharBuf*)klass);
+ if (!vtable) {
+ vtable = VTable_singleton((CharBuf*)klass, FILEHANDLE);
+ }
+ VTable_Override(vtable, S_no_op_method, Lucy_FH_Close_OFFSET);
+ fh = (FileHandle*)VTable_Make_Obj(vtable);
+ return FH_do_open(fh, NULL, 0);
+}
+
+void
+TestFH_run_tests() {
+ TestBatch *batch = TestBatch_new(2);
+ FileHandle *fh = S_new_filehandle();
+ ZombieCharBuf *foo = ZCB_WRAP_STR("foo", 3);
+
+ TestBatch_Plan(batch);
+
+ TEST_TRUE(batch, CB_Equals_Str(FH_Get_Path(fh), "", 0), "Get_Path");
+ FH_Set_Path(fh, (CharBuf*)foo);
+ TEST_TRUE(batch, CB_Equals(FH_Get_Path(fh), (Obj*)foo), "Set_Path");
+
+ DECREF(fh);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFileHandle.cfh b/core/Lucy/Test/Store/TestFileHandle.cfh
new file mode 100644
index 0000000..ebe12ea
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFileHandle.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestFileHandle cnick TestFH {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFolder.c b/core/Lucy/Test/Store/TestFolder.c
new file mode 100644
index 0000000..294ed81
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFolder.c
@@ -0,0 +1,506 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFOLDER
+#define C_LUCY_CHARBUF
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestFolder.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFolder.h"
+
+static CharBuf foo = ZCB_LITERAL("foo");
+static CharBuf bar = ZCB_LITERAL("bar");
+static CharBuf baz = ZCB_LITERAL("baz");
+static CharBuf boffo = ZCB_LITERAL("boffo");
+static CharBuf banana = ZCB_LITERAL("banana");
+static CharBuf foo_bar = ZCB_LITERAL("foo/bar");
+static CharBuf foo_boffo = ZCB_LITERAL("foo/boffo");
+static CharBuf foo_foo = ZCB_LITERAL("foo/foo");
+static CharBuf foo_bar_baz = ZCB_LITERAL("foo/bar/baz");
+static CharBuf foo_bar_baz_boffo = ZCB_LITERAL("foo/bar/baz/boffo");
+static CharBuf nope = ZCB_LITERAL("nope");
+
+static void
+test_Exists(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ fh = Folder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ TEST_TRUE(batch, Folder_Exists(folder, &foo), "Dir exists");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo), "File exists");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar),
+ "Nested dir exists");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "Nested file exists");
+
+ TEST_FALSE(batch, Folder_Exists(folder, &banana),
+ "Non-existent entry");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo_foo),
+ "Non-existent nested entry");
+
+ DECREF(folder);
+}
+
+static void
+test_Set_Path_and_Get_Path(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(&foo);
+ TEST_TRUE(batch, CB_Equals(Folder_Get_Path(folder), (Obj*)&foo),
+ "Get_Path");
+ Folder_Set_Path(folder, &bar);
+ TEST_TRUE(batch, CB_Equals(Folder_Get_Path(folder), (Obj*)&bar),
+ "Set_Path");
+ DECREF(folder);
+}
+
+static void
+test_MkDir_and_Is_Directory(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+
+ TEST_FALSE(batch, Folder_Is_Directory(folder, &foo),
+ "Is_Directory() false for non-existent entry");
+
+ TEST_TRUE(batch, Folder_MkDir(folder, &foo),
+ "MkDir returns true on success");
+ TEST_TRUE(batch, Folder_Is_Directory(folder, &foo),
+ "Is_Directory() true for local folder");
+
+ TEST_FALSE(batch, Folder_Is_Directory(folder, &foo_bar_baz),
+ "Is_Directory() false for non-existent deeply nested dir");
+ Err_set_error(NULL);
+ TEST_FALSE(batch, Folder_MkDir(folder, &foo_bar_baz),
+ "MkDir for deeply nested dir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "MkDir for deeply nested dir sets Err_error");
+
+ TEST_TRUE(batch, Folder_MkDir(folder, &foo_bar),
+ "MkDir for nested dir");
+ TEST_TRUE(batch, Folder_Is_Directory(folder, &foo_bar),
+ "Is_Directory() true for nested dir");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, Folder_MkDir(folder, &foo_bar),
+ "Overwrite dir with MkDir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Overwrite dir with MkDir sets Err_error");
+
+ fh = Folder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ Err_set_error(NULL);
+ TEST_FALSE(batch, Folder_MkDir(folder, &foo_boffo),
+ "Overwrite file with MkDir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Overwrite file with MkDir sets Err_error");
+ TEST_FALSE(batch, Folder_Is_Directory(folder, &foo_boffo),
+ "Is_Directory() false for nested file");
+
+ DECREF(folder);
+}
+
+static void
+test_Enclosing_Folder_and_Find_Folder(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ Folder_MkDir(folder, &foo_bar_baz);
+ fh = Folder_Open_FileHandle(folder, &foo_bar_baz_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+
+ {
+ Folder *encloser = Folder_Enclosing_Folder(folder, (CharBuf*)&nope);
+ Folder *found = Folder_Find_Folder(folder, (CharBuf*)&nope);
+ TEST_TRUE(batch, encloser == folder,
+ "Enclosing_Folder() - non-existent entry yields parent");
+ TEST_TRUE(batch, found == NULL,
+ "Find_Folder() - non-existent entry yields NULL");
+ }
+
+ {
+ Folder *encloser = Folder_Enclosing_Folder(folder, &foo_bar);
+ Folder *found = Folder_Find_Folder(folder, &foo_bar);
+ TEST_TRUE(batch,
+ encloser
+ && Folder_Is_A(encloser, FOLDER)
+ && CB_Ends_With(Folder_Get_Path(encloser), &foo),
+ "Enclosing_Folder() - find one directory down");
+ TEST_TRUE(batch,
+ found
+ && Folder_Is_A(found, FOLDER)
+ && CB_Ends_With(Folder_Get_Path(found), &bar),
+ "Find_Folder() - 'foo/bar'");
+ }
+
+ {
+ Folder *encloser = Folder_Enclosing_Folder(folder, &foo_bar_baz);
+ Folder *found = Folder_Find_Folder(folder, &foo_bar_baz);
+ TEST_TRUE(batch,
+ encloser
+ && Folder_Is_A(encloser, FOLDER)
+ && CB_Ends_With(Folder_Get_Path(encloser), &bar),
+ "Find two directories down");
+ TEST_TRUE(batch,
+ found
+ && Folder_Is_A(found, FOLDER)
+ && CB_Ends_With(Folder_Get_Path(found), &baz),
+ "Find_Folder() - 'foo/bar/baz'");
+ }
+
+ {
+ Folder *encloser
+ = Folder_Enclosing_Folder(folder, &foo_bar_baz_boffo);
+ Folder *found = Folder_Find_Folder(folder, &foo_bar_baz_boffo);
+ TEST_TRUE(batch,
+ encloser
+ && Folder_Is_A(encloser, FOLDER)
+ && CB_Ends_With(Folder_Get_Path(encloser), &baz),
+ "Recurse to find a directory containing a real file");
+ TEST_TRUE(batch, found == NULL,
+ "Find_Folder() - file instead of folder yields NULL");
+ }
+
+ DECREF(fh);
+ DECREF(folder);
+}
+
+static void
+test_List(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+ VArray *list;
+ CharBuf *elem;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ Folder_MkDir(folder, &foo_bar_baz);
+ fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ fh = Folder_Open_FileHandle(folder, &banana, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ list = Folder_List(folder, NULL);
+ VA_Sort(list, NULL, NULL);
+ TEST_INT_EQ(batch, VA_Get_Size(list), 3, "List");
+ elem = (CharBuf*)DOWNCAST(VA_Fetch(list, 0), CHARBUF);
+ TEST_TRUE(batch, elem && CB_Equals(elem, (Obj*)&banana),
+ "List first file");
+ elem = (CharBuf*)DOWNCAST(VA_Fetch(list, 1), CHARBUF);
+ TEST_TRUE(batch, elem && CB_Equals(elem, (Obj*)&boffo),
+ "List second file");
+ elem = (CharBuf*)DOWNCAST(VA_Fetch(list, 2), CHARBUF);
+ TEST_TRUE(batch, elem && CB_Equals(elem, (Obj*)&foo), "List dir");
+ DECREF(list);
+
+ list = Folder_List(folder, &foo_bar);
+ TEST_INT_EQ(batch, VA_Get_Size(list), 1, "List subdirectory contents");
+ elem = (CharBuf*)DOWNCAST(VA_Fetch(list, 0), CHARBUF);
+ TEST_TRUE(batch, elem && CB_Equals(elem, (Obj*)&baz),
+ "Just the filename");
+ DECREF(list);
+
+ DECREF(folder);
+}
+
+static void
+test_Open_Dir(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ DirHandle *dh;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+
+ dh = Folder_Open_Dir(folder, &foo);
+ TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE), "Open_Dir");
+ DECREF(dh);
+ dh = Folder_Open_Dir(folder, &foo_bar);
+ TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE), "Open_Dir nested dir");
+ DECREF(dh);
+
+ Err_set_error(NULL);
+ dh = Folder_Open_Dir(folder, &bar);
+ TEST_TRUE(batch, dh == NULL,
+ "Open_Dir on non-existent entry fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_Dir on non-existent entry sets Err_error");
+
+ Err_set_error(NULL);
+ dh = Folder_Open_Dir(folder, &foo_foo);
+ TEST_TRUE(batch, dh == NULL,
+ "Open_Dir on non-existent nested entry fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_Dir on non-existent nested entry sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Open_FileHandle(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+
+ Folder_MkDir(folder, &foo);
+
+ fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, FILEHANDLE), "Open_FileHandle");
+ DECREF(fh);
+
+ fh = Folder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, FILEHANDLE),
+ "Open_FileHandle for nested file");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = Folder_Open_FileHandle(folder, &foo, FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "Open_FileHandle on existing dir path fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_FileHandle on existing dir name sets Err_error");
+
+ Err_set_error(NULL);
+ fh = Folder_Open_FileHandle(folder, &foo_bar_baz_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "Open_FileHandle for entry within non-existent dir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_FileHandle for entry within non-existent dir sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Open_Out(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ OutStream *outstream;
+
+ Folder_MkDir(folder, &foo);
+
+ outstream = Folder_Open_Out(folder, &boffo);
+ TEST_TRUE(batch, outstream && OutStream_Is_A(outstream, OUTSTREAM),
+ "Open_Out");
+ DECREF(outstream);
+
+ outstream = Folder_Open_Out(folder, &foo_boffo);
+ TEST_TRUE(batch, outstream && OutStream_Is_A(outstream, OUTSTREAM),
+ "Open_Out for nested file");
+ DECREF(outstream);
+
+ Err_set_error(NULL);
+ outstream = Folder_Open_Out(folder, &boffo);
+ TEST_TRUE(batch, outstream == NULL,
+ "Open_OutStream on existing file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_Out on existing file sets Err_error");
+
+ Err_set_error(NULL);
+ outstream = Folder_Open_Out(folder, &foo);
+ TEST_TRUE(batch, outstream == NULL,
+ "Open_OutStream on existing dir path fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_Out on existing dir name sets Err_error");
+
+ Err_set_error(NULL);
+ outstream = Folder_Open_Out(folder, &foo_bar_baz_boffo);
+ TEST_TRUE(batch, outstream == NULL,
+ "Open_Out for entry within non-existent dir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_Out for entry within non-existent dir sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Open_In(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+ InStream *instream;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ fh = Folder_Open_FileHandle(folder, &foo_boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ instream = Folder_Open_In(folder, &boffo);
+ TEST_TRUE(batch, instream && InStream_Is_A(instream, INSTREAM),
+ "Open_In");
+ DECREF(instream);
+
+ instream = Folder_Open_In(folder, &foo_boffo);
+ TEST_TRUE(batch, instream && InStream_Is_A(instream, INSTREAM),
+ "Open_In for nested file");
+ DECREF(instream);
+
+ Err_set_error(NULL);
+ instream = Folder_Open_In(folder, &foo);
+ TEST_TRUE(batch, instream == NULL,
+ "Open_InStream on existing dir path fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_In on existing dir name sets Err_error");
+
+ Err_set_error(NULL);
+ instream = Folder_Open_In(folder, &foo_bar_baz_boffo);
+ TEST_TRUE(batch, instream == NULL,
+ "Open_In for entry within non-existent dir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Open_In for entry within non-existent dir sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Delete(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+ bool_t result;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ fh = Folder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ fh = Folder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ result = Folder_Delete(folder, &banana);
+ TEST_FALSE(batch, result, "Delete on non-existent entry returns false");
+
+ Err_set_error(NULL);
+ result = Folder_Delete(folder, &foo);
+ TEST_FALSE(batch, result, "Delete on non-empty dir returns false");
+
+ TEST_TRUE(batch, Folder_Delete(folder, &foo_boffo),
+ "Delete nested file");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo_boffo),
+ "File is really gone");
+ TEST_TRUE(batch, Folder_Delete(folder, &foo_bar),
+ "Delete nested dir");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo_bar),
+ "Dir is really gone");
+ TEST_TRUE(batch, Folder_Delete(folder, &foo), "Delete empty dir");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo), "Dir is really gone");
+
+ DECREF(folder);
+}
+
+static void
+test_Delete_Tree(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh;
+ bool_t result;
+
+ // Create tree to be deleted.
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ Folder_MkDir(folder, &foo_bar_baz);
+ fh = Folder_Open_FileHandle(folder, &foo_bar_baz_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ // Create bystanders.
+ Folder_MkDir(folder, &bar);
+ fh = Folder_Open_FileHandle(folder, &baz, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ result = Folder_Delete_Tree(folder, &foo);
+ TEST_TRUE(batch, result, "Delete_Tree() succeeded");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo), "Tree really gone");
+
+ TEST_TRUE(batch, Folder_Exists(folder, &bar),
+ "local dir with same name as nested dir left intact");
+ TEST_TRUE(batch, Folder_Exists(folder, &baz),
+ "local file with same name as nested dir left intact");
+
+ // Kill off the bystanders.
+ result = Folder_Delete_Tree(folder, &bar);
+ TEST_TRUE(batch, result, "Delete_Tree() on empty dir");
+ result = Folder_Delete_Tree(folder, &baz);
+ TEST_TRUE(batch, result, "Delete_Tree() on file");
+
+ // Create new tree to be deleted.
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ Folder_MkDir(folder, &foo_bar_baz);
+ fh = Folder_Open_FileHandle(folder, &foo_bar_baz_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ // Remove tree in subdir.
+ result = Folder_Delete_Tree(folder, &foo_bar);
+ TEST_TRUE(batch, result, "Delete_Tree() of subdir succeeded");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo_bar),
+ "subdir really gone");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo),
+ "enclosing dir left intact");
+
+ DECREF(folder);
+}
+
+static void
+test_Slurp_File(TestBatch *batch) {
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+ FileHandle *fh = Folder_Open_FileHandle(folder, &foo,
+ FH_CREATE | FH_WRITE_ONLY);
+ ByteBuf *contents;
+
+ FH_Write(fh, "stuff", 5);
+ FH_Close(fh);
+ DECREF(fh);
+ contents = Folder_Slurp_File(folder, &foo);
+ TEST_TRUE(batch, BB_Equals_Bytes(contents, "stuff", 5), "Slurp_File");
+
+ DECREF(contents);
+ DECREF(folder);
+}
+
+void
+TestFolder_run_tests() {
+ TestBatch *batch = TestBatch_new(79);
+
+ TestBatch_Plan(batch);
+ test_Exists(batch);
+ test_Set_Path_and_Get_Path(batch);
+ test_MkDir_and_Is_Directory(batch);
+ test_Enclosing_Folder_and_Find_Folder(batch);
+ test_List(batch);
+ test_Open_Dir(batch);
+ test_Open_FileHandle(batch);
+ test_Open_Out(batch);
+ test_Open_In(batch);
+ test_Delete(batch);
+ test_Delete_Tree(batch);
+ test_Slurp_File(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFolder.cfh b/core/Lucy/Test/Store/TestFolder.cfh
new file mode 100644
index 0000000..6d874b7
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFolder.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestFolder {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFolderCommon.c b/core/Lucy/Test/Store/TestFolderCommon.c
new file mode 100644
index 0000000..6c00d47
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFolderCommon.c
@@ -0,0 +1,527 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_CHARBUF
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestFolderCommon.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+#define set_up_t lucy_TestFolderCommon_set_up_t
+#define tear_down_t lucy_TestFolderCommon_tear_down_t
+
+static CharBuf foo = ZCB_LITERAL("foo");
+static CharBuf bar = ZCB_LITERAL("bar");
+static CharBuf baz = ZCB_LITERAL("baz");
+static CharBuf boffo = ZCB_LITERAL("boffo");
+static CharBuf banana = ZCB_LITERAL("banana");
+static CharBuf foo_bar = ZCB_LITERAL("foo/bar");
+static CharBuf foo_bar_baz = ZCB_LITERAL("foo/bar/baz");
+static CharBuf foo_bar_boffo = ZCB_LITERAL("foo/bar/boffo");
+static CharBuf foo_boffo = ZCB_LITERAL("foo/boffo");
+static CharBuf foo_foo = ZCB_LITERAL("foo/foo");
+static CharBuf nope = ZCB_LITERAL("nope");
+static CharBuf nope_nyet = ZCB_LITERAL("nope/nyet");
+
+static void
+test_Local_Exists(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ OutStream *outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+ Folder_Local_MkDir(folder, &foo);
+ outstream = Folder_Open_Out(folder, &foo_boffo);
+ DECREF(outstream);
+
+ TEST_TRUE(batch, Folder_Local_Exists(folder, &boffo),
+ "Local_Exists() returns true for file");
+ TEST_TRUE(batch, Folder_Local_Exists(folder, &foo),
+ "Local_Exists() returns true for dir");
+ TEST_FALSE(batch, Folder_Local_Exists(folder, &foo_boffo),
+ "Local_Exists() returns false for nested entry");
+ TEST_FALSE(batch, Folder_Local_Exists(folder, &bar),
+ "Local_Exists() returns false for non-existent entry");
+
+ Folder_Delete(folder, &foo_boffo);
+ Folder_Delete(folder, &foo);
+ Folder_Delete(folder, &boffo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Local_Is_Directory(TestBatch *batch, set_up_t set_up,
+ tear_down_t tear_down) {
+ Folder *folder = set_up();
+ OutStream *outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+ Folder_Local_MkDir(folder, &foo);
+
+ TEST_FALSE(batch, Folder_Local_Is_Directory(folder, &boffo),
+ "Local_Is_Directory() returns false for file");
+ TEST_TRUE(batch, Folder_Local_Is_Directory(folder, &foo),
+ "Local_Is_Directory() returns true for dir");
+ TEST_FALSE(batch, Folder_Local_Is_Directory(folder, &bar),
+ "Local_Is_Directory() returns false for non-existent entry");
+
+ Folder_Delete(folder, &boffo);
+ Folder_Delete(folder, &foo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Local_Find_Folder(TestBatch *batch, set_up_t set_up,
+ tear_down_t tear_down) {
+ Folder *folder = set_up();
+ Folder *local;
+ OutStream *outstream;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+ outstream = Folder_Open_Out(folder, &foo_boffo);
+ DECREF(outstream);
+
+ local = Folder_Local_Find_Folder(folder, &nope);
+ TEST_TRUE(batch, local == NULL, "Non-existent entry yields NULL");
+
+ local = Folder_Local_Find_Folder(folder, (CharBuf*)&EMPTY);
+ TEST_TRUE(batch, local == NULL, "Empty string yields NULL");
+
+ local = Folder_Local_Find_Folder(folder, &foo_bar);
+ TEST_TRUE(batch, local == NULL, "nested folder yields NULL");
+
+ local = Folder_Local_Find_Folder(folder, &foo_boffo);
+ TEST_TRUE(batch, local == NULL, "nested file yields NULL");
+
+ local = Folder_Local_Find_Folder(folder, &boffo);
+ TEST_TRUE(batch, local == NULL, "local file yields NULL");
+
+ local = Folder_Local_Find_Folder(folder, &bar);
+ TEST_TRUE(batch, local == NULL, "name of nested folder yields NULL");
+
+ local = Folder_Local_Find_Folder(folder, &foo);
+ TEST_TRUE(batch,
+ local
+ && Folder_Is_A(local, FOLDER)
+ && CB_Ends_With(Folder_Get_Path(local), &foo),
+ "Find local directory");
+
+ Folder_Delete(folder, &foo_bar);
+ Folder_Delete(folder, &foo_boffo);
+ Folder_Delete(folder, &foo);
+ Folder_Delete(folder, &boffo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Local_MkDir(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ bool_t result;
+
+ result = Folder_Local_MkDir(folder, &foo);
+ TEST_TRUE(batch, result, "Local_MkDir succeeds and returns true");
+
+ Err_set_error(NULL);
+ result = Folder_Local_MkDir(folder, &foo);
+ TEST_FALSE(batch, result,
+ "Local_MkDir returns false when a dir already exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_MkDir sets Err_error when a dir already exists");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo),
+ "Existing dir untouched after failed Local_MkDir");
+
+ {
+ OutStream *outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+ Err_set_error(NULL);
+ result = Folder_Local_MkDir(folder, &foo);
+ TEST_FALSE(batch, result,
+ "Local_MkDir returns false when a file already exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_MkDir sets Err_error when a file already exists");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo) &&
+ !Folder_Local_Is_Directory(folder, &boffo),
+ "Existing file untouched after failed Local_MkDir");
+ }
+
+ Folder_Delete(folder, &foo);
+ Folder_Delete(folder, &boffo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Local_Open_Dir(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ DirHandle *dh = Folder_Local_Open_Dir(folder);
+ TEST_TRUE(batch, dh && DH_Is_A(dh, DIRHANDLE),
+ "Local_Open_Dir returns an DirHandle");
+ DECREF(dh);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Local_Open_FileHandle(TestBatch *batch, set_up_t set_up,
+ tear_down_t tear_down) {
+ Folder *folder = set_up();
+ FileHandle *fh;
+
+ fh = Folder_Local_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, FILEHANDLE),
+ "opened FileHandle");
+ DECREF(fh);
+
+ fh = Folder_Local_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, FILEHANDLE),
+ "opened FileHandle for append");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = Folder_Local_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ TEST_TRUE(batch, fh == NULL, "FH_EXLUSIVE flag prevents clobber");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "failure due to FH_EXLUSIVE flag sets Err_error");
+
+ fh = Folder_Local_Open_FileHandle(folder, &boffo, FH_READ_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, FILEHANDLE),
+ "opened FileHandle for reading");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = Folder_Local_Open_FileHandle(folder, &nope, FH_READ_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "Can't open non-existent file for reading");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Opening non-existent file for reading sets Err_error");
+
+ Folder_Delete(folder, &boffo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Local_Delete(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ OutStream *outstream;
+
+ outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+ TEST_TRUE(batch, Folder_Local_Delete(folder, &boffo),
+ "Local_Delete on file succeeds");
+ TEST_FALSE(batch, Folder_Exists(folder, &boffo),
+ "File is really gone");
+
+ Folder_Local_MkDir(folder, &foo);
+ outstream = Folder_Open_Out(folder, &foo_boffo);
+ DECREF(outstream);
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, Folder_Local_Delete(folder, &foo),
+ "Local_Delete on non-empty dir fails");
+
+ Folder_Delete(folder, &foo_boffo);
+ TEST_TRUE(batch, Folder_Local_Delete(folder, &foo),
+ "Local_Delete on empty dir succeeds");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo),
+ "Dir is really gone");
+
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Rename(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ OutStream *outstream;
+ bool_t result;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ outstream = Folder_Open_Out(folder, &boffo);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+
+ // Move files.
+
+ result = Folder_Rename(folder, &boffo, &banana);
+ TEST_TRUE(batch, result, "Rename succeeds and returns true");
+ TEST_TRUE(batch, Folder_Exists(folder, &banana),
+ "File exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &boffo),
+ "File no longer exists at old path");
+
+ result = Folder_Rename(folder, &banana, &foo_bar_boffo);
+ TEST_TRUE(batch, result, "Rename to file in nested dir");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar_boffo),
+ "File exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &banana),
+ "File no longer exists at old path");
+
+ result = Folder_Rename(folder, &foo_bar_boffo, &boffo);
+ TEST_TRUE(batch, result, "Rename from file in nested dir");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo),
+ "File exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo_bar_boffo),
+ "File no longer exists at old path");
+
+ outstream = Folder_Open_Out(folder, &foo_boffo);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ result = Folder_Rename(folder, &boffo, &foo_boffo);
+ if (result) {
+ PASS(batch, "Rename clobbers on this system");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &boffo),
+ "File no longer exists at old path");
+ }
+ else {
+ PASS(batch, "Rename does not clobber on this system");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File exists at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo),
+ "File still exists at old path");
+ Folder_Delete(folder, &boffo);
+ }
+
+ // Move Dirs.
+
+ Folder_MkDir(folder, &baz);
+ result = Folder_Rename(folder, &baz, &boffo);
+ TEST_TRUE(batch, result, "Rename dir");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo),
+ "Folder exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &baz),
+ "Folder no longer exists at old path");
+
+ result = Folder_Rename(folder, &boffo, &foo_foo);
+ TEST_TRUE(batch, result, "Rename dir into nested subdir");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_foo),
+ "Folder exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &boffo),
+ "Folder no longer exists at old path");
+
+ result = Folder_Rename(folder, &foo_foo, &foo_bar_baz);
+ TEST_TRUE(batch, result, "Rename dir from nested subdir");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar_baz),
+ "Folder exists at new path");
+ TEST_FALSE(batch, Folder_Exists(folder, &foo_foo),
+ "Folder no longer exists at old path");
+
+ // Test failed clobbers.
+
+ Err_set_error(NULL);
+ result = Folder_Rename(folder, &foo_boffo, &foo_bar);
+ TEST_FALSE(batch, result, "Rename file clobbering dir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed rename sets Err_error");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File still exists at old path");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar),
+ "Dir still exists after failed clobber");
+
+ Err_set_error(NULL);
+ result = Folder_Rename(folder, &foo_bar, &foo_boffo);
+ TEST_FALSE(batch, result, "Rename dir clobbering file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed rename sets Err_error");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar),
+ "Dir still exists at old path");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File still exists after failed clobber");
+
+ // Test that "renaming" succeeds where to and from are the same.
+
+ result = Folder_Rename(folder, &foo_boffo, &foo_boffo);
+ TEST_TRUE(batch, result, "Renaming file to itself succeeds");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File still exists");
+
+ result = Folder_Rename(folder, &foo_bar, &foo_bar);
+ TEST_TRUE(batch, result, "Renaming dir to itself succeeds");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar),
+ "Dir still exists");
+
+ // Invalid filepaths.
+
+ Err_set_error(NULL);
+ result = Folder_Rename(folder, &foo_boffo, &nope_nyet);
+ TEST_FALSE(batch, result, "Rename into non-existent subdir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Renaming into non-existent subdir sets Err_error");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "Entry still exists at old path");
+
+ Err_set_error(NULL);
+ result = Folder_Rename(folder, &nope_nyet, &boffo);
+ TEST_FALSE(batch, result, "Rename non-existent file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Renaming non-existent source file sets Err_error");
+
+ Folder_Delete(folder, &foo_bar_baz);
+ Folder_Delete(folder, &foo_bar);
+ Folder_Delete(folder, &foo_boffo);
+ Folder_Delete(folder, &foo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Hard_Link(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ OutStream *outstream;
+ bool_t result;
+
+ Folder_MkDir(folder, &foo);
+ Folder_MkDir(folder, &foo_bar);
+ outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+
+ // Link files.
+
+ result = Folder_Hard_Link(folder, &boffo, &banana);
+ TEST_TRUE(batch, result, "Hard_Link succeeds and returns true");
+ TEST_TRUE(batch, Folder_Exists(folder, &banana),
+ "File exists at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo),
+ "File still exists at old path");
+ Folder_Delete(folder, &boffo);
+
+ result = Folder_Hard_Link(folder, &banana, &foo_bar_boffo);
+ TEST_TRUE(batch, result, "Hard_Link to target within nested dir");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar_boffo),
+ "File exists at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &banana),
+ "File still exists at old path");
+ Folder_Delete(folder, &banana);
+
+ result = Folder_Hard_Link(folder, &foo_bar_boffo, &foo_boffo);
+ TEST_TRUE(batch, result, "Hard_Link from file in nested dir");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File exists at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_bar_boffo),
+ "File still exists at old path");
+ Folder_Delete(folder, &foo_bar_boffo);
+
+ // Invalid clobbers.
+
+ outstream = Folder_Open_Out(folder, &boffo);
+ DECREF(outstream);
+ result = Folder_Hard_Link(folder, &foo_boffo, &boffo);
+ TEST_FALSE(batch, result, "Clobber of file fails");
+ TEST_TRUE(batch, Folder_Exists(folder, &boffo),
+ "File still exists at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File still exists at old path");
+ Folder_Delete(folder, &boffo);
+
+ Folder_MkDir(folder, &baz);
+ result = Folder_Hard_Link(folder, &foo_boffo, &baz);
+ TEST_FALSE(batch, result, "Clobber of dir fails");
+ TEST_TRUE(batch, Folder_Exists(folder, &baz),
+ "Dir still exists at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File still exists at old path");
+ Folder_Delete(folder, &baz);
+
+ // Invalid Hard_Link of dir.
+
+ Folder_MkDir(folder, &baz);
+ result = Folder_Hard_Link(folder, &baz, &banana);
+ TEST_FALSE(batch, result, "Hard_Link dir fails");
+ TEST_FALSE(batch, Folder_Exists(folder, &banana),
+ "Nothing at new path");
+ TEST_TRUE(batch, Folder_Exists(folder, &baz),
+ "Folder still exists at old path");
+ Folder_Delete(folder, &baz);
+
+ // Test that linking to yourself fails.
+
+ result = Folder_Hard_Link(folder, &foo_boffo, &foo_boffo);
+ TEST_FALSE(batch, result, "Hard_Link file to itself fails");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "File still exists");
+
+ // Invalid filepaths.
+
+ Err_set_error(NULL);
+ result = Folder_Rename(folder, &foo_boffo, &nope_nyet);
+ TEST_FALSE(batch, result, "Hard_Link into non-existent subdir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Hard_Link into non-existent subdir sets Err_error");
+ TEST_TRUE(batch, Folder_Exists(folder, &foo_boffo),
+ "Entry still exists at old path");
+
+ Err_set_error(NULL);
+ result = Folder_Rename(folder, &nope_nyet, &boffo);
+ TEST_FALSE(batch, result, "Hard_Link non-existent source file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Hard_Link non-existent source file sets Err_error");
+
+ Folder_Delete(folder, &foo_bar);
+ Folder_Delete(folder, &foo_boffo);
+ Folder_Delete(folder, &foo);
+ DECREF(folder);
+ tear_down();
+}
+
+static void
+test_Close(TestBatch *batch, set_up_t set_up, tear_down_t tear_down) {
+ Folder *folder = set_up();
+ Folder_Close(folder);
+ PASS(batch, "Close() concludes without incident");
+ Folder_Close(folder);
+ Folder_Close(folder);
+ PASS(batch, "Calling Close() multiple times is safe");
+ DECREF(folder);
+ tear_down();
+}
+
+uint32_t
+TestFolderCommon_num_tests() {
+ return 99;
+}
+
+void
+TestFolderCommon_run_tests(void *test_batch, set_up_t set_up,
+ tear_down_t tear_down) {
+ TestBatch *batch = (TestBatch*)test_batch;
+
+ test_Local_Exists(batch, set_up, tear_down);
+ test_Local_Is_Directory(batch, set_up, tear_down);
+ test_Local_Find_Folder(batch, set_up, tear_down);
+ test_Local_MkDir(batch, set_up, tear_down);
+ test_Local_Open_Dir(batch, set_up, tear_down);
+ test_Local_Open_FileHandle(batch, set_up, tear_down);
+ test_Local_Delete(batch, set_up, tear_down);
+ test_Rename(batch, set_up, tear_down);
+ test_Hard_Link(batch, set_up, tear_down);
+ test_Close(batch, set_up, tear_down);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestFolderCommon.cfh b/core/Lucy/Test/Store/TestFolderCommon.cfh
new file mode 100644
index 0000000..05fa760
--- /dev/null
+++ b/core/Lucy/Test/Store/TestFolderCommon.cfh
@@ -0,0 +1,40 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+typedef lucy_Folder*
+lucy_TestFolderCommon_set_up_t(void);
+typedef void
+lucy_TestFolderCommon_tear_down_t(void);
+#ifdef LUCY_USE_SHORT_NAMES
+ #define TestFolderCommon_set_up_t lucy_TestFolderCommon_set_up_t
+ #define TestFolderCommon_tear_down_t lucy_TestFolderCommon_tear_down_t
+#endif
+__END_C__
+
+inert class Lucy::Test::Store::TestFolderCommon {
+ inert uint32_t
+ num_tests();
+
+ inert void
+ run_tests(void *test_batch,
+ lucy_TestFolderCommon_set_up_t set_up,
+ lucy_TestFolderCommon_tear_down_t tear_down);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestIOChunks.c b/core/Lucy/Test/Store/TestIOChunks.c
new file mode 100644
index 0000000..b03440a
--- /dev/null
+++ b/core/Lucy/Test/Store/TestIOChunks.c
@@ -0,0 +1,124 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTINSTREAM
+#define C_LUCY_INSTREAM
+#define C_LUCY_FILEWINDOW
+#include <stdlib.h>
+#include <time.h>
+
+#include "Lucy/Util/ToolSet.h"
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Store/TestIOChunks.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/RAMFileHandle.h"
+#include "Lucy/Util/NumberUtils.h"
+
+static void
+test_Align(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+
+ for (int32_t i = 1; i < 32; i++) {
+ int64_t random_bytes = TestUtils_random_u64() % 32;
+ while (random_bytes--) { OutStream_Write_U8(outstream, 0); }
+ TEST_TRUE(batch, (OutStream_Align(outstream, i) % i) == 0,
+ "Align to %ld", (long)i);
+ }
+ DECREF(file);
+ DECREF(outstream);
+}
+
+static void
+test_Read_Write_Bytes(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ char buf[4];
+
+ OutStream_Write_Bytes(outstream, "foo", 4);
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ InStream_Read_Bytes(instream, buf, 4);
+ TEST_TRUE(batch, strcmp(buf, "foo") == 0, "Read_Bytes Write_Bytes");
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+}
+
+static void
+test_Buf(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ size_t size = IO_STREAM_BUF_SIZE * 2 + 5;
+ InStream *instream;
+ uint32_t i;
+ char *buf;
+
+ for (i = 0; i < size; i++) {
+ OutStream_Write_U8(outstream, 'a');
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ buf = InStream_Buf(instream, 5);
+ TEST_INT_EQ(batch, instream->limit - buf, IO_STREAM_BUF_SIZE,
+ "Small request bumped up");
+
+ buf += IO_STREAM_BUF_SIZE - 10; // 10 bytes left in buffer.
+ InStream_Advance_Buf(instream, buf);
+
+ buf = InStream_Buf(instream, 10);
+ TEST_INT_EQ(batch, instream->limit - buf, 10,
+ "Exact request doesn't trigger refill");
+
+ buf = InStream_Buf(instream, 11);
+ TEST_INT_EQ(batch, instream->limit - buf, IO_STREAM_BUF_SIZE,
+ "Requesting over limit triggers refill");
+
+ {
+ int64_t expected = InStream_Length(instream) - InStream_Tell(instream);
+ char *buff = InStream_Buf(instream, 100000);
+ int64_t got = PTR_TO_I64(instream->limit) - PTR_TO_I64(buff);
+ TEST_TRUE(batch, got == expected,
+ "Requests greater than file size get pared down");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+}
+
+void
+TestIOChunks_run_tests() {
+ TestBatch *batch = TestBatch_new(36);
+
+ srand((unsigned int)time((time_t*)NULL));
+ TestBatch_Plan(batch);
+
+ test_Align(batch);
+ test_Read_Write_Bytes(batch);
+ test_Buf(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestIOChunks.cfh b/core/Lucy/Test/Store/TestIOChunks.cfh
new file mode 100644
index 0000000..821b981
--- /dev/null
+++ b/core/Lucy/Test/Store/TestIOChunks.cfh
@@ -0,0 +1,26 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Tests reading and writing of composite types using InStream/OutStream.
+ */
+inert class Lucy::Test::Store::TestIOChunks {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestIOPrimitives.c b/core/Lucy/Test/Store/TestIOPrimitives.c
new file mode 100644
index 0000000..7ee187c
--- /dev/null
+++ b/core/Lucy/Test/Store/TestIOPrimitives.c
@@ -0,0 +1,433 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTINSTREAM
+#define C_LUCY_INSTREAM
+#define C_LUCY_FILEWINDOW
+#include <stdlib.h>
+#include <time.h>
+
+#include "Lucy/Util/ToolSet.h"
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Store/TestIOPrimitives.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/RAMFileHandle.h"
+#include "Lucy/Util/NumberUtils.h"
+
+static void
+test_i8(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ int i;
+
+ for (i = -128; i < 128; i++) {
+ OutStream_Write_I8(outstream, i);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = -128; i < 128; i++) {
+ if (InStream_Read_I8(instream) != i) { break; }
+ }
+ TEST_INT_EQ(batch, i, 128, "round trip i8 successful for %d out of 256",
+ i + 128);
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+}
+
+static void
+test_u8(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ int i;
+
+ for (i = 0; i < 256; i++) {
+ OutStream_Write_U8(outstream, i);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 256; i++) {
+ if (InStream_Read_U8(instream) != i) { break; }
+ }
+ TEST_INT_EQ(batch, i, 256,
+ "round trip u8 successful for %d out of 256", i);
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+}
+
+static void
+test_i32(TestBatch *batch) {
+ int64_t *ints = TestUtils_random_i64s(NULL, 1000, I32_MIN, I32_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = I32_MIN;
+ ints[1] = I32_MIN + 1;
+ ints[2] = I32_MAX;
+ ints[3] = I32_MAX - 1;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_I32(outstream, (int32_t)ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ int32_t got = InStream_Read_I32(instream);
+ if (got != ints[i]) {
+ FAIL(batch, "i32 round trip failed: %ld, %ld", (long)got,
+ (long)ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "i32 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
+test_u32(TestBatch *batch) {
+ uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, U32_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = 0;
+ ints[1] = 1;
+ ints[2] = U32_MAX;
+ ints[3] = U32_MAX - 1;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_U32(outstream, (uint32_t)ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ uint32_t got = InStream_Read_U32(instream);
+ if (got != ints[i]) {
+ FAIL(batch, "u32 round trip failed: %lu, %lu", (unsigned long)got,
+ (unsigned long)ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "u32 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
+test_i64(TestBatch *batch) {
+ int64_t *ints = TestUtils_random_i64s(NULL, 1000, I64_MIN, I64_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = I64_MIN;
+ ints[1] = I64_MIN + 1;
+ ints[2] = I64_MAX;
+ ints[3] = I64_MAX - 1;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_I64(outstream, ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ int64_t got = InStream_Read_I64(instream);
+ if (got != ints[i]) {
+ FAIL(batch, "i64 round trip failed: %" I64P ", %" I64P,
+ got, ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "i64 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+
+static void
+test_u64(TestBatch *batch) {
+ uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, U64_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = 0;
+ ints[1] = 1;
+ ints[2] = U64_MAX;
+ ints[3] = U64_MAX - 1;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_U64(outstream, ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ uint64_t got = InStream_Read_U64(instream);
+ if (got != ints[i]) {
+ FAIL(batch, "u64 round trip failed: %" U64P ", %" U64P,
+ got, ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "u64 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
+test_c32(TestBatch *batch) {
+ uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, U32_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = 0;
+ ints[1] = 1;
+ ints[2] = U32_MAX;
+ ints[3] = U32_MAX - 1;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_C32(outstream, (uint32_t)ints[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ uint32_t got = InStream_Read_C32(instream);
+ if (got != ints[i]) {
+ FAIL(batch, "c32 round trip failed: %lu, %lu", (unsigned long)got,
+ (unsigned long)ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "c32 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
+test_c64(TestBatch *batch) {
+ uint64_t *ints = TestUtils_random_u64s(NULL, 1000, 0, U64_MAX);
+ RAMFile *file = RAMFile_new(NULL, false);
+ RAMFile *raw_file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ OutStream *raw_outstream = OutStream_open((Obj*)raw_file);
+ InStream *instream;
+ InStream *raw_instream;
+ uint32_t i;
+
+ // Test boundaries.
+ ints[0] = 0;
+ ints[1] = 1;
+ ints[2] = U64_MAX;
+ ints[3] = U64_MAX - 1;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_C64(outstream, ints[i]);
+ OutStream_Write_C64(raw_outstream, ints[i]);
+ }
+ OutStream_Close(outstream);
+ OutStream_Close(raw_outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ uint64_t got = InStream_Read_C64(instream);
+ if (got != ints[i]) {
+ FAIL(batch, "c64 round trip failed: %" U64P ", %" U64P,
+ got, ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "c64 round trip");
+ }
+
+ raw_instream = InStream_open((Obj*)raw_file);
+ for (i = 0; i < 1000; i++) {
+ char buffer[10];
+ char *buf = buffer;
+ size_t size = InStream_Read_Raw_C64(raw_instream, buffer);
+ uint64_t got = NumUtil_decode_c64(&buf);
+ UNUSED_VAR(size);
+ if (got != ints[i]) {
+ FAIL(batch, "Read_Raw_C64 failed: %" U64P ", %" U64P,
+ got, ints[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "Read_Raw_C64");
+ }
+
+ DECREF(raw_instream);
+ DECREF(instream);
+ DECREF(raw_outstream);
+ DECREF(outstream);
+ DECREF(raw_file);
+ DECREF(file);
+ FREEMEM(ints);
+}
+
+static void
+test_f32(TestBatch *batch) {
+ double *f64s = TestUtils_random_f64s(NULL, 1000);
+ float *values = (float*)MALLOCATE(1000 * sizeof(float));
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Truncate.
+ for (i = 0; i < 1000; i++) {
+ values[i] = (float)f64s[i];
+ }
+
+ // Test boundaries.
+ values[0] = 0.0f;
+ values[1] = 1.0f;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_F32(outstream, values[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ float got = InStream_Read_F32(instream);
+ if (got != values[i]) {
+ FAIL(batch, "f32 round trip failed: %f, %f", got, values[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "f32 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(values);
+ FREEMEM(f64s);
+}
+
+static void
+test_f64(TestBatch *batch) {
+ double *values = TestUtils_random_f64s(NULL, 1000);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ uint32_t i;
+
+ // Test boundaries.
+ values[0] = 0.0;
+ values[1] = 1.0;
+
+ for (i = 0; i < 1000; i++) {
+ OutStream_Write_F64(outstream, values[i]);
+ }
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ for (i = 0; i < 1000; i++) {
+ double got = InStream_Read_F64(instream);
+ if (got != values[i]) {
+ FAIL(batch, "f64 round trip failed: %f, %f", got, values[i]);
+ break;
+ }
+ }
+ if (i == 1000) {
+ PASS(batch, "f64 round trip");
+ }
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+ FREEMEM(values);
+}
+
+void
+TestIOPrimitives_run_tests() {
+ TestBatch *batch = TestBatch_new(11);
+
+ srand((unsigned int)time((time_t*)NULL));
+ TestBatch_Plan(batch);
+
+ test_i8(batch);
+ test_u8(batch);
+ test_i32(batch);
+ test_u32(batch);
+ test_i64(batch);
+ test_u64(batch);
+ test_c32(batch);
+ test_c64(batch);
+ test_f32(batch);
+ test_f64(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestIOPrimitives.cfh b/core/Lucy/Test/Store/TestIOPrimitives.cfh
new file mode 100644
index 0000000..9a0ae56
--- /dev/null
+++ b/core/Lucy/Test/Store/TestIOPrimitives.cfh
@@ -0,0 +1,26 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Tests reading and writing of primitive types using InStream/OutStream.
+ */
+inert class Lucy::Test::Store::TestIOPrimitives {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestInStream.c b/core/Lucy/Test/Store/TestInStream.c
new file mode 100644
index 0000000..bb3885f
--- /dev/null
+++ b/core/Lucy/Test/Store/TestInStream.c
@@ -0,0 +1,219 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTINSTREAM
+#define C_LUCY_INSTREAM
+#define C_LUCY_FILEWINDOW
+
+#include "Lucy/Util/ToolSet.h"
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Store/TestInStream.h"
+#include "Lucy/Test/Store/MockFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Store/RAMFileHandle.h"
+#include "Lucy/Util/NumberUtils.h"
+
+static void
+test_refill(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ InStream *instream;
+ char scratch[5];
+ int32_t i;
+
+ for (i = 0; i < 1023; i++) {
+ OutStream_Write_U8(outstream, 'x');
+ }
+ OutStream_Write_U8(outstream, 'y');
+ OutStream_Write_U8(outstream, 'z');
+ OutStream_Close(outstream);
+
+ instream = InStream_open((Obj*)file);
+ InStream_Refill(instream);
+ TEST_INT_EQ(batch, instream->limit - instream->buf, IO_STREAM_BUF_SIZE,
+ "Refill");
+ TEST_INT_EQ(batch, (long)InStream_Tell(instream), 0,
+ "Correct file pos after standing-start Refill()");
+ DECREF(instream);
+
+ instream = InStream_open((Obj*)file);
+ InStream_Fill(instream, 30);
+ TEST_INT_EQ(batch, instream->limit - instream->buf, 30, "Fill()");
+ TEST_INT_EQ(batch, (long)InStream_Tell(instream), 0,
+ "Correct file pos after standing-start Fill()");
+ DECREF(instream);
+
+ instream = InStream_open((Obj*)file);
+ InStream_Read_Bytes(instream, scratch, 5);
+ TEST_INT_EQ(batch, instream->limit - instream->buf,
+ IO_STREAM_BUF_SIZE - 5, "small read triggers refill");
+ DECREF(instream);
+
+ instream = InStream_open((Obj*)file);
+ TEST_INT_EQ(batch, InStream_Read_U8(instream), 'x', "Read_U8");
+ InStream_Seek(instream, 1023);
+ TEST_INT_EQ(batch, (long)instream->window->offset, 0,
+ "no unnecessary refill on Seek");
+ TEST_INT_EQ(batch, (long)InStream_Tell(instream), 1023, "Seek/Tell");
+ TEST_INT_EQ(batch, InStream_Read_U8(instream), 'y',
+ "correct data after in-buffer Seek()");
+ TEST_INT_EQ(batch, InStream_Read_U8(instream), 'z', "automatic Refill");
+ TEST_TRUE(batch, (instream->window->offset != 0), "refilled");
+
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(file);
+}
+
+static void
+test_Clone_and_Reopen(TestBatch *batch) {
+ ZombieCharBuf *foo = ZCB_WRAP_STR("foo", 3);
+ ZombieCharBuf *bar = ZCB_WRAP_STR("bar", 3);
+ RAMFile *file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)file);
+ RAMFileHandle *fh;
+ InStream *instream;
+ InStream *clone;
+ InStream *reopened;
+ uint32_t i;
+
+ for (i = 0; i < 26; i++) {
+ OutStream_Write_U8(outstream, 'a' + i);
+ }
+ OutStream_Close(outstream);
+
+ fh = RAMFH_open((CharBuf*)foo, FH_READ_ONLY, file);
+ instream = InStream_open((Obj*)fh);
+ InStream_Seek(instream, 1);
+ TEST_TRUE(batch, CB_Equals(InStream_Get_Filename(instream), (Obj*)foo),
+ "Get_Filename");
+
+ clone = InStream_Clone(instream);
+ TEST_TRUE(batch, CB_Equals(InStream_Get_Filename(clone), (Obj*)foo),
+ "Clones have same filename");
+ TEST_TRUE(batch, InStream_Length(instream) == InStream_Length(clone),
+ "Clones have same length");
+ TEST_TRUE(batch, InStream_Read_U8(instream) == InStream_Read_U8(clone),
+ "Clones start at same file position");
+
+ reopened = InStream_Reopen(instream, (CharBuf*)bar, 25, 1);
+ TEST_TRUE(batch, CB_Equals(InStream_Get_Filename(reopened), (Obj*)bar),
+ "Reopened InStreams take new filename");
+ TEST_TRUE(batch, InStream_Read_U8(reopened) == 'z',
+ "Reopened stream starts at supplied offset");
+ TEST_TRUE(batch, InStream_Length(reopened) == 1,
+ "Reopened stream uses supplied length");
+ TEST_TRUE(batch, InStream_Tell(reopened) == 1,
+ "Tell() uses supplied offset for reopened stream");
+ InStream_Seek(reopened, 0);
+ TEST_TRUE(batch, InStream_Read_U8(reopened) == 'z',
+ "Seek() uses supplied offset for reopened stream");
+
+ DECREF(reopened);
+ DECREF(clone);
+ DECREF(instream);
+ DECREF(outstream);
+ DECREF(fh);
+ DECREF(file);
+}
+
+static void
+test_Close(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ InStream *instream = InStream_open((Obj*)file);
+ InStream_Close(instream);
+ TEST_TRUE(batch, instream->file_handle == NULL,
+ "Close decrements FileHandle's refcount");
+ DECREF(instream);
+ DECREF(file);
+}
+
+static void
+test_Seek_and_Tell(TestBatch *batch) {
+ int64_t gb1 = I64_C(0x40000000);
+ int64_t gb3 = gb1 * 3;
+ int64_t gb6 = gb1 * 6;
+ int64_t gb12 = gb1 * 12;
+ FileHandle *fh = (FileHandle*)MockFileHandle_new(NULL, gb12);
+ InStream *instream = InStream_open((Obj*)fh);
+
+ InStream_Buf(instream, 10000);
+ TEST_TRUE(batch, instream->limit == ((char*)NULL) + 10000,
+ "InStream_Buf sets limit");
+
+ InStream_Seek(instream, gb6);
+ TEST_TRUE(batch, InStream_Tell(instream) == gb6,
+ "Tell after seek forwards outside buffer");
+ TEST_TRUE(batch, instream->buf == NULL,
+ "Seek forwards outside buffer sets buf to NULL");
+ TEST_TRUE(batch, instream->limit == NULL,
+ "Seek forwards outside buffer sets limit to NULL");
+ TEST_TRUE(batch, instream->window->offset == gb6,
+ "Seek forwards outside buffer tracks pos in window offset");
+
+ InStream_Buf(instream, (size_t)gb1);
+ TEST_TRUE(batch, instream->limit == ((char*)NULL) + gb1,
+ "InStream_Buf sets limit");
+
+ InStream_Seek(instream, gb6 + 10);
+ TEST_TRUE(batch, InStream_Tell(instream) == gb6 + 10,
+ "Tell after seek forwards within buffer");
+ TEST_TRUE(batch, instream->buf == ((char*)NULL) + 10,
+ "Seek within buffer sets buf");
+ TEST_TRUE(batch, instream->limit == ((char*)NULL) + gb1,
+ "Seek within buffer leaves limit alone");
+
+ InStream_Seek(instream, gb6 + 1);
+ TEST_TRUE(batch, InStream_Tell(instream) == gb6 + 1,
+ "Tell after seek backwards within buffer");
+ TEST_TRUE(batch, instream->buf == ((char*)NULL) + 1,
+ "Seek backwards within buffer sets buf");
+ TEST_TRUE(batch, instream->limit == ((char*)NULL) + gb1,
+ "Seek backwards within buffer leaves limit alone");
+
+ InStream_Seek(instream, gb3);
+ TEST_TRUE(batch, InStream_Tell(instream) == gb3,
+ "Tell after seek backwards outside buffer");
+ TEST_TRUE(batch, instream->buf == NULL,
+ "Seek backwards outside buffer sets buf to NULL");
+ TEST_TRUE(batch, instream->limit == NULL,
+ "Seek backwards outside buffer sets limit to NULL");
+ TEST_TRUE(batch, instream->window->offset == gb3,
+ "Seek backwards outside buffer tracks pos in window offset");
+
+ DECREF(instream);
+ DECREF(fh);
+}
+
+void
+TestInStream_run_tests() {
+ TestBatch *batch = TestBatch_new(37);
+
+ TestBatch_Plan(batch);
+
+ test_refill(batch);
+ test_Clone_and_Reopen(batch);
+ test_Close(batch);
+ test_Seek_and_Tell(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestInStream.cfh b/core/Lucy/Test/Store/TestInStream.cfh
new file mode 100644
index 0000000..75d35c2
--- /dev/null
+++ b/core/Lucy/Test/Store/TestInStream.cfh
@@ -0,0 +1,28 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Tests for basic functionality of InStream using both memory-mapped and
+ * streamed sources.
+ */
+
+inert class Lucy::Test::Store::TestInStream {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestRAMDirHandle.c b/core/Lucy/Test/Store/TestRAMDirHandle.c
new file mode 100644
index 0000000..f95011c
--- /dev/null
+++ b/core/Lucy/Test/Store/TestRAMDirHandle.c
@@ -0,0 +1,89 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestRAMDirHandle.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Store/RAMDirHandle.h"
+
+static void
+test_all(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+ CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
+ CharBuf *foo_boffo = (CharBuf*)ZCB_WRAP_STR("foo/boffo", 9);
+ bool_t saw_foo = false;
+ bool_t saw_boffo = false;
+ bool_t foo_was_dir = false;
+ bool_t boffo_was_dir = false;
+ int count = 0;
+
+ // Set up folder contents.
+ RAMFolder_MkDir(folder, foo);
+ FileHandle *fh = RAMFolder_Open_FileHandle(folder, boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ fh = RAMFolder_Open_FileHandle(folder, foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ RAMDirHandle *dh = RAMDH_new(folder);
+ CharBuf *entry = RAMDH_Get_Entry(dh);
+ while (RAMDH_Next(dh)) {
+ count++;
+ if (CB_Equals(entry, (Obj*)foo)) {
+ saw_foo = true;
+ foo_was_dir = RAMDH_Entry_Is_Dir(dh);
+ }
+ else if (CB_Equals(entry, (Obj*)boffo)) {
+ saw_boffo = true;
+ boffo_was_dir = RAMDH_Entry_Is_Dir(dh);
+ }
+ }
+ TEST_INT_EQ(batch, 2, count, "correct number of entries");
+ TEST_TRUE(batch, saw_foo, "Directory was iterated over");
+ TEST_TRUE(batch, foo_was_dir,
+ "Dir correctly identified by Entry_Is_Dir");
+ TEST_TRUE(batch, saw_boffo, "File was iterated over");
+ TEST_FALSE(batch, boffo_was_dir,
+ "File correctly identified by Entry_Is_Dir");
+
+ {
+ uint32_t refcount = RAMFolder_Get_RefCount(folder);
+ RAMDH_Close(dh);
+ TEST_INT_EQ(batch, RAMFolder_Get_RefCount(folder), refcount - 1,
+ "Folder reference released by Close()");
+ }
+
+ DECREF(dh);
+ DECREF(folder);
+}
+
+void
+TestRAMDH_run_tests() {
+ TestBatch *batch = TestBatch_new(6);
+
+ TestBatch_Plan(batch);
+ test_all(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestRAMDirHandle.cfh b/core/Lucy/Test/Store/TestRAMDirHandle.cfh
new file mode 100644
index 0000000..362f58b
--- /dev/null
+++ b/core/Lucy/Test/Store/TestRAMDirHandle.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestRAMDirHandle cnick TestRAMDH {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestRAMFileHandle.c b/core/Lucy/Test/Store/TestRAMFileHandle.c
new file mode 100644
index 0000000..1ec40f7
--- /dev/null
+++ b/core/Lucy/Test/Store/TestRAMFileHandle.c
@@ -0,0 +1,174 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <string.h>
+
+#define C_LUCY_TESTINSTREAM
+#define C_LUCY_INSTREAM
+#define C_LUCY_FILEWINDOW
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestRAMFileHandle.h"
+#include "Lucy/Store/RAMFileHandle.h"
+#include "Lucy/Store/FileWindow.h"
+#include "Lucy/Store/RAMFile.h"
+
+static void
+test_open(TestBatch *batch) {
+ RAMFileHandle *fh;
+
+ Err_set_error(NULL);
+ fh = RAMFH_open(NULL, FH_WRITE_ONLY, NULL);
+ TEST_TRUE(batch, fh == NULL,
+ "open() without a RAMFile or FH_CREATE returns NULL");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "open() without a RAMFile or FH_CREATE sets error");
+}
+
+static void
+test_Read_Write(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ RAMFileHandle *fh = RAMFH_open(NULL, FH_WRITE_ONLY, file);
+ const char *foo = "foo";
+ const char *bar = "bar";
+ char buffer[12];
+ char *buf = buffer;
+
+ TEST_TRUE(batch, CB_Equals_Str(RAMFH_Get_Path(fh), "", 0),
+ "NULL arg as filepath yields empty string");
+
+ TEST_TRUE(batch, RAMFH_Write(fh, foo, 3), "Write returns success");
+ TEST_TRUE(batch, RAMFH_Length(fh) == 3, "Length after one Write");
+ TEST_TRUE(batch, RAMFH_Write(fh, bar, 3), "Write returns success");
+ TEST_TRUE(batch, RAMFH_Length(fh) == 6, "Length after two Writes");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFH_Read(fh, buf, 0, 2),
+ "Reading from a write-only handle returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Reading from a write-only handle sets error");
+
+ // Reopen for reading.
+ DECREF(fh);
+ fh = RAMFH_open(NULL, FH_READ_ONLY, file);
+ TEST_TRUE(batch, RAMFile_Read_Only(file),
+ "FH_READ_ONLY propagates to RAMFile's read_only property");
+
+ TEST_TRUE(batch, RAMFH_Read(fh, buf, 0, 6), "Read returns success");
+ TEST_TRUE(batch, strncmp(buf, "foobar", 6) == 0, "Read/Write");
+ TEST_TRUE(batch, RAMFH_Read(fh, buf, 2, 3), "Read returns success");
+ TEST_TRUE(batch, strncmp(buf, "oba", 3) == 0, "Read with offset");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFH_Read(fh, buf, -1, 4),
+ "Read() with a negative offset returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Read() with a negative offset sets error");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFH_Read(fh, buf, 6, 1),
+ "Read() past EOF returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Read() past EOF sets error");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFH_Write(fh, foo, 3),
+ "Writing to a read-only handle returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Writing to a read-only handle sets error");
+
+ DECREF(fh);
+ DECREF(file);
+}
+
+static void
+test_Grow_and_Get_File(TestBatch *batch) {
+ RAMFileHandle *fh = RAMFH_open(NULL, FH_WRITE_ONLY | FH_CREATE, NULL);
+ RAMFile *ram_file = RAMFH_Get_File(fh);
+ ByteBuf *contents = RAMFile_Get_Contents(ram_file);
+
+ RAMFH_Grow(fh, 100);
+ TEST_TRUE(batch, BB_Get_Capacity(contents) >= 100, "Grow");
+
+ DECREF(fh);
+}
+
+static void
+test_Close(TestBatch *batch) {
+ RAMFileHandle *fh = RAMFH_open(NULL, FH_WRITE_ONLY | FH_CREATE, NULL);
+ TEST_TRUE(batch, RAMFH_Close(fh), "Close returns true");
+ DECREF(fh);
+}
+
+static void
+test_Window(TestBatch *batch) {
+ RAMFile *file = RAMFile_new(NULL, false);
+ RAMFileHandle *fh = RAMFH_open(NULL, FH_WRITE_ONLY, file);
+ FileWindow *window = FileWindow_new();
+ uint32_t i;
+
+ for (i = 0; i < 1024; i++) {
+ RAMFH_Write(fh, "foo ", 4);
+ }
+ RAMFH_Close(fh);
+
+ // Reopen for reading.
+ DECREF(fh);
+ fh = RAMFH_open(NULL, FH_READ_ONLY, file);
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFH_Window(fh, window, -1, 4),
+ "Window() with a negative offset returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Window() with a negative offset sets error");
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFH_Window(fh, window, 4000, 1000),
+ "Window() past EOF returns false");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Window() past EOF sets error");
+
+ TEST_TRUE(batch, RAMFH_Window(fh, window, 1021, 2),
+ "Window() returns true");
+ TEST_TRUE(batch, strncmp(window->buf, "oo", 2) == 0, "Window()");
+
+ TEST_TRUE(batch, RAMFH_Release_Window(fh, window),
+ "Release_Window() returns true");
+ TEST_TRUE(batch, window->buf == NULL, "Release_Window() resets buf");
+ TEST_TRUE(batch, window->offset == 0, "Release_Window() resets offset");
+ TEST_TRUE(batch, window->len == 0, "Release_Window() resets len");
+
+ DECREF(window);
+ DECREF(fh);
+ DECREF(file);
+}
+
+void
+TestRAMFH_run_tests() {
+ TestBatch *batch = TestBatch_new(32);
+
+ TestBatch_Plan(batch);
+ test_open(batch);
+ test_Read_Write(batch);
+ test_Grow_and_Get_File(batch);
+ test_Close(batch);
+ test_Window(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestRAMFileHandle.cfh b/core/Lucy/Test/Store/TestRAMFileHandle.cfh
new file mode 100644
index 0000000..3621839
--- /dev/null
+++ b/core/Lucy/Test/Store/TestRAMFileHandle.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestRAMFileHandle cnick TestRAMFH {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Store/TestRAMFolder.c b/core/Lucy/Test/Store/TestRAMFolder.c
new file mode 100644
index 0000000..1594212
--- /dev/null
+++ b/core/Lucy/Test/Store/TestRAMFolder.c
@@ -0,0 +1,486 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_RAMFOLDER
+#define C_LUCY_CHARBUF
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestRAMFolder.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/RAMDirHandle.h"
+#include "Lucy/Store/RAMFileHandle.h"
+
+static CharBuf foo = ZCB_LITERAL("foo");
+static CharBuf bar = ZCB_LITERAL("bar");
+static CharBuf baz = ZCB_LITERAL("baz");
+static CharBuf boffo = ZCB_LITERAL("boffo");
+static CharBuf banana = ZCB_LITERAL("banana");
+static CharBuf foo_bar = ZCB_LITERAL("foo/bar");
+static CharBuf foo_bar_baz = ZCB_LITERAL("foo/bar/baz");
+static CharBuf foo_bar_boffo = ZCB_LITERAL("foo/bar/boffo");
+static CharBuf foo_boffo = ZCB_LITERAL("foo/boffo");
+static CharBuf foo_foo = ZCB_LITERAL("foo/foo");
+static CharBuf nope = ZCB_LITERAL("nope");
+static CharBuf nope_nyet = ZCB_LITERAL("nope/nyet");
+
+static void
+test_Initialize_and_Check(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ RAMFolder_Initialize(folder);
+ PASS(batch, "Initialized concludes without incident");
+ TEST_TRUE(batch, RAMFolder_Check(folder), "Check succeeds");
+ DECREF(folder);
+}
+
+static void
+test_Local_Exists(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ FileHandle *fh = RAMFolder_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ RAMFolder_Local_MkDir(folder, &foo);
+
+ TEST_TRUE(batch, RAMFolder_Local_Exists(folder, &boffo),
+ "Local_Exists() returns true for file");
+ TEST_TRUE(batch, RAMFolder_Local_Exists(folder, &foo),
+ "Local_Exists() returns true for dir");
+ TEST_FALSE(batch, RAMFolder_Local_Exists(folder, &bar),
+ "Local_Exists() returns false for non-existent entry");
+
+ DECREF(folder);
+}
+
+static void
+test_Local_Is_Directory(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ FileHandle *fh = RAMFolder_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ RAMFolder_Local_MkDir(folder, &foo);
+
+ TEST_FALSE(batch, RAMFolder_Local_Is_Directory(folder, &boffo),
+ "Local_Is_Directory() returns false for file");
+ TEST_TRUE(batch, RAMFolder_Local_Is_Directory(folder, &foo),
+ "Local_Is_Directory() returns true for dir");
+ TEST_FALSE(batch, RAMFolder_Local_Is_Directory(folder, &bar),
+ "Local_Is_Directory() returns false for non-existent entry");
+
+ DECREF(folder);
+}
+
+static void
+test_Local_Find_Folder(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ RAMFolder *local;
+ FileHandle *fh;
+
+ RAMFolder_MkDir(folder, &foo);
+ RAMFolder_MkDir(folder, &foo_bar);
+ fh = RAMFolder_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ fh = RAMFolder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, &nope);
+ TEST_TRUE(batch, local == NULL, "Non-existent entry yields NULL");
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, (CharBuf*)&EMPTY);
+ TEST_TRUE(batch, local == NULL, "Empty string yields NULL");
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, &foo_bar);
+ TEST_TRUE(batch, local == NULL, "nested folder yields NULL");
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, &foo_boffo);
+ TEST_TRUE(batch, local == NULL, "nested file yields NULL");
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, &boffo);
+ TEST_TRUE(batch, local == NULL, "local file yields NULL");
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, &bar);
+ TEST_TRUE(batch, local == NULL, "name of nested folder yields NULL");
+
+ local = (RAMFolder*)RAMFolder_Local_Find_Folder(folder, &foo);
+ TEST_TRUE(batch,
+ local
+ && RAMFolder_Is_A(local, RAMFOLDER)
+ && CB_Equals_Str(RAMFolder_Get_Path(local), "foo", 3),
+ "Find local directory");
+
+ DECREF(folder);
+}
+
+static void
+test_Local_MkDir(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ bool_t result;
+
+ result = RAMFolder_Local_MkDir(folder, &foo);
+ TEST_TRUE(batch, result, "Local_MkDir succeeds and returns true");
+
+ Err_set_error(NULL);
+ result = RAMFolder_Local_MkDir(folder, &foo);
+ TEST_FALSE(batch, result,
+ "Local_MkDir returns false when a dir already exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_MkDir sets Err_error when a dir already exists");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo),
+ "Existing dir untouched after failed Local_MkDir");
+
+ {
+ FileHandle *fh = RAMFolder_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ Err_set_error(NULL);
+ result = RAMFolder_Local_MkDir(folder, &foo);
+ TEST_FALSE(batch, result,
+ "Local_MkDir returns false when a file already exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Local_MkDir sets Err_error when a file already exists");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &boffo) &&
+ !RAMFolder_Local_Is_Directory(folder, &boffo),
+ "Existing file untouched after failed Local_MkDir");
+ }
+
+ DECREF(folder);
+}
+
+static void
+test_Local_Open_Dir(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ DirHandle *dh = RAMFolder_Local_Open_Dir(folder);
+ TEST_TRUE(batch, dh && DH_Is_A(dh, RAMDIRHANDLE),
+ "Local_Open_Dir returns a RAMDirHandle");
+ DECREF(dh);
+ DECREF(folder);
+}
+
+static void
+test_Local_Open_FileHandle(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ FileHandle *fh;
+
+ fh = RAMFolder_Local_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, RAMFILEHANDLE),
+ "opened FileHandle");
+ DECREF(fh);
+
+ fh = RAMFolder_Local_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, RAMFILEHANDLE),
+ "opened FileHandle for append");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = RAMFolder_Local_Open_FileHandle(folder, &boffo,
+ FH_CREATE | FH_WRITE_ONLY | FH_EXCLUSIVE);
+ TEST_TRUE(batch, fh == NULL, "FH_EXLUSIVE flag prevents open");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "failure due to FH_EXLUSIVE flag sets Err_error");
+
+ fh = RAMFolder_Local_Open_FileHandle(folder, &boffo, FH_READ_ONLY);
+ TEST_TRUE(batch, fh && FH_Is_A(fh, RAMFILEHANDLE),
+ "opened FileHandle for reading");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = RAMFolder_Local_Open_FileHandle(folder, &nope, FH_READ_ONLY);
+ TEST_TRUE(batch, fh == NULL,
+ "Can't open non-existent file for reading");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Opening non-existent file for reading sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Local_Delete(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ FileHandle *fh;
+
+ fh = RAMFolder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ TEST_TRUE(batch, RAMFolder_Local_Delete(folder, &boffo),
+ "Local_Delete on file succeeds");
+
+ RAMFolder_Local_MkDir(folder, &foo);
+ fh = RAMFolder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ TEST_FALSE(batch, RAMFolder_Local_Delete(folder, &foo),
+ "Local_Delete on non-empty dir fails");
+
+ RAMFolder_Delete(folder, &foo_boffo);
+ TEST_TRUE(batch, RAMFolder_Local_Delete(folder, &foo),
+ "Local_Delete on empty dir succeeds");
+
+ DECREF(folder);
+}
+
+static void
+test_Rename(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ FileHandle *fh;
+ bool_t result;
+
+ RAMFolder_MkDir(folder, &foo);
+ RAMFolder_MkDir(folder, &foo_bar);
+ fh = RAMFolder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ // Move files.
+
+ result = RAMFolder_Rename(folder, &boffo, &banana);
+ TEST_TRUE(batch, result, "Rename succeeds and returns true");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &banana),
+ "File exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &boffo),
+ "File no longer exists at old path");
+
+ result = RAMFolder_Rename(folder, &banana, &foo_bar_boffo);
+ TEST_TRUE(batch, result, "Rename to file in nested dir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar_boffo),
+ "File exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &banana),
+ "File no longer exists at old path");
+
+ result = RAMFolder_Rename(folder, &foo_bar_boffo, &boffo);
+ TEST_TRUE(batch, result, "Rename from file in nested dir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &boffo),
+ "File exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &foo_bar_boffo),
+ "File no longer exists at old path");
+
+ fh = RAMFolder_Open_FileHandle(folder, &foo_boffo,
+ FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ result = RAMFolder_Rename(folder, &boffo, &foo_boffo);
+ TEST_TRUE(batch, result, "Clobber");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &boffo),
+ "File no longer exists at old path");
+
+ // Move Dirs.
+
+ RAMFolder_MkDir(folder, &baz);
+ result = RAMFolder_Rename(folder, &baz, &boffo);
+ TEST_TRUE(batch, result, "Rename dir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &boffo),
+ "Folder exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &baz),
+ "Folder no longer exists at old path");
+
+ result = RAMFolder_Rename(folder, &boffo, &foo_foo);
+ TEST_TRUE(batch, result, "Rename dir into nested subdir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_foo),
+ "Folder exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &boffo),
+ "Folder no longer exists at old path");
+
+ result = RAMFolder_Rename(folder, &foo_foo, &foo_bar_baz);
+ TEST_TRUE(batch, result, "Rename dir from nested subdir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar_baz),
+ "Folder exists at new path");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &foo_foo),
+ "Folder no longer exists at old path");
+
+ // Test failed clobbers.
+
+ Err_set_error(NULL);
+ result = RAMFolder_Rename(folder, &foo_boffo, &foo_bar);
+ TEST_FALSE(batch, result, "Rename file clobbering dir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed rename sets Err_error");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File still exists at old path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar),
+ "Dir still exists after failed clobber");
+
+ Err_set_error(NULL);
+ result = RAMFolder_Rename(folder, &foo_bar, &foo_boffo);
+ TEST_FALSE(batch, result, "Rename dir clobbering file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed rename sets Err_error");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar),
+ "Dir still exists at old path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File still exists after failed clobber");
+
+ // Test that "renaming" succeeds where to and from are the same.
+
+ result = RAMFolder_Rename(folder, &foo_boffo, &foo_boffo);
+ TEST_TRUE(batch, result, "Renaming file to itself succeeds");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File still exists");
+
+ result = RAMFolder_Rename(folder, &foo_bar, &foo_bar);
+ TEST_TRUE(batch, result, "Renaming dir to itself succeeds");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar),
+ "Dir still exists");
+
+ // Invalid filepaths.
+
+ Err_set_error(NULL);
+ result = RAMFolder_Rename(folder, &foo_boffo, &nope_nyet);
+ TEST_FALSE(batch, result, "Rename into non-existent subdir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Renaming into non-existent subdir sets Err_error");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "Entry still exists at old path");
+
+ Err_set_error(NULL);
+ result = RAMFolder_Rename(folder, &nope_nyet, &boffo);
+ TEST_FALSE(batch, result, "Rename non-existent file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Renaming non-existent source file sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Hard_Link(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ FileHandle *fh;
+ bool_t result;
+
+ RAMFolder_MkDir(folder, &foo);
+ RAMFolder_MkDir(folder, &foo_bar);
+ fh = RAMFolder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+
+ // Link files.
+
+ result = RAMFolder_Hard_Link(folder, &boffo, &banana);
+ TEST_TRUE(batch, result, "Hard_Link succeeds and returns true");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &banana),
+ "File exists at new path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &boffo),
+ "File still exists at old path");
+ RAMFolder_Delete(folder, &boffo);
+
+ result = RAMFolder_Hard_Link(folder, &banana, &foo_bar_boffo);
+ TEST_TRUE(batch, result, "Hard_Link to target within nested dir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar_boffo),
+ "File exists at new path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &banana),
+ "File still exists at old path");
+ RAMFolder_Delete(folder, &banana);
+
+ result = RAMFolder_Hard_Link(folder, &foo_bar_boffo, &foo_boffo);
+ TEST_TRUE(batch, result, "Hard_Link from file in nested dir");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File exists at new path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_bar_boffo),
+ "File still exists at old path");
+ RAMFolder_Delete(folder, &foo_bar_boffo);
+
+ // Invalid clobbers.
+
+ fh = RAMFolder_Open_FileHandle(folder, &boffo, FH_CREATE | FH_WRITE_ONLY);
+ DECREF(fh);
+ result = RAMFolder_Hard_Link(folder, &foo_boffo, &boffo);
+ TEST_FALSE(batch, result, "Clobber of file fails");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &boffo),
+ "File still exists at new path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File still exists at old path");
+ RAMFolder_Delete(folder, &boffo);
+
+ RAMFolder_MkDir(folder, &baz);
+ result = RAMFolder_Hard_Link(folder, &foo_boffo, &baz);
+ TEST_FALSE(batch, result, "Clobber of dir fails");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &baz),
+ "Dir still exists at new path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File still exists at old path");
+ RAMFolder_Delete(folder, &baz);
+
+ // Invalid Hard_Link of dir.
+
+ RAMFolder_MkDir(folder, &baz);
+ result = RAMFolder_Hard_Link(folder, &baz, &banana);
+ TEST_FALSE(batch, result, "Hard_Link dir fails");
+ TEST_FALSE(batch, RAMFolder_Exists(folder, &banana),
+ "Nothing at new path");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &baz),
+ "Folder still exists at old path");
+ RAMFolder_Delete(folder, &baz);
+
+ // Test that linking to yourself fails.
+
+ result = RAMFolder_Hard_Link(folder, &foo_boffo, &foo_boffo);
+ TEST_FALSE(batch, result, "Hard_Link file to itself fails");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "File still exists");
+
+ // Invalid filepaths.
+
+ Err_set_error(NULL);
+ result = RAMFolder_Rename(folder, &foo_boffo, &nope_nyet);
+ TEST_FALSE(batch, result, "Hard_Link into non-existent subdir fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Hard_Link into non-existent subdir sets Err_error");
+ TEST_TRUE(batch, RAMFolder_Exists(folder, &foo_boffo),
+ "Entry still exists at old path");
+
+ Err_set_error(NULL);
+ result = RAMFolder_Rename(folder, &nope_nyet, &boffo);
+ TEST_FALSE(batch, result, "Hard_Link non-existent source file fails");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Hard_Link non-existent source file sets Err_error");
+
+ DECREF(folder);
+}
+
+static void
+test_Close(TestBatch *batch) {
+ RAMFolder *folder = RAMFolder_new(NULL);
+ RAMFolder_Close(folder);
+ PASS(batch, "Close() concludes without incident");
+ RAMFolder_Close(folder);
+ RAMFolder_Close(folder);
+ PASS(batch, "Calling Close() multiple times is safe");
+ DECREF(folder);
+}
+
+void
+TestRAMFolder_run_tests() {
+ TestBatch *batch = TestBatch_new(98);
+
+ TestBatch_Plan(batch);
+ test_Initialize_and_Check(batch);
+ test_Local_Exists(batch);
+ test_Local_Is_Directory(batch);
+ test_Local_Find_Folder(batch);
+ test_Local_MkDir(batch);
+ test_Local_Open_Dir(batch);
+ test_Local_Open_FileHandle(batch);
+ test_Local_Delete(batch);
+ test_Rename(batch);
+ test_Hard_Link(batch);
+ test_Close(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Store/TestRAMFolder.cfh b/core/Lucy/Test/Store/TestRAMFolder.cfh
new file mode 100644
index 0000000..2b80d2b
--- /dev/null
+++ b/core/Lucy/Test/Store/TestRAMFolder.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestRAMFolder {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/TestSchema.c b/core/Lucy/Test/TestSchema.c
new file mode 100644
index 0000000..37f2098
--- /dev/null
+++ b/core/Lucy/Test/TestSchema.c
@@ -0,0 +1,105 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTSCHEMA
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Plan/TestArchitecture.h"
+#include "Lucy/Test/TestSchema.h"
+#include "Lucy/Analysis/CaseFolder.h"
+#include "Lucy/Analysis/RegexTokenizer.h"
+#include "Lucy/Plan/FullTextType.h"
+#include "Lucy/Plan/Architecture.h"
+
+TestSchema*
+TestSchema_new() {
+ TestSchema *self = (TestSchema*)VTable_Make_Obj(TESTSCHEMA);
+ return TestSchema_init(self);
+}
+
+TestSchema*
+TestSchema_init(TestSchema *self) {
+ RegexTokenizer *tokenizer = RegexTokenizer_new(NULL);
+ FullTextType *type = FullTextType_new((Analyzer*)tokenizer);
+
+ Schema_init((Schema*)self);
+ FullTextType_Set_Highlightable(type, true);
+ CharBuf *content = (CharBuf*)ZCB_WRAP_STR("content", 7);
+ TestSchema_Spec_Field(self, content, (FieldType*)type);
+ DECREF(type);
+ DECREF(tokenizer);
+
+ return self;
+}
+
+Architecture*
+TestSchema_architecture(TestSchema *self) {
+ UNUSED_VAR(self);
+ return (Architecture*)TestArch_new();
+}
+
+static void
+test_Equals(TestBatch *batch) {
+ TestSchema *schema = TestSchema_new();
+ TestSchema *arch_differs = TestSchema_new();
+ TestSchema *spec_differs = TestSchema_new();
+ CharBuf *content = (CharBuf*)ZCB_WRAP_STR("content", 7);
+ FullTextType *type = (FullTextType*)TestSchema_Fetch_Type(spec_differs,
+ content);
+ CaseFolder *case_folder = CaseFolder_new();
+
+ TEST_TRUE(batch, TestSchema_Equals(schema, (Obj*)schema), "Equals");
+
+ FullTextType_Set_Boost(type, 2.0f);
+ TEST_FALSE(batch, TestSchema_Equals(schema, (Obj*)spec_differs),
+ "Equals spoiled by differing FieldType");
+
+ DECREF(arch_differs->arch);
+ arch_differs->arch = Arch_new();
+ TEST_FALSE(batch, TestSchema_Equals(schema, (Obj*)arch_differs),
+ "Equals spoiled by differing Architecture");
+
+ DECREF(schema);
+ DECREF(arch_differs);
+ DECREF(spec_differs);
+ DECREF(case_folder);
+}
+
+static void
+test_Dump_and_Load(TestBatch *batch) {
+ TestSchema *schema = TestSchema_new();
+ Obj *dump = (Obj*)TestSchema_Dump(schema);
+ TestSchema *loaded = (TestSchema*)Obj_Load(dump, dump);
+
+ TEST_FALSE(batch, TestSchema_Equals(schema, (Obj*)loaded),
+ "Dump => Load round trip");
+
+ DECREF(schema);
+ DECREF(dump);
+ DECREF(loaded);
+}
+
+void
+TestSchema_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ TestBatch_Plan(batch);
+ test_Equals(batch);
+ test_Dump_and_Load(batch);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/TestSchema.cfh b/core/Lucy/Test/TestSchema.cfh
new file mode 100644
index 0000000..3a7b332
--- /dev/null
+++ b/core/Lucy/Test/TestSchema.cfh
@@ -0,0 +1,39 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Schema for use by the test suite.
+ *
+ * Exposes problems faced by much larger indexes by using an TestArchitecture,
+ * which returns absurdly low values for Index_Interval() and Skip_Interval().
+ */
+
+class Lucy::Test::TestSchema inherits Lucy::Plan::Schema {
+ inert incremented TestSchema*
+ new();
+
+ inert TestSchema*
+ init(TestSchema *self);
+
+ public incremented Architecture*
+ Architecture(TestSchema *self);
+
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/TestUtils.c b/core/Lucy/Test/TestUtils.c
new file mode 100644
index 0000000..04b3e04
--- /dev/null
+++ b/core/Lucy/Test/TestUtils.c
@@ -0,0 +1,236 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTUTILS
+#include "Lucy/Util/ToolSet.h"
+#include <stdarg.h>
+#include <string.h>
+
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test.h"
+#include "Lucy/Analysis/Analyzer.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Search/PhraseQuery.h"
+#include "Lucy/Search/LeafQuery.h"
+#include "Lucy/Search/ANDQuery.h"
+#include "Lucy/Search/NOTQuery.h"
+#include "Lucy/Search/ORQuery.h"
+#include "Lucy/Search/RangeQuery.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFile.h"
+#include "Lucy/Util/Freezer.h"
+
+uint64_t
+TestUtils_random_u64() {
+ uint64_t num = ((uint64_t)rand() << 60)
+ | ((uint64_t)rand() << 45)
+ | ((uint64_t)rand() << 30)
+ | ((uint64_t)rand() << 15)
+ | ((uint64_t)rand() << 0);
+ return num;
+}
+
+int64_t*
+TestUtils_random_i64s(int64_t *buf, size_t count, int64_t min,
+ int64_t limit) {
+ uint64_t range = min < limit ? limit - min : 0;
+ int64_t *ints = buf ? buf : (int64_t*)CALLOCATE(count, sizeof(int64_t));
+ size_t i;
+ for (i = 0; i < count; i++) {
+ ints[i] = min + TestUtils_random_u64() % range;
+ }
+ return ints;
+}
+
+uint64_t*
+TestUtils_random_u64s(uint64_t *buf, size_t count, uint64_t min,
+ uint64_t limit) {
+ uint64_t range = min < limit ? limit - min : 0;
+ uint64_t *ints = buf ? buf : (uint64_t*)CALLOCATE(count, sizeof(uint64_t));
+ size_t i;
+ for (i = 0; i < count; i++) {
+ ints[i] = min + TestUtils_random_u64() % range;
+ }
+ return ints;
+}
+
+double*
+TestUtils_random_f64s(double *buf, size_t count) {
+ double *f64s = buf ? buf : (double*)CALLOCATE(count, sizeof(double));
+ size_t i;
+ for (i = 0; i < count; i++) {
+ uint64_t num = TestUtils_random_u64();
+ f64s[i] = (double)num / U64_MAX;
+ }
+ return f64s;
+}
+
+VArray*
+TestUtils_doc_set() {
+ VArray *docs = VA_new(10);
+
+ VA_Push(docs, (Obj*)TestUtils_get_cb("x"));
+ VA_Push(docs, (Obj*)TestUtils_get_cb("y"));
+ VA_Push(docs, (Obj*)TestUtils_get_cb("z"));
+ VA_Push(docs, (Obj*)TestUtils_get_cb("x a"));
+ VA_Push(docs, (Obj*)TestUtils_get_cb("x a b"));
+ VA_Push(docs, (Obj*)TestUtils_get_cb("x a b c"));
+ VA_Push(docs, (Obj*)TestUtils_get_cb("x foo a b c d"));
+
+ return docs;
+}
+
+CharBuf*
+TestUtils_get_cb(const char *ptr) {
+ return CB_new_from_utf8(ptr, strlen(ptr));
+}
+
+PolyQuery*
+TestUtils_make_poly_query(uint32_t boolop, ...) {
+ va_list args;
+ Query *child;
+ PolyQuery *retval;
+ VArray *children = VA_new(0);
+
+ va_start(args, boolop);
+ while (NULL != (child = va_arg(args, Query*))) {
+ VA_Push(children, (Obj*)child);
+ }
+ va_end(args);
+
+ retval = boolop == BOOLOP_OR
+ ? (PolyQuery*)ORQuery_new(children)
+ : (PolyQuery*)ANDQuery_new(children);
+ DECREF(children);
+ return retval;
+}
+
+TermQuery*
+TestUtils_make_term_query(const char *field, const char *term) {
+ CharBuf *field_cb = (CharBuf*)ZCB_WRAP_STR(field, strlen(field));
+ CharBuf *term_cb = (CharBuf*)ZCB_WRAP_STR(term, strlen(term));
+ return TermQuery_new((CharBuf*)field_cb, (Obj*)term_cb);
+}
+
+PhraseQuery*
+TestUtils_make_phrase_query(const char *field, ...) {
+ CharBuf *field_cb = (CharBuf*)ZCB_WRAP_STR(field, strlen(field));
+ va_list args;
+ VArray *terms = VA_new(0);
+ PhraseQuery *query;
+ char *term_str;
+
+ va_start(args, field);
+ while (NULL != (term_str = va_arg(args, char*))) {
+ VA_Push(terms, (Obj*)TestUtils_get_cb(term_str));
+ }
+ va_end(args);
+
+ query = PhraseQuery_new(field_cb, terms);
+ DECREF(terms);
+ return query;
+}
+
+LeafQuery*
+TestUtils_make_leaf_query(const char *field, const char *term) {
+ CharBuf *term_cb = (CharBuf*)ZCB_WRAP_STR(term, strlen(term));
+ CharBuf *field_cb = field
+ ? (CharBuf*)ZCB_WRAP_STR(field, strlen(field))
+ : NULL;
+ return LeafQuery_new(field_cb, term_cb);
+}
+
+NOTQuery*
+TestUtils_make_not_query(Query* negated_query) {
+ NOTQuery *not_query = NOTQuery_new(negated_query);
+ DECREF(negated_query);
+ return not_query;
+}
+
+RangeQuery*
+TestUtils_make_range_query(const char *field, const char *lower_term,
+ const char *upper_term, bool_t include_lower,
+ bool_t include_upper) {
+ CharBuf *f = (CharBuf*)ZCB_WRAP_STR(field, strlen(field));
+ CharBuf *lterm = (CharBuf*)ZCB_WRAP_STR(lower_term, strlen(lower_term));
+ CharBuf *uterm = (CharBuf*)ZCB_WRAP_STR(upper_term, strlen(upper_term));
+ return RangeQuery_new(f, (Obj*)lterm, (Obj*)uterm, include_lower,
+ include_upper);
+}
+
+Obj*
+TestUtils_freeze_thaw(Obj *object) {
+ if (object) {
+ RAMFile *ram_file = RAMFile_new(NULL, false);
+ OutStream *outstream = OutStream_open((Obj*)ram_file);
+ FREEZE(object, outstream);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ {
+ InStream *instream = InStream_open((Obj*)ram_file);
+ Obj *retval = THAW(instream);
+ DECREF(instream);
+ DECREF(ram_file);
+ return retval;
+ }
+ }
+ else {
+ return NULL;
+ }
+}
+
+void
+TestUtils_test_analyzer(TestBatch *batch, Analyzer *analyzer, CharBuf *source,
+ VArray *expected, char *message) {
+ Token *seed = Token_new((char*)CB_Get_Ptr8(source), CB_Get_Size(source),
+ 0, 0, 1.0f, 1);
+ Inversion *starter = Inversion_new(seed);
+ Inversion *transformed = Analyzer_Transform(analyzer, starter);
+ VArray *got = VA_new(1);
+ Token *token;
+ while (NULL != (token = Inversion_Next(transformed))) {
+ CharBuf *token_text
+ = CB_new_from_utf8(Token_Get_Text(token), Token_Get_Len(token));
+ VA_Push(got, (Obj*)token_text);
+ }
+ TEST_TRUE(batch, VA_Equals(expected, (Obj*)got),
+ "Transform(): %s", message);
+ DECREF(transformed);
+
+ transformed = Analyzer_Transform_Text(analyzer, source);
+ VA_Clear(got);
+ while (NULL != (token = Inversion_Next(transformed))) {
+ CharBuf *token_text
+ = CB_new_from_utf8(Token_Get_Text(token), Token_Get_Len(token));
+ VA_Push(got, (Obj*)token_text);
+ }
+ TEST_TRUE(batch, VA_Equals(expected, (Obj*)got),
+ "Transform_Text(): %s", message);
+ DECREF(transformed);
+
+ DECREF(got);
+ got = Analyzer_Split(analyzer, source);
+ TEST_TRUE(batch, VA_Equals(expected, (Obj*)got), "Split(): %s", message);
+
+ DECREF(got);
+ DECREF(starter);
+ DECREF(seed);
+}
+
+
diff --git a/core/Lucy/Test/TestUtils.cfh b/core/Lucy/Test/TestUtils.cfh
new file mode 100644
index 0000000..b82c650
--- /dev/null
+++ b/core/Lucy/Test/TestUtils.cfh
@@ -0,0 +1,123 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::TestUtils {
+
+ /** Testing-only CharBuf factory which uses strlen().
+ */
+ inert incremented CharBuf*
+ get_cb(const char *utf8);
+
+ /** Return a random unsigned 64-bit integer.
+ */
+ inert uint64_t
+ random_u64();
+
+ /** Return an array of <code>count</code> random 64-bit integers where
+ * <code>min <= n < limit</code>.
+ *
+ * If <code>buf</code> is NULL, it will be allocated, otherwise it will
+ * be used.
+ */
+ inert int64_t*
+ random_i64s(int64_t *buf, size_t count, int64_t min, int64_t limit);
+
+ /** Return an array of <code>count</code> random unsigned, 64-bit integers
+ * where <code>min <= n < limit</code>.
+ *
+ * If <code>buf</code> is NULL, it will be allocated, otherwise it will
+ * be used.
+ */
+ inert uint64_t*
+ random_u64s(uint64_t *buf, size_t count, uint64_t min, uint64_t limit);
+
+ /** Return an array of <code>count</code> random double-precision floating
+ * point numbers between 0 and 1.
+ *
+ * If <code>buf</code> is NULL, it will be allocated, otherwise it will
+ * be used.
+ */
+ inert double*
+ random_f64s(double *buf, size_t count);
+
+ /** Return a VArray of CharBufs, each representing the content for a
+ * document in the shared collection.
+ */
+ inert incremented VArray*
+ doc_set();
+
+ /** Testing-only TermQuery factory.
+ */
+ inert incremented TermQuery*
+ make_term_query(const char *field, const char *term);
+
+ /** Testing-only PhraseQuery factory.
+ */
+ inert incremented PhraseQuery*
+ make_phrase_query(const char *field, ...);
+
+ /** Testing-only LeafQuery factory.
+ */
+ inert incremented LeafQuery*
+ make_leaf_query(const char *field, const char *term);
+
+ /** Return a new NOTQuery, decrementing the refcount for
+ * <code>negated_query</code>.
+ */
+ inert incremented NOTQuery*
+ make_not_query(Query *negated_query);
+
+ inert incremented RangeQuery*
+ make_range_query(const char *field, const char *lower_term = NULL,
+ const char *upper_term = NULL,
+ bool_t include_lower = true,
+ bool_t include_upper = true);
+
+ /** Return either an ORQuery or an ANDQuery depending on the value of
+ * <code>boolop</code>. Takes a NULL-terminated list of Query objects.
+ * Decrements the refcounts of all supplied children, under the assumption
+ * that they were created solely for inclusion within the aggregate query.
+ */
+ inert incremented PolyQuery*
+ make_poly_query(uint32_t boolop, ...);
+
+ /** Return the result of round-tripping the object through FREEZE and
+ * THAW.
+ */
+ inert incremented Obj*
+ freeze_thaw(Obj *object);
+
+ /** Verify an Analyzer's transform, transform_text, and split methods.
+ */
+ inert void
+ test_analyzer(TestBatch *batch, Analyzer *analyzer, CharBuf *source,
+ VArray *expected, char *message);
+}
+
+__C__
+
+#define LUCY_TESTUTILS_BOOLOP_OR 1
+#define LUCY_TESTUTILS_BOOLOP_AND 2
+#ifdef LUCY_USE_SHORT_NAMES
+ #define BOOLOP_OR LUCY_TESTUTILS_BOOLOP_OR
+ #define BOOLOP_AND LUCY_TESTUTILS_BOOLOP_AND
+#endif
+
+__END_C__
+
+
diff --git a/core/Lucy/Test/Util/BBSortEx.c b/core/Lucy/Test/Util/BBSortEx.c
new file mode 100644
index 0000000..acf765d
--- /dev/null
+++ b/core/Lucy/Test/Util/BBSortEx.c
@@ -0,0 +1,169 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_BBSORTEX
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test/Util/BBSortEx.h"
+
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/OutStream.h"
+
+BBSortEx*
+BBSortEx_new(uint32_t mem_threshold, VArray *external) {
+ BBSortEx *self = (BBSortEx*)VTable_Make_Obj(BBSORTEX);
+ return BBSortEx_init(self, mem_threshold, external);
+}
+
+BBSortEx*
+BBSortEx_init(BBSortEx *self, uint32_t mem_threshold, VArray *external) {
+ SortEx_init((SortExternal*)self, sizeof(Obj*));
+ self->external_tick = 0;
+ self->external = (VArray*)INCREF(external);
+ self->mem_consumed = 0;
+ BBSortEx_Set_Mem_Thresh(self, mem_threshold);
+ return self;
+}
+
+void
+BBSortEx_destroy(BBSortEx *self) {
+ DECREF(self->external);
+ SUPER_DESTROY(self, BBSORTEX);
+}
+
+void
+BBSortEx_clear_cache(BBSortEx *self) {
+ Obj **const cache = (Obj**)self->cache;
+ for (uint32_t i = self->cache_tick, max = self->cache_max; i < max; i++) {
+ DECREF(cache[i]);
+ }
+ self->mem_consumed = 0;
+ BBSortEx_clear_cache_t super_clear_cache
+ = (BBSortEx_clear_cache_t)SUPER_METHOD(
+ self->vtable, SortEx, Clear_Cache);
+ super_clear_cache(self);
+}
+
+void
+BBSortEx_feed(BBSortEx *self, void *data) {
+ SortEx_feed((SortExternal*)self, data);
+
+ // Flush() if necessary.
+ ByteBuf *bytebuf = (ByteBuf*)CERTIFY(*(ByteBuf**)data, BYTEBUF);
+ self->mem_consumed += BB_Get_Size(bytebuf);
+ if (self->mem_consumed >= self->mem_thresh) {
+ BBSortEx_Flush(self);
+ }
+}
+
+void
+BBSortEx_flush(BBSortEx *self) {
+ uint32_t cache_count = self->cache_max - self->cache_tick;
+ Obj **cache = (Obj**)self->cache;
+ VArray *elems;
+ BBSortEx *run;
+ uint32_t i;
+
+ if (!cache_count) { return; }
+ else { elems = VA_new(cache_count); }
+
+ // Sort, then create a new run.
+ BBSortEx_Sort_Cache(self);
+ for (i = self->cache_tick; i < self->cache_max; i++) {
+ VA_Push(elems, cache[i]);
+ }
+ run = BBSortEx_new(0, elems);
+ DECREF(elems);
+ BBSortEx_Add_Run(self, (SortExternal*)run);
+
+ // Blank the cache vars.
+ self->cache_tick += cache_count;
+ BBSortEx_Clear_Cache(self);
+}
+
+uint32_t
+BBSortEx_refill(BBSortEx *self) {
+ // Make sure cache is empty, then set cache tick vars.
+ if (self->cache_max - self->cache_tick > 0) {
+ THROW(ERR, "Refill called but cache contains %u32 items",
+ self->cache_max - self->cache_tick);
+ }
+ self->cache_tick = 0;
+ self->cache_max = 0;
+
+ // Read in elements.
+ while (1) {
+ ByteBuf *elem = NULL;
+
+ if (self->mem_consumed >= self->mem_thresh) {
+ self->mem_consumed = 0;
+ break;
+ }
+ else if (self->external_tick >= VA_Get_Size(self->external)) {
+ break;
+ }
+ else {
+ elem = (ByteBuf*)VA_Fetch(self->external, self->external_tick);
+ self->external_tick++;
+ // Should be + sizeof(ByteBuf), but that's ok.
+ self->mem_consumed += BB_Get_Size(elem);
+ }
+
+ if (self->cache_max == self->cache_cap) {
+ BBSortEx_Grow_Cache(self,
+ Memory_oversize(self->cache_max + 1, self->width));
+ }
+ Obj **cache = (Obj**)self->cache;
+ cache[self->cache_max++] = INCREF(elem);
+ }
+
+ return self->cache_max;
+}
+
+void
+BBSortEx_flip(BBSortEx *self) {
+ uint32_t i;
+ uint32_t run_mem_thresh = 65536;
+
+ BBSortEx_Flush(self);
+
+ // Recalculate the approximate mem allowed for each run.
+ uint32_t num_runs = VA_Get_Size(self->runs);
+ if (num_runs) {
+ run_mem_thresh = (self->mem_thresh / 2) / num_runs;
+ if (run_mem_thresh < 65536) {
+ run_mem_thresh = 65536;
+ }
+ }
+
+ for (i = 0; i < num_runs; i++) {
+ BBSortEx *run = (BBSortEx*)VA_Fetch(self->runs, i);
+ BBSortEx_Set_Mem_Thresh(run, run_mem_thresh);
+ }
+
+ // OK to fetch now.
+ self->flipped = true;
+}
+
+int
+BBSortEx_compare(BBSortEx *self, void *va, void *vb) {
+ UNUSED_VAR(self);
+ return BB_compare((ByteBuf**)va, (ByteBuf**)vb);
+}
+
+
diff --git a/core/Lucy/Test/Util/BBSortEx.cfh b/core/Lucy/Test/Util/BBSortEx.cfh
new file mode 100644
index 0000000..f82954f
--- /dev/null
+++ b/core/Lucy/Test/Util/BBSortEx.cfh
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** SortExternal for ByteBufs.
+ */
+
+class Lucy::Test::Util::BBSortEx
+ inherits Lucy::Util::SortExternal {
+
+ VArray *external;
+ uint32_t external_tick;
+ uint32_t mem_consumed;
+
+ inert BBSortEx*
+ new(uint32_t mem_thresh = 0x1000000, VArray *external = NULL);
+
+ inert BBSortEx*
+ init(BBSortEx *self, uint32_t mem_thresh = 0x1000000,
+ VArray *external = NULL);
+
+ void
+ Feed(BBSortEx *self, void *data);
+
+ void
+ Flush(BBSortEx *self);
+
+ uint32_t
+ Refill(BBSortEx *self);
+
+ void
+ Clear_Cache(BBSortEx *self);
+
+ void
+ Flip(BBSortEx *self);
+
+ int
+ Compare(BBSortEx *self, void *va, void *vb);
+
+ public void
+ Destroy(BBSortEx *self);
+}
+
+
diff --git a/core/Lucy/Test/Util/TestAtomic.c b/core/Lucy/Test/Util/TestAtomic.c
new file mode 100644
index 0000000..ce47b84
--- /dev/null
+++ b/core/Lucy/Test/Util/TestAtomic.c
@@ -0,0 +1,61 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestAtomic.h"
+#include "Lucy/Util/Atomic.h"
+
+static void
+test_cas_ptr(TestBatch *batch) {
+ int foo = 1;
+ int bar = 2;
+ int *foo_pointer = &foo;
+ int *bar_pointer = &bar;
+ int *target = NULL;
+
+ TEST_TRUE(batch,
+ Atomic_cas_ptr((void**)&target, NULL, foo_pointer),
+ "cas_ptr returns true on success");
+ TEST_TRUE(batch, target == foo_pointer, "cas_ptr sets target");
+
+ target = NULL;
+ TEST_FALSE(batch,
+ Atomic_cas_ptr((void**)&target, bar_pointer, foo_pointer),
+ "cas_ptr returns false when it old_value doesn't match");
+ TEST_TRUE(batch, target == NULL,
+ "cas_ptr doesn't do anything to target when old_value doesn't match");
+
+ target = foo_pointer;
+ TEST_TRUE(batch,
+ Atomic_cas_ptr((void**)&target, foo_pointer, bar_pointer),
+ "cas_ptr from one value to another");
+ TEST_TRUE(batch, target == bar_pointer, "cas_ptr sets target");
+}
+
+void
+TestAtomic_run_tests() {
+ TestBatch *batch = TestBatch_new(6);
+
+ TestBatch_Plan(batch);
+
+ test_cas_ptr(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Util/TestAtomic.cfh b/core/Lucy/Test/Util/TestAtomic.cfh
new file mode 100644
index 0000000..b359bd5
--- /dev/null
+++ b/core/Lucy/Test/Util/TestAtomic.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestAtomic {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestIndexFileNames.c b/core/Lucy/Test/Util/TestIndexFileNames.c
new file mode 100644
index 0000000..e450e76
--- /dev/null
+++ b/core/Lucy/Test/Util/TestIndexFileNames.c
@@ -0,0 +1,86 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestIndexFileNames.h"
+#include "Lucy/Util/IndexFileNames.h"
+
+static void
+test_local_part(TestBatch *batch) {
+ ZombieCharBuf *source = ZCB_BLANK();
+ ZombieCharBuf *got = ZCB_BLANK();
+
+ got = IxFileNames_local_part((CharBuf*)source, got);
+ TEST_TRUE(batch, ZCB_Equals(got, (Obj*)source), "simple name");
+
+ ZCB_Assign_Str(source, "foo.txt", 7);
+ got = IxFileNames_local_part((CharBuf*)source, got);
+ TEST_TRUE(batch, ZCB_Equals(got, (Obj*)source), "name with extension");
+
+ ZCB_Assign_Str(source, "/foo", 4);
+ got = IxFileNames_local_part((CharBuf*)source, got);
+ TEST_TRUE(batch, ZCB_Equals_Str(got, "foo", 3), "strip leading slash");
+
+ ZCB_Assign_Str(source, "/foo/", 5);
+ got = IxFileNames_local_part((CharBuf*)source, got);
+ TEST_TRUE(batch, ZCB_Equals_Str(got, "foo", 3), "strip trailing slash");
+
+ ZCB_Assign_Str(source, "foo/bar\\ ", 9);
+ got = IxFileNames_local_part((CharBuf*)source, got);
+ TEST_TRUE(batch, ZCB_Equals_Str(got, "bar\\ ", 5),
+ "Include garbage like backslashes and spaces");
+
+ ZCB_Assign_Str(source, "foo/bar/baz.txt", 15);
+ got = IxFileNames_local_part((CharBuf*)source, got);
+ TEST_TRUE(batch, ZCB_Equals_Str(got, "baz.txt", 7), "find last component");
+}
+
+static void
+test_extract_gen(TestBatch *batch) {
+ ZombieCharBuf *source = ZCB_WRAP_STR("", 0);
+
+ ZCB_Assign_Str(source, "seg_9", 5);
+ TEST_TRUE(batch, IxFileNames_extract_gen((CharBuf*)source) == 9,
+ "extract_gen");
+
+ ZCB_Assign_Str(source, "seg_9/", 6);
+ TEST_TRUE(batch, IxFileNames_extract_gen((CharBuf*)source) == 9,
+ "deal with trailing slash");
+
+ ZCB_Assign_Str(source, "seg_9_8", 7);
+ TEST_TRUE(batch, IxFileNames_extract_gen((CharBuf*)source) == 9,
+ "Only go past first underscore");
+
+ ZCB_Assign_Str(source, "snapshot_5.json", 15);
+ TEST_TRUE(batch, IxFileNames_extract_gen((CharBuf*)source) == 5,
+ "Deal with file suffix");
+}
+
+void
+TestIxFileNames_run_tests() {
+ TestBatch *batch = TestBatch_new(10);
+
+ TestBatch_Plan(batch);
+
+ test_local_part(batch);
+ test_extract_gen(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Util/TestIndexFileNames.cfh b/core/Lucy/Test/Util/TestIndexFileNames.cfh
new file mode 100644
index 0000000..e659d65
--- /dev/null
+++ b/core/Lucy/Test/Util/TestIndexFileNames.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestIndexFileNames cnick TestIxFileNames {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestJson.c b/core/Lucy/Test/Util/TestJson.c
new file mode 100644
index 0000000..2a47d3a
--- /dev/null
+++ b/core/Lucy/Test/Util/TestJson.c
@@ -0,0 +1,245 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestJson.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/RAMFolder.h"
+
+// Create a test data structure including at least one each of Hash, VArray,
+// and CharBuf.
+static Obj*
+S_make_dump() {
+ Hash *dump = Hash_new(0);
+ Hash_Store_Str(dump, "foo", 3, (Obj*)CB_newf("foo"));
+ Hash_Store_Str(dump, "stuff", 5, (Obj*)VA_new(0));
+ return (Obj*)dump;
+}
+
+// Test escapes for control characters ASCII 0-31.
+static char* control_escapes[] = {
+ "\\u0000",
+ "\\u0001",
+ "\\u0002",
+ "\\u0003",
+ "\\u0004",
+ "\\u0005",
+ "\\u0006",
+ "\\u0007",
+ "\\b",
+ "\\t",
+ "\\n",
+ "\\u000b",
+ "\\f",
+ "\\r",
+ "\\u000e",
+ "\\u000f",
+ "\\u0010",
+ "\\u0011",
+ "\\u0012",
+ "\\u0013",
+ "\\u0014",
+ "\\u0015",
+ "\\u0016",
+ "\\u0017",
+ "\\u0018",
+ "\\u0019",
+ "\\u001a",
+ "\\u001b",
+ "\\u001c",
+ "\\u001d",
+ "\\u001e",
+ "\\u001f",
+ NULL
+};
+
+// Test quote and backslash escape in isolation, then in context.
+static char* quote_escapes_source[] = {
+ "\"",
+ "\\",
+ "abc\"",
+ "abc\\",
+ "\"xyz",
+ "\\xyz",
+ "\\\"",
+ "\"\\",
+ NULL
+};
+static char* quote_escapes_json[] = {
+ "\\\"",
+ "\\\\",
+ "abc\\\"",
+ "abc\\\\",
+ "\\\"xyz",
+ "\\\\xyz",
+ "\\\\\\\"",
+ "\\\"\\\\",
+ NULL
+};
+
+static void
+test_escapes(TestBatch *batch) {
+ CharBuf *string = CB_new(10);
+ CharBuf *json_wanted = CB_new(10);
+
+ for (int i = 0; control_escapes[i] != NULL; i++) {
+ CB_Truncate(string, 0);
+ CB_Cat_Char(string, i);
+ char *escaped = control_escapes[i];
+ CharBuf *json = Json_to_json((Obj*)string);
+ CharBuf *decoded = (CharBuf*)Json_from_json(json);
+
+ CB_setf(json_wanted, "\"%s\"", escaped);
+ CB_Trim(json);
+ TEST_TRUE(batch, json != NULL && CB_Equals(json_wanted, (Obj*)json),
+ "encode control escape: %s", escaped);
+
+ TEST_TRUE(batch, decoded != NULL && CB_Equals(string, (Obj*)decoded),
+ "decode control escape: %s", escaped);
+
+ DECREF(json);
+ DECREF(decoded);
+ }
+
+ for (int i = 0; quote_escapes_source[i] != NULL; i++) {
+ char *source = quote_escapes_source[i];
+ char *escaped = quote_escapes_json[i];
+ CB_setf(string, source, strlen(source));
+ CharBuf *json = Json_to_json((Obj*)string);
+ CharBuf *decoded = (CharBuf*)Json_from_json(json);
+
+ CB_setf(json_wanted, "\"%s\"", escaped);
+ CB_Trim(json);
+ TEST_TRUE(batch, json != NULL && CB_Equals(json_wanted, (Obj*)json),
+ "encode quote/backslash escapes: %s", source);
+
+ TEST_TRUE(batch, decoded != NULL && CB_Equals(string, (Obj*)decoded),
+ "decode quote/backslash escapes: %s", source);
+
+ DECREF(json);
+ DECREF(decoded);
+ }
+
+ DECREF(json_wanted);
+ DECREF(string);
+}
+
+static void
+test_numbers(TestBatch *batch) {
+ Integer64 *i64 = Int64_new(33);
+ CharBuf *json = Json_to_json((Obj*)i64);
+ CB_Trim(json);
+ TEST_TRUE(batch, json && CB_Equals_Str(json, "33", 2), "Integer");
+ DECREF(json);
+
+ Float64 *f64 = Float64_new(33.33);
+ json = Json_to_json((Obj*)f64);
+ if (json) {
+ double value = CB_To_F64(json);
+ double diff = 33.33 - value;
+ if (diff < 0.0) { diff = 0.0 - diff; }
+ TEST_TRUE(batch, diff < 0.0001, "Float");
+ DECREF(json);
+ }
+ else {
+ FAIL(batch, "Float conversion to json failed.");
+ }
+
+ DECREF(i64);
+ DECREF(f64);
+}
+
+static void
+test_to_and_from(TestBatch *batch) {
+ Obj *dump = S_make_dump();
+ CharBuf *json = Json_to_json(dump);
+ Obj *got = Json_from_json(json);
+ TEST_TRUE(batch, got != NULL && Obj_Equals(dump, got),
+ "Round trip through to_json and from_json");
+ DECREF(dump);
+ DECREF(json);
+ DECREF(got);
+}
+
+static void
+test_spew_and_slurp(TestBatch *batch) {
+ Obj *dump = S_make_dump();
+ Folder *folder = (Folder*)RAMFolder_new(NULL);
+
+ CharBuf *foo = (CharBuf*)ZCB_WRAP_STR("foo", 3);
+ bool_t result = Json_spew_json(dump, folder, foo);
+ TEST_TRUE(batch, result, "spew_json returns true on success");
+ TEST_TRUE(batch, Folder_Exists(folder, foo),
+ "spew_json wrote file");
+
+ Obj *got = Json_slurp_json(folder, foo);
+ TEST_TRUE(batch, got && Obj_Equals(dump, got),
+ "Round trip through spew_json and slurp_json");
+ DECREF(got);
+
+ Err_set_error(NULL);
+ result = Json_spew_json(dump, folder, foo);
+ TEST_FALSE(batch, result, "Can't spew_json when file exists");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed spew_json sets Err_error");
+
+ Err_set_error(NULL);
+ CharBuf *bar = (CharBuf*)ZCB_WRAP_STR("bar", 3);
+ got = Json_slurp_json(folder, bar);
+ TEST_TRUE(batch, got == NULL,
+ "slurp_json returns NULL when file doesn't exist");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed slurp_json sets Err_error");
+
+ CharBuf *boffo = (CharBuf*)ZCB_WRAP_STR("boffo", 5);
+ {
+ FileHandle *fh
+ = Folder_Open_FileHandle(folder, boffo, FH_CREATE | FH_WRITE_ONLY);
+ FH_Write(fh, "garbage", 7);
+ DECREF(fh);
+ }
+ Err_set_error(NULL);
+ got = Json_slurp_json(folder, boffo);
+ TEST_TRUE(batch, got == NULL,
+ "slurp_json returns NULL when file doesn't contain valid JSON");
+ TEST_TRUE(batch, Err_get_error() != NULL,
+ "Failed slurp_json sets Err_error");
+ DECREF(got);
+
+ DECREF(dump);
+ DECREF(folder);
+}
+
+void
+TestJson_run_tests() {
+ TestBatch *batch = TestBatch_new(92);
+
+ // Liberalize for testing.
+ Json_set_tolerant(true);
+
+ TestBatch_Plan(batch);
+ test_to_and_from(batch);
+ test_escapes(batch);
+ test_numbers(batch);
+ test_spew_and_slurp(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Util/TestJson.cfh b/core/Lucy/Test/Util/TestJson.cfh
new file mode 100644
index 0000000..e54bb67
--- /dev/null
+++ b/core/Lucy/Test/Util/TestJson.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestJson {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestMemory.c b/core/Lucy/Test/Util/TestMemory.c
new file mode 100644
index 0000000..f369c01
--- /dev/null
+++ b/core/Lucy/Test/Util/TestMemory.c
@@ -0,0 +1,114 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTMEMORYPOOL
+#define C_LUCY_MEMORYPOOL
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestMemory.h"
+
+static void
+test_oversize__growth_rate(TestBatch *batch) {
+ bool_t success = true;
+ uint64_t size = 0;
+ double growth_count = 0;
+ double average_growth_rate = 0.0;
+
+ while (size < SIZE_MAX) {
+ uint64_t next_size = Memory_oversize((size_t)size + 1, sizeof(void*));
+ if (next_size < size) {
+ success = false;
+ FAIL(batch, "Asked for %" I64P ", got smaller amount %" I64P,
+ size + 1, next_size);
+ break;
+ }
+ if (size > 0) {
+ growth_count += 1;
+ double growth_rate = (double)next_size / (double)size;
+ double sum = growth_rate + (growth_count - 1) * average_growth_rate;
+ average_growth_rate = sum / growth_count;
+ if (average_growth_rate < 1.1) {
+ FAIL(batch, "Average growth rate dropped below 1.1x: %f",
+ average_growth_rate);
+ success = false;
+ break;
+ }
+ }
+ size = next_size;
+ }
+ TEST_TRUE(batch, growth_count > 0, "Grew %f times", growth_count);
+ if (success) {
+ TEST_TRUE(batch, average_growth_rate > 1.1,
+ "Growth rate of oversize() averages above 1.1: %.3f",
+ average_growth_rate);
+ }
+
+ for (int minimum = 1; minimum < 8; minimum++) {
+ uint64_t next_size = Memory_oversize(minimum, sizeof(void*));
+ double growth_rate = (double)next_size / (double)minimum;
+ TEST_TRUE(batch, growth_rate > 1.2,
+ "Growth rate is higher for smaller arrays (%d, %.3f)", minimum,
+ growth_rate);
+ }
+}
+
+static void
+test_oversize__ceiling(TestBatch *batch) {
+ for (int width = 0; width < 10; width++) {
+ size_t size = Memory_oversize(SIZE_MAX, width);
+ TEST_TRUE(batch, size == SIZE_MAX,
+ "Memory_oversize hits ceiling at SIZE_MAX (width %d)", width);
+ size = Memory_oversize(SIZE_MAX - 1, width);
+ TEST_TRUE(batch, size == SIZE_MAX,
+ "Memory_oversize hits ceiling at SIZE_MAX (width %d)", width);
+ }
+}
+
+static void
+test_oversize__rounding(TestBatch *batch) {
+ bool_t success = true;
+ int widths[] = { 1, 2, 4, 0 };
+
+ for (int width_tick = 0; widths[width_tick] != 0; width_tick++) {
+ int width = widths[width_tick];
+ for (int i = 0; i < 25; i++) {
+ size_t size = Memory_oversize(i, width);
+ size_t bytes = size * width;
+ if (bytes % sizeof(void*) != 0) {
+ FAIL(batch, "Rounding failure for %d, width %d",
+ i, width);
+ success = false;
+ return;
+ }
+ }
+ }
+ PASS(batch, "Round allocations up to the size of a pointer");
+}
+
+void
+TestMemory_run_tests() {
+ TestBatch *batch = TestBatch_new(30);
+
+ TestBatch_Plan(batch);
+ test_oversize__growth_rate(batch);
+ test_oversize__ceiling(batch);
+ test_oversize__rounding(batch);
+
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Util/TestMemory.cfh b/core/Lucy/Test/Util/TestMemory.cfh
new file mode 100644
index 0000000..bf5e565
--- /dev/null
+++ b/core/Lucy/Test/Util/TestMemory.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestMemory {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestMemoryPool.c b/core/Lucy/Test/Util/TestMemoryPool.c
new file mode 100644
index 0000000..35551ed
--- /dev/null
+++ b/core/Lucy/Test/Util/TestMemoryPool.c
@@ -0,0 +1,56 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTMEMORYPOOL
+#define C_LUCY_MEMORYPOOL
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestMemoryPool.h"
+#include "Lucy/Util/MemoryPool.h"
+
+void
+TestMemPool_run_tests() {
+ TestBatch *batch = TestBatch_new(4);
+ MemoryPool *mem_pool = MemPool_new(0);
+ MemoryPool *other = MemPool_new(0);
+ char *ptr_a, *ptr_b;
+
+ TestBatch_Plan(batch);
+
+ ptr_a = (char*)MemPool_Grab(mem_pool, 10);
+ strcpy(ptr_a, "foo");
+ MemPool_Release_All(mem_pool);
+
+ ptr_b = (char*)MemPool_Grab(mem_pool, 10);
+ TEST_STR_EQ(batch, ptr_b, "foo", "Recycle RAM on Release_All");
+
+ ptr_a = mem_pool->buf;
+ MemPool_Resize(mem_pool, ptr_b, 6);
+ TEST_TRUE(batch, mem_pool->buf < ptr_a, "Resize");
+
+ ptr_a = (char*)MemPool_Grab(other, 20);
+ MemPool_Release_All(other);
+ MemPool_Eat(other, mem_pool);
+ TEST_TRUE(batch, other->buf == mem_pool->buf, "Eat");
+ TEST_TRUE(batch, other->buf != NULL, "Eat");
+
+ DECREF(mem_pool);
+ DECREF(other);
+ DECREF(batch);
+}
+
+
diff --git a/core/Lucy/Test/Util/TestMemoryPool.cfh b/core/Lucy/Test/Util/TestMemoryPool.cfh
new file mode 100644
index 0000000..a61d8f5
--- /dev/null
+++ b/core/Lucy/Test/Util/TestMemoryPool.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestMemoryPool cnick TestMemPool {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestNumberUtils.c b/core/Lucy/Test/Util/TestNumberUtils.c
new file mode 100644
index 0000000..6e63b1e
--- /dev/null
+++ b/core/Lucy/Test/Util/TestNumberUtils.c
@@ -0,0 +1,370 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTNUMBERUTILS
+#include "Lucy/Util/ToolSet.h"
+#include <stdlib.h>
+#include <time.h>
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/TestUtils.h"
+#include "Lucy/Test/Util/TestNumberUtils.h"
+#include "Lucy/Util/NumberUtils.h"
+
+static void
+test_u1(TestBatch *batch) {
+ size_t count = 64;
+ uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, 2);
+ size_t amount = count / 8;
+ uint8_t *bits = (uint8_t*)CALLOCATE(amount, sizeof(uint8_t));
+
+ for (size_t i = 0; i < count; i++) {
+ if (ints[i]) { NumUtil_u1set(bits, i); }
+ }
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(batch, NumUtil_u1get(bits, i), (long)ints[i],
+ "u1 set/get");
+ }
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_u1flip(bits, i);
+ }
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(batch, NumUtil_u1get(bits, i), !ints[i], "u1 flip");
+ }
+
+ FREEMEM(bits);
+ FREEMEM(ints);
+}
+
+static void
+test_u2(TestBatch *batch) {
+ size_t count = 32;
+ uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, 4);
+ uint8_t *bits = (uint8_t*)CALLOCATE((count / 4), sizeof(uint8_t));
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_u2set(bits, i, (uint8_t)ints[i]);
+ }
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(batch, NumUtil_u2get(bits, i), (long)ints[i], "u2");
+ }
+
+ FREEMEM(bits);
+ FREEMEM(ints);
+}
+
+static void
+test_u4(TestBatch *batch) {
+ size_t count = 128;
+ uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, 16);
+ uint8_t *bits = (uint8_t*)CALLOCATE((count / 2), sizeof(uint8_t));
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_u4set(bits, i, (uint8_t)ints[i]);
+ }
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(batch, NumUtil_u4get(bits, i), (long)ints[i], "u4");
+ }
+
+ FREEMEM(bits);
+ FREEMEM(ints);
+}
+
+static void
+test_c32(TestBatch *batch) {
+ uint64_t mins[] = { 0, 0x4000 - 100, (uint32_t)I32_MAX - 100, U32_MAX - 10 };
+ uint64_t limits[] = { 500, 0x4000 + 100, (uint32_t)I32_MAX + 100, U32_MAX };
+ uint32_t set_num;
+ uint32_t num_sets = sizeof(mins) / sizeof(uint64_t);
+ size_t count = 64;
+ uint64_t *ints = NULL;
+ size_t amount = count * C32_MAX_BYTES;
+ char *encoded = (char*)CALLOCATE(amount, sizeof(char));
+ char *target = encoded;
+ char *limit = target + amount;
+
+ for (set_num = 0; set_num < num_sets; set_num++) {
+ char *skip;
+ ints = TestUtils_random_u64s(ints, count,
+ mins[set_num], limits[set_num]);
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_c32((uint32_t)ints[i], &target);
+ }
+ target = encoded;
+ skip = encoded;
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(batch, NumUtil_decode_c32(&target), (long)ints[i],
+ "c32 %lu", (long)ints[i]);
+ NumUtil_skip_cint(&skip);
+ if (target > limit) { THROW(ERR, "overrun"); }
+ }
+ TEST_TRUE(batch, skip == target, "skip %lu == %lu",
+ (unsigned long)skip, (unsigned long)target);
+
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_padded_c32((uint32_t)ints[i], &target);
+ }
+ TEST_TRUE(batch, target == limit,
+ "padded c32 uses 5 bytes (%lu == %lu)", (unsigned long)target,
+ (unsigned long)limit);
+ target = encoded;
+ skip = encoded;
+ for (size_t i = 0; i < count; i++) {
+ TEST_INT_EQ(batch, NumUtil_decode_c32(&target), (long)ints[i],
+ "padded c32 %lu", (long)ints[i]);
+ NumUtil_skip_cint(&skip);
+ if (target > limit) { THROW(ERR, "overrun"); }
+ }
+ TEST_TRUE(batch, skip == target, "skip padded %lu == %lu",
+ (unsigned long)skip, (unsigned long)target);
+ }
+
+ target = encoded;
+ NumUtil_encode_c32(U32_MAX, &target);
+ target = encoded;
+ TEST_INT_EQ(batch, NumUtil_decode_c32(&target), U32_MAX, "c32 U32_MAX");
+
+ FREEMEM(encoded);
+ FREEMEM(ints);
+}
+
+static void
+test_c64(TestBatch *batch) {
+ uint64_t mins[] = { 0, 0x4000 - 100, (uint64_t)U32_MAX - 100, U64_MAX - 10 };
+ uint64_t limits[] = { 500, 0x4000 + 100, (uint64_t)U32_MAX + 1000, U64_MAX };
+ uint32_t set_num;
+ uint32_t num_sets = sizeof(mins) / sizeof(uint64_t);
+ size_t count = 64;
+ uint64_t *ints = NULL;
+ size_t amount = count * C64_MAX_BYTES;
+ char *encoded = (char*)CALLOCATE(amount, sizeof(char));
+ char *target = encoded;
+ char *limit = target + amount;
+
+ for (set_num = 0; set_num < num_sets; set_num++) {
+ char *skip;
+ ints = TestUtils_random_u64s(ints, count,
+ mins[set_num], limits[set_num]);
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_c64(ints[i], &target);
+ }
+ target = encoded;
+ skip = encoded;
+ for (size_t i = 0; i < count; i++) {
+ uint64_t got = NumUtil_decode_c64(&target);
+ TEST_TRUE(batch, got == ints[i],
+ "c64 %" U64P " == %" U64P, got, ints[i]);
+ if (target > limit) { THROW(ERR, "overrun"); }
+ NumUtil_skip_cint(&skip);
+ }
+ TEST_TRUE(batch, skip == target, "skip %lu == %lu",
+ (unsigned long)skip, (unsigned long)target);
+ }
+
+ target = encoded;
+ NumUtil_encode_c64(U64_MAX, &target);
+ target = encoded;
+ {
+ uint64_t got = NumUtil_decode_c64(&target);
+ TEST_TRUE(batch, got == U64_MAX, "c64 U64_MAX");
+ }
+
+ FREEMEM(encoded);
+ FREEMEM(ints);
+}
+
+static void
+test_bigend_u16(TestBatch *batch) {
+ size_t count = 32;
+ uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, U16_MAX + 1);
+ size_t amount = (count + 1) * sizeof(uint16_t);
+ char *allocated = (char*)CALLOCATE(amount, sizeof(char));
+ char *encoded = allocated + 1; // Intentionally misaligned.
+ char *target = encoded;
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_bigend_u16((uint16_t)ints[i], &target);
+ target += sizeof(uint16_t);
+ }
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ uint16_t got = NumUtil_decode_bigend_u16(target);
+ TEST_INT_EQ(batch, got, (long)ints[i], "bigend u16");
+ target += sizeof(uint16_t);
+ }
+
+ target = encoded;
+ NumUtil_encode_bigend_u16(1, &target);
+ TEST_INT_EQ(batch, encoded[0], 0, "Truly big-endian u16");
+ TEST_INT_EQ(batch, encoded[1], 1, "Truly big-endian u16");
+
+ FREEMEM(allocated);
+ FREEMEM(ints);
+}
+
+static void
+test_bigend_u32(TestBatch *batch) {
+ size_t count = 32;
+ uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, U64_C(1) + U32_MAX);
+ size_t amount = (count + 1) * sizeof(uint32_t);
+ char *allocated = (char*)CALLOCATE(amount, sizeof(char));
+ char *encoded = allocated + 1; // Intentionally misaligned.
+ char *target = encoded;
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_bigend_u32((uint32_t)ints[i], &target);
+ target += sizeof(uint32_t);
+ }
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ uint32_t got = NumUtil_decode_bigend_u32(target);
+ TEST_INT_EQ(batch, got, (long)ints[i], "bigend u32");
+ target += sizeof(uint32_t);
+ }
+
+ target = encoded;
+ NumUtil_encode_bigend_u32(1, &target);
+ TEST_INT_EQ(batch, encoded[0], 0, "Truly big-endian u32");
+ TEST_INT_EQ(batch, encoded[3], 1, "Truly big-endian u32");
+
+ FREEMEM(allocated);
+ FREEMEM(ints);
+}
+
+static void
+test_bigend_u64(TestBatch *batch) {
+ size_t count = 32;
+ uint64_t *ints = TestUtils_random_u64s(NULL, count, 0, U64_MAX);
+ size_t amount = (count + 1) * sizeof(uint64_t);
+ char *allocated = (char*)CALLOCATE(amount, sizeof(char));
+ char *encoded = allocated + 1; // Intentionally misaligned.
+ char *target = encoded;
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_bigend_u64(ints[i], &target);
+ target += sizeof(uint64_t);
+ }
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ uint64_t got = NumUtil_decode_bigend_u64(target);
+ TEST_TRUE(batch, got == ints[i], "bigend u64");
+ target += sizeof(uint64_t);
+ }
+
+ target = encoded;
+ NumUtil_encode_bigend_u64(1, &target);
+ TEST_INT_EQ(batch, encoded[0], 0, "Truly big-endian");
+ TEST_INT_EQ(batch, encoded[7], 1, "Truly big-endian");
+
+ FREEMEM(allocated);
+ FREEMEM(ints);
+}
+
+static void
+test_bigend_f32(TestBatch *batch) {
+ float source[] = { -1.3f, 0.0f, 100.2f };
+ size_t count = 3;
+ size_t amount = (count + 1) * sizeof(float);
+ uint8_t *allocated = (uint8_t*)CALLOCATE(amount, sizeof(uint8_t));
+ uint8_t *encoded = allocated + 1; // Intentionally misaligned.
+ uint8_t *target = encoded;
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_bigend_f32(source[i], &target);
+ target += sizeof(float);
+ }
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ float got = NumUtil_decode_bigend_f32(target);
+ TEST_TRUE(batch, got == source[i], "bigend f32");
+ target += sizeof(float);
+ }
+
+ target = encoded;
+ NumUtil_encode_bigend_f32(-2.0f, &target);
+ TEST_INT_EQ(batch, (encoded[0] & 0x80), 0x80,
+ "Truly big-endian (IEEE 754 sign bit set for negative number)");
+ TEST_INT_EQ(batch, encoded[0], 0xC0,
+ "IEEE 754 representation of -2.0f, byte 0");
+ for (size_t i = 1; i < sizeof(float); i++) {
+ TEST_INT_EQ(batch, encoded[i], 0,
+ "IEEE 754 representation of -2.0f, byte %d", (int)i);
+ }
+
+ FREEMEM(allocated);
+}
+
+static void
+test_bigend_f64(TestBatch *batch) {
+ double source[] = { -1.3, 0.0, 100.2 };
+ size_t count = 3;
+ size_t amount = (count + 1) * sizeof(double);
+ uint8_t *allocated = (uint8_t*)CALLOCATE(amount, sizeof(uint8_t));
+ uint8_t *encoded = allocated + 1; // Intentionally misaligned.
+ uint8_t *target = encoded;
+
+ for (size_t i = 0; i < count; i++) {
+ NumUtil_encode_bigend_f64(source[i], &target);
+ target += sizeof(double);
+ }
+ target = encoded;
+ for (size_t i = 0; i < count; i++) {
+ double got = NumUtil_decode_bigend_f64(target);
+ TEST_TRUE(batch, got == source[i], "bigend f64");
+ target += sizeof(double);
+ }
+
+ target = encoded;
+ NumUtil_encode_bigend_f64(-2.0, &target);
+ TEST_INT_EQ(batch, (encoded[0] & 0x80), 0x80,
+ "Truly big-endian (IEEE 754 sign bit set for negative number)");
+ TEST_INT_EQ(batch, encoded[0], 0xC0,
+ "IEEE 754 representation of -2.0, byte 0");
+ for (size_t i = 1; i < sizeof(double); i++) {
+ TEST_INT_EQ(batch, encoded[i], 0,
+ "IEEE 754 representation of -2.0, byte %d", (int)i);
+ }
+
+ FREEMEM(allocated);
+}
+
+void
+TestNumUtil_run_tests() {
+ TestBatch *batch = TestBatch_new(1196);
+
+ TestBatch_Plan(batch);
+ srand((unsigned int)time((time_t*)NULL));
+
+ test_u1(batch);
+ test_u2(batch);
+ test_u4(batch);
+ test_c32(batch);
+ test_c64(batch);
+ test_bigend_u16(batch);
+ test_bigend_u32(batch);
+ test_bigend_u64(batch);
+ test_bigend_f32(batch);
+ test_bigend_f64(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Util/TestNumberUtils.cfh b/core/Lucy/Test/Util/TestNumberUtils.cfh
new file mode 100644
index 0000000..2b45719
--- /dev/null
+++ b/core/Lucy/Test/Util/TestNumberUtils.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestNumberUtils cnick TestNumUtil {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestPriorityQueue.c b/core/Lucy/Test/Util/TestPriorityQueue.c
new file mode 100644
index 0000000..e08d78b
--- /dev/null
+++ b/core/Lucy/Test/Util/TestPriorityQueue.c
@@ -0,0 +1,165 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_TESTPRIORITYQUEUE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestPriorityQueue.h"
+#include "Lucy/Util/PriorityQueue.h"
+
+NumPriorityQueue*
+NumPriQ_new(uint32_t max_size) {
+ NumPriorityQueue *self
+ = (NumPriorityQueue*)VTable_Make_Obj(NUMPRIORITYQUEUE);
+ return (NumPriorityQueue*)PriQ_init((PriorityQueue*)self, max_size);
+}
+
+bool_t
+NumPriQ_less_than(NumPriorityQueue *self, Obj *a, Obj *b) {
+ Float64 *num_a = (Float64*)a;
+ Float64 *num_b = (Float64*)b;
+ UNUSED_VAR(self);
+ return Float64_Get_Value(num_a) < Float64_Get_Value(num_b) ? true : false;
+}
+
+static void
+S_insert_num(NumPriorityQueue *pq, int32_t value) {
+ NumPriQ_Insert(pq, (Obj*)Float64_new((double)value));
+}
+
+static int32_t
+S_pop_num(NumPriorityQueue *pq) {
+ Float64 *num = (Float64*)NumPriQ_Pop(pq);
+ int32_t retval;
+ if (!num) { THROW(ERR, "Queue is empty"); }
+ retval = (int32_t)Float64_Get_Value(num);
+ DECREF(num);
+ return retval;
+}
+
+static void
+test_Peek_and_Pop_All(TestBatch *batch) {
+ NumPriorityQueue *pq = NumPriQ_new(5);
+ Float64 *val;
+
+ S_insert_num(pq, 3);
+ S_insert_num(pq, 1);
+ S_insert_num(pq, 2);
+ S_insert_num(pq, 20);
+ S_insert_num(pq, 10);
+ val = (Float64*)CERTIFY(NumPriQ_Peek(pq), FLOAT64);
+ TEST_INT_EQ(batch, (long)Float64_Get_Value(val), 1,
+ "peek at the least item in the queue");
+ {
+ VArray *got = NumPriQ_Pop_All(pq);
+
+ val = (Float64*)CERTIFY(VA_Fetch(got, 0), FLOAT64);
+ TEST_INT_EQ(batch, (long)Float64_Get_Value(val), 20, "pop_all");
+ val = (Float64*)CERTIFY(VA_Fetch(got, 1), FLOAT64);
+ TEST_INT_EQ(batch, (long)Float64_Get_Value(val), 10, "pop_all");
+ val = (Float64*)CERTIFY(VA_Fetch(got, 2), FLOAT64);
+ TEST_INT_EQ(batch, (long)Float64_Get_Value(val), 3, "pop_all");
+ val = (Float64*)CERTIFY(VA_Fetch(got, 3), FLOAT64);
+ TEST_INT_EQ(batch, (long)Float64_Get_Value(val), 2, "pop_all");
+ val = (Float64*)CERTIFY(VA_Fetch(got, 4), FLOAT64);
+ TEST_INT_EQ(batch, (long)Float64_Get_Value(val), 1, "pop_all");
+
+ DECREF(got);
+ }
+
+ DECREF(pq);
+}
+
+static void
+test_Insert_and_Pop(TestBatch *batch) {
+ NumPriorityQueue *pq = NumPriQ_new(5);
+
+ S_insert_num(pq, 3);
+ S_insert_num(pq, 1);
+ S_insert_num(pq, 2);
+ S_insert_num(pq, 20);
+ S_insert_num(pq, 10);
+
+ TEST_INT_EQ(batch, S_pop_num(pq), 1, "Pop");
+ TEST_INT_EQ(batch, S_pop_num(pq), 2, "Pop");
+ TEST_INT_EQ(batch, S_pop_num(pq), 3, "Pop");
+ TEST_INT_EQ(batch, S_pop_num(pq), 10, "Pop");
+
+ S_insert_num(pq, 7);
+ TEST_INT_EQ(batch, S_pop_num(pq), 7,
+ "Insert after Pop still sorts correctly");
+
+ DECREF(pq);
+}
+
+static void
+test_discard(TestBatch *batch) {
+ int32_t i;
+ NumPriorityQueue *pq = NumPriQ_new(5);
+
+ for (i = 1; i <= 10; i++) { S_insert_num(pq, i); }
+ S_insert_num(pq, -3);
+ for (i = 1590; i <= 1600; i++) { S_insert_num(pq, i); }
+ S_insert_num(pq, 5);
+
+ TEST_INT_EQ(batch, S_pop_num(pq), 1596, "discard waste");
+ TEST_INT_EQ(batch, S_pop_num(pq), 1597, "discard waste");
+ TEST_INT_EQ(batch, S_pop_num(pq), 1598, "discard waste");
+ TEST_INT_EQ(batch, S_pop_num(pq), 1599, "discard waste");
+ TEST_INT_EQ(batch, S_pop_num(pq), 1600, "discard waste");
+
+ DECREF(pq);
+}
+
+static void
+test_random_insertion(TestBatch *batch) {
+ int i;
+ int shuffled[64];
+ NumPriorityQueue *pq = NumPriQ_new(64);
+
+ for (i = 0; i < 64; i++) { shuffled[i] = i; }
+ for (i = 0; i < 64; i++) {
+ int shuffle_pos = rand() % 64;
+ int temp = shuffled[shuffle_pos];
+ shuffled[shuffle_pos] = shuffled[i];
+ shuffled[i] = temp;
+ }
+ for (i = 0; i < 64; i++) { S_insert_num(pq, shuffled[i]); }
+ for (i = 0; i < 64; i++) {
+ if (S_pop_num(pq) != i) { break; }
+ }
+ TEST_INT_EQ(batch, i, 64, "random insertion");
+
+ DECREF(pq);
+}
+
+void
+TestPriQ_run_tests() {
+ TestBatch *batch = TestBatch_new(17);
+
+ TestBatch_Plan(batch);
+
+ test_Peek_and_Pop_All(batch);
+ test_Insert_and_Pop(batch);
+ test_discard(batch);
+ test_random_insertion(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Util/TestPriorityQueue.cfh b/core/Lucy/Test/Util/TestPriorityQueue.cfh
new file mode 100644
index 0000000..e389fc1
--- /dev/null
+++ b/core/Lucy/Test/Util/TestPriorityQueue.cfh
@@ -0,0 +1,34 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class Lucy::Test::Util::NumPriorityQueue cnick NumPriQ
+ inherits Lucy::Util::PriorityQueue {
+
+ inert incremented NumPriorityQueue*
+ new(uint32_t max_size);
+
+ bool_t
+ Less_Than(NumPriorityQueue *self, Obj *a, Obj *b);
+}
+
+inert class Lucy::Test::Util::TestPriorityQueue cnick TestPriQ {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Test/Util/TestStringHelper.c b/core/Lucy/Test/Util/TestStringHelper.c
new file mode 100644
index 0000000..55cb538
--- /dev/null
+++ b/core/Lucy/Test/Util/TestStringHelper.c
@@ -0,0 +1,127 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Util/TestStringHelper.h"
+#include "Lucy/Util/StringHelper.h"
+
+static void
+test_overlap(TestBatch *batch) {
+ int32_t result;
+ result = StrHelp_overlap("", "", 0, 0);
+ TEST_INT_EQ(batch, result, 0, "two empty strings");
+ result = StrHelp_overlap("", "foo", 0, 3);
+ TEST_INT_EQ(batch, result, 0, "first string is empty");
+ result = StrHelp_overlap("foo", "", 3, 0);
+ TEST_INT_EQ(batch, result, 0, "second string is empty");
+ result = StrHelp_overlap("foo", "foo", 3, 3);
+ TEST_INT_EQ(batch, result, 3, "equal strings");
+ result = StrHelp_overlap("foo bar", "foo", 7, 3);
+ TEST_INT_EQ(batch, result, 3, "first string is longer");
+ result = StrHelp_overlap("foo", "foo bar", 3, 7);
+ TEST_INT_EQ(batch, result, 3, "second string is longer");
+}
+
+
+static void
+test_to_base36(TestBatch *batch) {
+ char buffer[StrHelp_MAX_BASE36_BYTES];
+ StrHelp_to_base36(U64_MAX, buffer);
+ TEST_STR_EQ(batch, "3w5e11264sgsf", buffer, "base36 U64_MAX");
+ StrHelp_to_base36(1, buffer);
+ TEST_STR_EQ(batch, "1", buffer, "base36 1");
+ TEST_INT_EQ(batch, buffer[1], 0, "base36 NULL termination");
+}
+
+static void
+S_round_trip_utf8_code_point(TestBatch *batch, uint32_t code_point) {
+ char buffer[4];
+ uint32_t len = StrHelp_encode_utf8_char(code_point, buffer);
+ char *start = buffer;
+ char *end = start + len;
+ TEST_TRUE(batch, StrHelp_utf8_valid(buffer, len), "Valid UTF-8 for %lu",
+ (unsigned long)code_point);
+ TEST_INT_EQ(batch, len, StrHelp_UTF8_COUNT[(unsigned char)buffer[0]],
+ "length returned for %lu", (unsigned long)code_point);
+ TEST_TRUE(batch, StrHelp_back_utf8_char(end, start) == start,
+ "back_utf8_char for %lu", (unsigned long)code_point);
+ TEST_INT_EQ(batch, StrHelp_decode_utf8_char(buffer), code_point,
+ "round trip encode and decode for %lu", (unsigned long)code_point);
+}
+
+static void
+test_utf8_round_trip(TestBatch *batch) {
+ uint32_t code_points[] = {
+ 0,
+ 0xA, // newline
+ 'a',
+ 128, // two-byte
+ 0x263A, // smiley (three-byte)
+ 0x10FFFF, // Max legal code point (four-byte).
+ };
+ uint32_t num_code_points = sizeof(code_points) / sizeof(uint32_t);
+ uint32_t i;
+ for (i = 0; i < num_code_points; i++) {
+ S_round_trip_utf8_code_point(batch, code_points[i]);
+ }
+}
+
+static void
+test_is_whitespace(TestBatch *batch) {
+ TEST_TRUE(batch, StrHelp_is_whitespace(' '), "space is whitespace");
+ TEST_TRUE(batch, StrHelp_is_whitespace('\n'), "newline is whitespace");
+ TEST_TRUE(batch, StrHelp_is_whitespace('\t'), "tab is whitespace");
+ TEST_TRUE(batch, StrHelp_is_whitespace('\v'),
+ "vertical tab is whitespace");
+ TEST_TRUE(batch, StrHelp_is_whitespace(0x180E),
+ "Mongolian vowel separator is whitespace");
+ TEST_FALSE(batch, StrHelp_is_whitespace('a'), "'a' isn't whitespace");
+ TEST_FALSE(batch, StrHelp_is_whitespace(0), "NULL isn't whitespace");
+ TEST_FALSE(batch, StrHelp_is_whitespace(0x263A),
+ "Smiley isn't whitespace");
+}
+
+static void
+test_back_utf8_char(TestBatch *batch) {
+ char buffer[4];
+ char *buf = buffer + 1;
+ uint32_t len = StrHelp_encode_utf8_char(0x263A, buffer);
+ char *end = buffer + len;
+ TEST_TRUE(batch, StrHelp_back_utf8_char(end, buffer) == buffer,
+ "back_utf8_char");
+ TEST_TRUE(batch, StrHelp_back_utf8_char(end, buf) == NULL,
+ "back_utf8_char returns NULL rather than back up beyond start");
+}
+
+void
+TestStrHelp_run_tests() {
+ TestBatch *batch = TestBatch_new(43);
+
+ TestBatch_Plan(batch);
+
+ test_overlap(batch);
+ test_to_base36(batch);
+ test_utf8_round_trip(batch);
+ test_is_whitespace(batch);
+ test_back_utf8_char(batch);
+
+ DECREF(batch);
+}
+
+
+
diff --git a/core/Lucy/Test/Util/TestStringHelper.cfh b/core/Lucy/Test/Util/TestStringHelper.cfh
new file mode 100644
index 0000000..2cee286
--- /dev/null
+++ b/core/Lucy/Test/Util/TestStringHelper.cfh
@@ -0,0 +1,24 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Test::Util::TestStringHelper cnick TestStrHelp {
+ inert void
+ run_tests();
+}
+
+
diff --git a/core/Lucy/Util/Atomic.c b/core/Lucy/Util/Atomic.c
new file mode 100644
index 0000000..18681cf
--- /dev/null
+++ b/core/Lucy/Util/Atomic.c
@@ -0,0 +1,40 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_ATOMIC
+#define LUCY_USE_SHORT_NAMES
+#include "Lucy/Util/Atomic.h"
+
+/********************************** Windows ********************************/
+#ifdef CHY_HAS_WINDOWS_H
+#include <windows.h>
+
+chy_bool_t
+lucy_Atomic_wrapped_cas_ptr(void *volatile *target, void *old_value,
+ void *new_value) {
+ return InterlockedCompareExchangePointer(target, new_value, old_value)
+ == old_value;
+}
+
+/************************** Fall back to ptheads ***************************/
+#elif defined(CHY_HAS_PTHREAD_H)
+
+#include <pthread.h>
+pthread_mutex_t lucy_Atomic_mutex = PTHREAD_MUTEX_INITIALIZER;
+
+#endif
+
+
diff --git a/core/Lucy/Util/Atomic.cfh b/core/Lucy/Util/Atomic.cfh
new file mode 100644
index 0000000..18d14c3
--- /dev/null
+++ b/core/Lucy/Util/Atomic.cfh
@@ -0,0 +1,95 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Provide atomic memory operations.
+ */
+inert class Lucy::Util::Atomic { }
+
+__C__
+
+/** Compare and swap a pointer. Test whether the value at <code>target</code>
+ * matches <code>old_value</code>. If it does, set <code>target</code> to
+ * <code>new_value</code> and return true. Otherwise, return false.
+ */
+static CHY_INLINE chy_bool_t
+lucy_Atomic_cas_ptr(void *volatile *target, void *old_value, void *new_value);
+
+/************************** Mac OS X 10.4 and later ***********************/
+#ifdef CHY_HAS_OSATOMIC_CAS_PTR
+#include <libkern/OSAtomic.h>
+
+static CHY_INLINE chy_bool_t
+lucy_Atomic_cas_ptr(void *volatile *target, void *old_value, void *new_value) {
+ return OSAtomicCompareAndSwapPtr(old_value, new_value, target);
+}
+
+/********************************** Windows *******************************/
+#elif defined(CHY_HAS_WINDOWS_H)
+
+chy_bool_t
+lucy_Atomic_wrapped_cas_ptr(void *volatile *target, void *old_value,
+ void *new_value);
+
+static CHY_INLINE chy_bool_t
+lucy_Atomic_cas_ptr(void *volatile *target, void *old_value, void *new_value) {
+ return lucy_Atomic_wrapped_cas_ptr(target, old_value, new_value);
+}
+
+/**************************** Solaris 10 and later ************************/
+#elif defined(CHY_HAS_SYS_ATOMIC_H)
+#include <sys/atomic.h>
+
+static CHY_INLINE chy_bool_t
+lucy_Atomic_cas_ptr(void *volatile *target, void *old_value, void *new_value) {
+ return atomic_cas_ptr(target, old_value, new_value) == old_value;
+}
+
+/************************ Fall back to pthread.h. **************************/
+#elif defined(CHY_HAS_PTHREAD_H)
+#include <pthread.h>
+
+extern pthread_mutex_t lucy_Atomic_mutex;
+
+static CHY_INLINE chy_bool_t
+lucy_Atomic_cas_ptr(void *volatile *target, void *old_value, void *new_value) {
+ pthread_mutex_lock(&lucy_Atomic_mutex);
+ if (*target == old_value) {
+ *target = new_value;
+ pthread_mutex_unlock(&lucy_Atomic_mutex);
+ return true;
+ }
+ else {
+ pthread_mutex_unlock(&lucy_Atomic_mutex);
+ return false;
+ }
+}
+
+/******************** No support for atomics at all. ***********************/
+#else
+
+#error "No support for atomic operations."
+
+#endif // Big platform if-else chain.
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define Atomic_cas_ptr lucy_Atomic_cas_ptr
+#endif
+
+__END_C__
+
+
diff --git a/core/Lucy/Util/Debug.c b/core/Lucy/Util/Debug.c
new file mode 100644
index 0000000..7201cab
--- /dev/null
+++ b/core/Lucy/Util/Debug.c
@@ -0,0 +1,128 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DEBUG
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "Lucy/Util/Debug.h"
+
+int32_t Debug_num_allocated = 0;
+int32_t Debug_num_freed = 0;
+int32_t Debug_num_globals = 0;
+
+#if DEBUG_ENABLED
+
+#include <stdarg.h>
+#include <ctype.h>
+#include <string.h>
+#include <stdlib.h>
+
+static char env_cache_buf[256];
+static char *env_cache = NULL;
+static char *env_cache_limit = NULL;
+static int env_cache_is_current = 0;
+
+// Cache the system call to getenv.
+static void
+S_cache_debug_env_var(char *override) {
+ const char *debug_env = override ? override : getenv("DEBUG");
+ if (debug_env != NULL) {
+ size_t len = strlen(debug_env);
+ if (len > sizeof(env_cache_buf) - 1) {
+ len = sizeof(env_cache_buf) - 1;
+ }
+ strncpy(env_cache_buf, debug_env, len);
+ env_cache = env_cache_buf;
+ env_cache_limit = env_cache + len;
+ }
+ env_cache_is_current = 1;
+}
+
+void
+Debug_print_mess(const char *file, int line, const char *func,
+ const char *pat, ...) {
+ va_list args;
+ fprintf(stderr, "%s:%d %s(): ", file, line, func);
+ va_start(args, pat);
+ vfprintf(stderr, pat, args);
+ va_end(args);
+ fprintf(stderr, "\n");
+}
+
+int
+Debug_debug_should_print(const char *path, const char *func) {
+ if (!env_cache_is_current) {
+ S_cache_debug_env_var(NULL);
+ }
+
+ if (!env_cache) {
+ // Do not print if DEBUG environment var is not set.
+ return 0;
+ }
+ else {
+ const char *test, *next;
+ const char *file = strrchr(path, '/');
+ const int filename_len = file ? strlen(file) : 0;
+ const int funcname_len = func ? strlen(func) : 0;
+
+ // Use just file name if given path.
+ if (file) { file++; }
+ else { file = path; }
+
+ // Split criteria on commas. Bail when we run out of critieria.
+ for (test = env_cache; test != NULL; test = next) {
+ const char *last_char;
+
+ // Skip whitespace.
+ while (isspace(*test)) { test++; }
+ if (test >= env_cache_limit) { return 0; }
+
+ // Find end of criteria or end of string.
+ next = strchr(test, ',');
+ last_char = next ? next - 1 : env_cache_limit - 1;
+ while (last_char > test && isspace(*last_char)) { last_char--; }
+
+ if (*last_char == '*') {
+ const int len = last_char - test;
+ if (!strncmp(test, file, len)) { return 1; }
+ if (!strncmp(test, func, len)) { return 1; }
+ }
+ else {
+ if (!strncmp(test, file, filename_len)) { return 1; }
+ if (!strncmp(test, func, funcname_len)) { return 1; }
+ }
+ }
+
+ // No matches against the DEBUG environment var, so don't print.
+ return 0;
+ }
+}
+
+void
+Debug_set_env_cache(char *override) {
+ S_cache_debug_env_var(override);
+}
+
+#else // DEBUG
+
+void
+Debug_set_env_cache(char *override) {
+ (void)override;
+}
+
+#endif // DEBUG
+
diff --git a/core/Lucy/Util/Debug.cfh b/core/Lucy/Util/Debug.cfh
new file mode 100644
index 0000000..05e1df8
--- /dev/null
+++ b/core/Lucy/Util/Debug.cfh
@@ -0,0 +1,143 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** The Debug module provides multiple levels of debugging verbosity. Code for
+ * debug statements is only compiled "#ifdef LUCY_DEBUG" at compile-time.
+ * Some statements will then always print; additional output can be enabled
+ * using the environment variable LUCY_DEBUG. Examples:
+ *
+ * LUCY_DEBUG=file.C -> all debug statements in path/file.C
+ * LUCY_DEBUG=func -> all in functions named exactly 'func'
+ * LUCY_DEBUG=f* -> all in functions (or files) starting with 'f'
+ * LUCY_DEBUG=file* -> all in files (or functions) ending with file*'
+ * LUCY_DEBUG=func1,func2 -> either in func1 or in func2
+ * LUCY_DEBUG=* -> just print all debug statements
+ *
+ * The wildcard character '*' can only go at the end of an identifier.
+ */
+
+inert class Lucy::Util::Debug {
+
+ /** Private function, used only by the DEBUG macros.
+ */
+ inert void
+ print_mess(const char *file, int line, const char *func,
+ const char *pat, ...);
+
+ /** Private function, used only by the DEBUG macros.
+ */
+ inert int
+ debug_should_print(const char *path, const char *func);
+
+ /** Force override in cached value of LUCY_DEBUG environment variable.
+ */
+ inert void
+ set_env_cache(char *override);
+
+ /* Under LUCY_DEBUG, track the number of objects allocated, the number
+ * freed, and the number of global objects. If, after all non-global
+ * objects should have been cleaned up, these numbers don't balance out,
+ * there's a memory leak somewhere.
+ */
+ inert int32_t num_allocated;
+ inert int32_t num_freed;
+ inert int32_t num_globals;
+}
+
+__C__
+#ifdef LUCY_DEBUG
+
+#undef LUCY_DEBUG // undef prior to redefining the command line argument
+#define LUCY_DEBUG_ENABLED 1
+
+#include <stdio.h>
+#include <stdlib.h>
+
+/** Unconditionally print debug statement prepending file and line info.
+ */
+#define LUCY_DEBUG_PRINT(args...) \
+ lucy_Debug_print_mess(__FILE__, __LINE__, __func__, ##args)
+
+/** Conditionally execute code if debugging enabled via LUCY_DEBUG environment
+ * variable.
+ */
+#define LUCY_DEBUG_DO(actions) \
+ do { \
+ static int initialized = 0; \
+ static int do_it = 0; \
+ if (!initialized) { \
+ initialized = 1; \
+ do_it = lucy_Debug_debug_should_print(__FILE__, __func__); \
+ } \
+ if (do_it) { actions; } \
+ } while (0)
+
+/** Execute code so long as LUCY_DEBUG was defined during compilation.
+ */
+#define LUCY_IFDEF_DEBUG(actions) do { actions; } while (0)
+
+/** Conditionally print debug statement depending on LUCY_DEBUG env variable.
+ */
+#define LUCY_DEBUG(args...) \
+ LUCY_DEBUG_DO(LUCY_DEBUG_PRINT(args));
+
+/** Abort on error if test fails.
+ *
+ * Note: unlike the system assert(), this ASSERT() is #ifdef LUCY_DEBUG.
+ */
+#define LUCY_ASSERT(test , args...) \
+ do { \
+ if (!(test)) { \
+ LUCY_DEBUG_PRINT("ASSERT FAILED (" #test ")\n" args); \
+ abort(); \
+ } \
+ } while (0)
+
+#elif defined(CHY_HAS_GNUC_VARIADIC_MACROS) // not LUCY_DEBUG
+
+#undef LUCY_DEBUG
+#define LUCY_DEBUG_ENABLED 0
+#define LUCY_DEBUG_DO(actions)
+#define LUCY_IFDEF_DEBUG(actions)
+#define LUCY_DEBUG_PRINT(args...)
+#define LUCY_DEBUG(args...)
+#define LUCY_ASSERT(test, args...)
+
+#else // also not LUCY_DEBUG
+
+#undef LUCY_DEBUG
+#define LUCY_DEBUG_ENABLED 0
+#define LUCY_DEBUG_DO(actions)
+#define LUCY_IFDEF_DEBUG(actions)
+static void LUCY_DEBUG_PRINT(char *_ignore_me, ...) { }
+static void LUCY_DEBUG(char *_ignore_me, ...) { }
+static void LUCY_ASSERT(int _ignore_me, ...) { }
+
+#endif // LUCY_DEBUG
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define DEBUG_ENABLED LUCY_DEBUG_ENABLED
+ #define DEBUG_PRINT LUCY_DEBUG_PRINT
+ #define DEBUG_DO LUCY_DEBUG_DO
+ #define IFDEF_DEBUG LUCY_IFDEF_DEBUG
+ #define DEBUG LUCY_DEBUG
+ #define ASSERT LUCY_ASSERT
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Util/Freezer.c b/core/Lucy/Util/Freezer.c
new file mode 100644
index 0000000..9a109c3
--- /dev/null
+++ b/core/Lucy/Util/Freezer.c
@@ -0,0 +1,39 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FREEZER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/Freezer.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+
+void
+Freezer_freeze(Obj *obj, OutStream *outstream) {
+ CB_Serialize(Obj_Get_Class_Name(obj), outstream);
+ Obj_Serialize(obj, outstream);
+}
+
+Obj*
+Freezer_thaw(InStream *instream) {
+ CharBuf *class_name = CB_deserialize(NULL, instream);
+ VTable *vtable = VTable_singleton(class_name, NULL);
+ Obj *blank = VTable_Make_Obj(vtable);
+ DECREF(class_name);
+ return Obj_Deserialize(blank, instream);
+}
+
+
diff --git a/core/Lucy/Util/Freezer.cfh b/core/Lucy/Util/Freezer.cfh
new file mode 100644
index 0000000..f00a961
--- /dev/null
+++ b/core/Lucy/Util/Freezer.cfh
@@ -0,0 +1,45 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Util::Freezer {
+
+ /** Store an arbitrary object to the outstream.
+ */
+ inert void
+ freeze(Obj *obj, OutStream *outstream);
+
+ /** Retrieve an arbitrary object from the instream.
+ */
+ inert incremented Obj*
+ thaw(InStream *instream);
+}
+
+__C__
+#define LUCY_FREEZE(_obj, _outstream) \
+ lucy_Freezer_freeze((Obj*)(_obj), (outstream))
+
+#define LUCY_THAW(_instream) \
+ lucy_Freezer_thaw(instream)
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define FREEZE LUCY_FREEZE
+ #define THAW LUCY_THAW
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Util/IndexFileNames.c b/core/Lucy/Util/IndexFileNames.c
new file mode 100644
index 0000000..7969f3e
--- /dev/null
+++ b/core/Lucy/Util/IndexFileNames.c
@@ -0,0 +1,94 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INDEXFILENAMES
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/IndexFileNames.h"
+#include "Lucy/Store/DirHandle.h"
+#include "Lucy/Store/Folder.h"
+#include "Lucy/Util/StringHelper.h"
+
+CharBuf*
+IxFileNames_latest_snapshot(Folder *folder) {
+ DirHandle *dh = Folder_Open_Dir(folder, NULL);
+ CharBuf *entry = dh ? DH_Get_Entry(dh) : NULL;
+ CharBuf *retval = NULL;
+ uint64_t latest_gen = 0;
+
+ if (!dh) { RETHROW(INCREF(Err_get_error())); }
+
+ while (DH_Next(dh)) {
+ if (CB_Starts_With_Str(entry, "snapshot_", 9)
+ && CB_Ends_With_Str(entry, ".json", 5)
+ ) {
+ uint64_t gen = IxFileNames_extract_gen(entry);
+ if (gen > latest_gen) {
+ latest_gen = gen;
+ if (!retval) { retval = CB_Clone(entry); }
+ else { CB_Mimic(retval, (Obj*)entry); }
+ }
+ }
+ }
+
+ DECREF(dh);
+ return retval;
+}
+
+uint64_t
+IxFileNames_extract_gen(const CharBuf *name) {
+ ZombieCharBuf *num_string = ZCB_WRAP(name);
+
+ // Advance past first underscore. Bail if we run out of string or if we
+ // encounter a NULL.
+ while (1) {
+ uint32_t code_point = ZCB_Nip_One(num_string);
+ if (code_point == 0) { return 0; }
+ else if (code_point == '_') { break; }
+ }
+
+ return (uint64_t)ZCB_BaseX_To_I64(num_string, 36);
+}
+
+ZombieCharBuf*
+IxFileNames_local_part(const CharBuf *path, ZombieCharBuf *target) {
+ ZombieCharBuf *scratch = ZCB_WRAP(path);
+ size_t local_part_start = CB_Length(path);
+ uint32_t code_point;
+
+ ZCB_Assign(target, path);
+
+ // Trim trailing slash.
+ while (ZCB_Code_Point_From(target, 1) == '/') {
+ ZCB_Chop(target, 1);
+ ZCB_Chop(scratch, 1);
+ local_part_start--;
+ }
+
+ // Substring should start after last slash.
+ while (0 != (code_point = ZCB_Code_Point_From(scratch, 1))) {
+ if (code_point == '/') {
+ ZCB_Nip(target, local_part_start);
+ break;
+ }
+ ZCB_Chop(scratch, 1);
+ local_part_start--;
+ }
+
+ return target;
+}
+
+
diff --git a/core/Lucy/Util/IndexFileNames.cfh b/core/Lucy/Util/IndexFileNames.cfh
new file mode 100644
index 0000000..2286f66
--- /dev/null
+++ b/core/Lucy/Util/IndexFileNames.cfh
@@ -0,0 +1,49 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Utilities for parsing, interpreting and generating index file names.
+ */
+inert class Lucy::Util::IndexFileNames cnick IxFileNames {
+
+ /** Skip past the first instance of an underscore in the CharBuf, then
+ * attempt to decode a base 36 number. For example, "snapshot_5.json"
+ * yields 5, and "seg_a1" yields 27.
+ *
+ * @return a generation number, or 0 if no number can be extracted.
+ */
+ inert uint64_t
+ extract_gen(const CharBuf *name);
+
+ /** Return the name of the latest generation snapshot file in the Folder,
+ * or NULL if no such file exists.
+ */
+ inert incremented nullable CharBuf*
+ latest_snapshot(Folder *folder);
+
+ /** Split the <code>path</code> on '/' and assign the last component to
+ * <code>target</code>, which will remain valid only as long as
+ * <code>path</code> is unmodified. Trailing slashes will be stripped.
+ *
+ * @param target The target string to assign to.
+ * @return target, allowing an assignment idiom.
+ */
+ inert incremented ZombieCharBuf*
+ local_part(const CharBuf *path, ZombieCharBuf *target);
+}
+
+
diff --git a/core/Lucy/Util/Json.cfh b/core/Lucy/Util/Json.cfh
new file mode 100644
index 0000000..a3fc298
--- /dev/null
+++ b/core/Lucy/Util/Json.cfh
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Encode/decode JSON.
+ *
+ * Provides utility functions for encoding/decoding JSON.
+ */
+class Lucy::Util::Json inherits Lucy::Object::Obj {
+
+ /** Encode <code>dump</code> as JSON.
+ */
+ inert incremented CharBuf*
+ to_json(Obj *dump);
+
+ /** Decode the supplied JSON and return a data structure made
+ * of Hashes, VArrays, and CharBufs.
+ */
+ inert incremented Obj*
+ from_json(CharBuf *json);
+
+ /** Encode <code>dump</code> as JSON and attempt to write to the indicated
+ * file.
+ * @return true if the write succeeds, false on failure (sets Err_error).
+ */
+ inert bool_t
+ spew_json(Obj *dump, Folder *folder, const CharBuf *path);
+
+ /** Decode the JSON in the file at <code>path</code> and return a data
+ * structure made of Hashes, VArrays, and CharBufs. Returns NULL and sets
+ * Err_error if the file can't be can't be opened or if the file doesn't
+ * contain valid JSON.
+ */
+ inert incremented nullable Obj*
+ slurp_json(Folder *folder, const CharBuf *path);
+
+ /** Allow the encoder to output strings, etc, instead of throwing an error
+ * on anything other than a hash or an array. Testing only.
+ */
+ inert void
+ set_tolerant(bool_t tolerant);
+}
+
+
diff --git a/core/Lucy/Util/Memory.c b/core/Lucy/Util/Memory.c
new file mode 100644
index 0000000..b859cde
--- /dev/null
+++ b/core/Lucy/Util/Memory.c
@@ -0,0 +1,112 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MEMORY
+#include <stdlib.h>
+#include <stdio.h>
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+#include "Lucy/Util/Memory.h"
+
+void*
+Memory_wrapped_malloc(size_t count) {
+ void *pointer = malloc(count);
+ if (pointer == NULL && count != 0) {
+ fprintf(stderr, "Can't malloc %" U64P " bytes.\n", (uint64_t)count);
+ exit(1);
+ }
+ return pointer;
+}
+
+void*
+Memory_wrapped_calloc(size_t count, size_t size) {
+ void *pointer = calloc(count, size);
+ if (pointer == NULL && count != 0) {
+ fprintf(stderr, "Can't calloc %" U64P " elements of size %" U64P ".\n",
+ (uint64_t)count, (uint64_t)size);
+ exit(1);
+ }
+ return pointer;
+}
+
+void*
+Memory_wrapped_realloc(void *ptr, size_t size) {
+ void *pointer = realloc(ptr, size);
+ if (pointer == NULL && size != 0) {
+ fprintf(stderr, "Can't realloc %" U64P " bytes.\n", (uint64_t)size);
+ exit(1);
+ }
+ return pointer;
+}
+
+void
+Memory_wrapped_free(void *ptr) {
+ free(ptr);
+}
+
+#ifndef SIZE_MAX
+#define SIZE_MAX ((size_t)-1)
+#endif
+
+size_t
+Memory_oversize(size_t minimum, size_t width) {
+ // For larger arrays, grow by an excess of 1/8; grow faster when the array
+ // is small.
+ size_t extra = minimum / 8;
+ if (extra < 3) {
+ extra = 3;
+ }
+ size_t amount = minimum + extra;
+
+ // Detect wraparound and return SIZE_MAX instead.
+ if (amount + 7 < minimum) {
+ return SIZE_MAX;
+ }
+
+ // Round up for small widths so that the number of bytes requested will be
+ // a multiple of the machine's word size.
+ if (sizeof(size_t) == 8) { // 64-bit
+ switch (width) {
+ case 1:
+ amount = (amount + 7) & CHY_I64_C(0xFFFFFFFFFFFFFFF8);
+ break;
+ case 2:
+ amount = (amount + 3) & CHY_I64_C(0xFFFFFFFFFFFFFFFC);
+ break;
+ case 4:
+ amount = (amount + 1) & CHY_I64_C(0xFFFFFFFFFFFFFFFE);
+ break;
+ default:
+ break;
+ }
+ }
+ else { // 32-bit
+ switch (width) {
+ case 1:
+ amount = (amount + 3) & ((size_t)0xFFFFFFFC);
+ break;
+ case 2:
+ amount = (amount + 1) & ((size_t)0xFFFFFFFE);
+ break;
+ default:
+ break;
+ }
+ }
+
+ return amount;
+}
+
+
diff --git a/core/Lucy/Util/Memory.cfh b/core/Lucy/Util/Memory.cfh
new file mode 100644
index 0000000..ab0fbc9
--- /dev/null
+++ b/core/Lucy/Util/Memory.cfh
@@ -0,0 +1,73 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Util::Memory {
+
+ /** Attempt to allocate memory with malloc, but print an error and exit if the
+ * call fails.
+ */
+ inert nullable void*
+ wrapped_malloc(size_t count);
+
+ /** Attempt to allocate memory with calloc, but print an error and exit if the
+ * call fails.
+ */
+ inert nullable void*
+ wrapped_calloc(size_t count, size_t size);
+
+ /** Attempt to allocate memory with realloc, but print an error and exit if
+ * the call fails.
+ */
+ inert nullable void*
+ wrapped_realloc(void *ptr, size_t size);
+
+ /** Free memory. (Wrapping is necessary in cases where memory allocated
+ * within the Lucy library has to be freed in an external environment where
+ * "free" may have been redefined.)
+ */
+ inert void
+ wrapped_free(void *ptr);
+
+ /** Provide a number which is somewhat larger than the supplied number, so
+ * that incremental array growth does not trigger pathological
+ * reallocation.
+ *
+ * @param minimum The minimum number of array elements.
+ * @param width The size of each array element in bytes.
+ */
+ inert size_t
+ oversize(size_t minimum, size_t width);
+}
+
+__C__
+
+#define LUCY_MALLOCATE lucy_Memory_wrapped_malloc
+#define LUCY_CALLOCATE lucy_Memory_wrapped_calloc
+#define LUCY_REALLOCATE lucy_Memory_wrapped_realloc
+#define LUCY_FREEMEM lucy_Memory_wrapped_free
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define MALLOCATE LUCY_MALLOCATE
+ #define CALLOCATE LUCY_CALLOCATE
+ #define REALLOCATE LUCY_REALLOCATE
+ #define FREEMEM LUCY_FREEMEM
+#endif
+
+__END_C__
+
+
diff --git a/core/Lucy/Util/MemoryPool.c b/core/Lucy/Util/MemoryPool.c
new file mode 100644
index 0000000..1d44591
--- /dev/null
+++ b/core/Lucy/Util/MemoryPool.c
@@ -0,0 +1,171 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MEMORYPOOL
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/MemoryPool.h"
+
+static void
+S_init_arena(MemoryPool *self, size_t amount);
+
+#define DEFAULT_BUF_SIZE 0x100000 // 1 MiB
+
+// Enlarge amount so pointers will always be aligned.
+#define INCREASE_TO_WORD_MULTIPLE(_amount) \
+ do { \
+ const size_t _remainder = _amount % sizeof(void*); \
+ if (_remainder) { \
+ _amount += sizeof(void*); \
+ _amount -= _remainder; \
+ } \
+ } while (0)
+
+MemoryPool*
+MemPool_new(uint32_t arena_size) {
+ MemoryPool *self = (MemoryPool*)VTable_Make_Obj(MEMORYPOOL);
+ return MemPool_init(self, arena_size);
+}
+
+MemoryPool*
+MemPool_init(MemoryPool *self, uint32_t arena_size) {
+ self->arena_size = arena_size == 0 ? DEFAULT_BUF_SIZE : arena_size;
+ self->arenas = VA_new(16);
+ self->tick = -1;
+ self->buf = NULL;
+ self->limit = NULL;
+ self->consumed = 0;
+
+ return self;
+}
+
+void
+MemPool_destroy(MemoryPool *self) {
+ DECREF(self->arenas);
+ SUPER_DESTROY(self, MEMORYPOOL);
+}
+
+static void
+S_init_arena(MemoryPool *self, size_t amount) {
+ ByteBuf *bb;
+ int32_t i;
+
+ // Indicate which arena we're using at present.
+ self->tick++;
+
+ if (self->tick < (int32_t)VA_Get_Size(self->arenas)) {
+ // In recycle mode, use previously acquired memory.
+ bb = (ByteBuf*)VA_Fetch(self->arenas, self->tick);
+ if (amount >= BB_Get_Size(bb)) {
+ BB_Grow(bb, amount);
+ BB_Set_Size(bb, amount);
+ }
+ }
+ else {
+ // In add mode, get more mem from system.
+ size_t buf_size = (amount + 1) > self->arena_size
+ ? (amount + 1)
+ : self->arena_size;
+ char *ptr = (char*)MALLOCATE(buf_size);
+ bb = BB_new_steal_bytes(ptr, buf_size - 1, buf_size);
+ VA_Push(self->arenas, (Obj*)bb);
+ }
+
+ // Recalculate consumption to take into account blocked off space.
+ self->consumed = 0;
+ for (i = 0; i < self->tick; i++) {
+ ByteBuf *bb = (ByteBuf*)VA_Fetch(self->arenas, i);
+ self->consumed += BB_Get_Size(bb);
+ }
+
+ self->buf = BB_Get_Buf(bb);
+ self->limit = self->buf + BB_Get_Size(bb);
+}
+
+size_t
+MemPool_get_consumed(MemoryPool *self) {
+ return self->consumed;
+}
+
+void*
+MemPool_grab(MemoryPool *self, size_t amount) {
+ INCREASE_TO_WORD_MULTIPLE(amount);
+ self->last_buf = self->buf;
+
+ // Verify that we have enough stocked up, otherwise get more.
+ self->buf += amount;
+ if (self->buf >= self->limit) {
+ // Get enough mem from system or die trying.
+ S_init_arena(self, amount);
+ self->last_buf = self->buf;
+ self->buf += amount;
+ }
+
+ // Track bytes we've allocated from this pool.
+ self->consumed += amount;
+
+ return self->last_buf;
+}
+
+void
+MemPool_resize(MemoryPool *self, void *ptr, size_t new_amount) {
+ const size_t last_amount = self->buf - self->last_buf;
+ INCREASE_TO_WORD_MULTIPLE(new_amount);
+
+ if (ptr != self->last_buf) {
+ THROW(ERR, "Not the last pointer allocated.");
+ }
+ else {
+ if (new_amount <= last_amount) {
+ const size_t difference = last_amount - new_amount;
+ self->buf -= difference;
+ self->consumed -= difference;
+ }
+ else {
+ THROW(ERR, "Can't resize to greater amount: %u64 > %u64",
+ (uint64_t)new_amount, (uint64_t)last_amount);
+ }
+ }
+}
+
+void
+MemPool_release_all(MemoryPool *self) {
+ self->tick = -1;
+ self->buf = NULL;
+ self->last_buf = NULL;
+ self->limit = NULL;
+}
+
+void
+MemPool_eat(MemoryPool *self, MemoryPool *other) {
+ int32_t i;
+ if (self->buf != NULL) {
+ THROW(ERR, "Memory pool is not empty");
+ }
+
+ // Move active arenas from other to self.
+ for (i = 0; i <= other->tick; i++) {
+ ByteBuf *arena = (ByteBuf*)VA_Shift(other->arenas);
+ // Maybe displace existing arena.
+ VA_Store(self->arenas, i, (Obj*)arena);
+ }
+ self->tick = other->tick;
+ self->last_buf = other->last_buf;
+ self->buf = other->buf;
+ self->limit = other->limit;
+}
+
+
diff --git a/core/Lucy/Util/MemoryPool.cfh b/core/Lucy/Util/MemoryPool.cfh
new file mode 100644
index 0000000..61934d1
--- /dev/null
+++ b/core/Lucy/Util/MemoryPool.cfh
@@ -0,0 +1,80 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Specialized memory allocator.
+ *
+ * Grab memory from the system in 1 MB chunks. Don't release it until object
+ * destruction. Parcel the memory out on request.
+ *
+ * The release mechanism is fast but extremely crude, limiting the use of this
+ * class to specific applications.
+ */
+
+class Lucy::Util::MemoryPool cnick MemPool
+ inherits Lucy::Object::Obj {
+
+ uint32_t arena_size;
+ VArray *arenas;
+ int32_t tick;
+ char *buf;
+ char *last_buf;
+ char *limit;
+ size_t consumed; /* bytes allocated (not cap) */
+
+ /**
+ * @param arena_size The size of each internally allocated memory slab.
+ * If 0, it will be set to 1 MiB.
+ */
+ inert incremented MemoryPool*
+ new(uint32_t arena_size);
+
+ inert MemoryPool*
+ init(MemoryPool *self, uint32_t arena_size);
+
+ /** Allocate memory from the pool.
+ */
+ void*
+ Grab(MemoryPool *self, size_t amount);
+
+ /** Resize the last allocation. (*Only* the last allocation).
+ */
+ void
+ Resize(MemoryPool *self, void *ptr, size_t revised_amount);
+
+ /** Tell the pool to consider all previous allocations released.
+ */
+ void
+ Release_All(MemoryPool *self);
+
+ /** Take ownership of all the arenas in another MemoryPool. Can only be
+ * called when the original memory pool has no outstanding allocations,
+ * typically just after a call to Release_All. The purpose is to support
+ * bulk reallocation.
+ */
+ void
+ Eat(MemoryPool *self, MemoryPool *other);
+
+ size_t
+ Get_Consumed(MemoryPool *self);
+
+ public void
+ Destroy(MemoryPool *self);
+}
+
+
diff --git a/core/Lucy/Util/NumberUtils.c b/core/Lucy/Util/NumberUtils.c
new file mode 100644
index 0000000..88c21df
--- /dev/null
+++ b/core/Lucy/Util/NumberUtils.c
@@ -0,0 +1,35 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_NUMBERUTILS
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <string.h>
+
+#include "Lucy/Util/NumberUtils.h"
+
+const uint8_t NumUtil_u1masks[8] = {
+ 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80
+};
+
+const uint8_t NumUtil_u2shifts[4] = { 0x0, 0x2, 0x4, 0x6 };
+const uint8_t NumUtil_u2masks[4] = { 0x3, 0xC, 0x30, 0xC0 };
+
+const uint8_t NumUtil_u4shifts[2] = { 0x00, 0x04 };
+const uint8_t NumUtil_u4masks[2] = { 0x0F, 0xF0 };
+
+
diff --git a/core/Lucy/Util/NumberUtils.cfh b/core/Lucy/Util/NumberUtils.cfh
new file mode 100644
index 0000000..b1da962
--- /dev/null
+++ b/core/Lucy/Util/NumberUtils.cfh
@@ -0,0 +1,472 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Provide various number-related utilies.
+ *
+ * Provide utilities for dealing with endian issues, sub-byte-width arrays,
+ * compressed integers, and so on.
+ */
+inert class Lucy::Util::NumberUtils cnick NumUtil {
+
+ inert const uint8_t[8] u1masks;
+ inert const uint8_t[4] u2masks;
+ inert const uint8_t[4] u2shifts;
+ inert const uint8_t[2] u4masks;
+ inert const uint8_t[2] u4shifts;
+
+ /** Encode an unsigned 16-bit integer as 2 bytes in the buffer provided,
+ * using big-endian byte order.
+ */
+ inert inline void
+ encode_bigend_u16(uint16_t value, void *dest);
+
+ /** Encode an unsigned 32-bit integer as 4 bytes in the buffer provided,
+ * using big-endian byte order.
+ */
+ inert inline void
+ encode_bigend_u32(uint32_t value, void *dest);
+
+ /** Encode an unsigned 64-bit integer as 8 bytes in the buffer provided,
+ * using big-endian byte order.
+ */
+ inert inline void
+ encode_bigend_u64(uint64_t value, void *dest);
+
+ /** Interpret a sequence of bytes as a big-endian unsigned 16-bit int.
+ */
+ inert inline uint16_t
+ decode_bigend_u16(void *source);
+
+ /** Interpret a sequence of bytes as a big-endian unsigned 32-bit int.
+ */
+ inert inline uint32_t
+ decode_bigend_u32(void *source);
+
+ /** Interpret a sequence of bytes as a big-endian unsigned 64-bit int.
+ */
+ inert inline uint64_t
+ decode_bigend_u64(void *source);
+
+ /** Encode a 32-bit floating point number as 4 bytes in the buffer
+ * provided, using big-endian byte order.
+ */
+ inert inline void
+ encode_bigend_f32(float value, void *dest);
+
+ /** Encode a 64-bit floating point number as 8 bytes in the buffer
+ * provided, using big-endian byte order.
+ */
+ inert inline void
+ encode_bigend_f64(double value, void *dest);
+
+ /** Interpret a sequence of bytes as a 32-bit float stored in big-endian
+ * byte order.
+ */
+ inert inline float
+ decode_bigend_f32(void *source);
+
+ /** Interpret a sequence of bytes as a 64-bit float stored in big-endian
+ * byte order.
+ */
+ inert inline double
+ decode_bigend_f64(void *source);
+
+ /** Encode a C32 at the space pointed to by <code>dest</code>. As a side
+ * effect, <code>dest</code> will be advanced to immediately after the end
+ * of the C32.
+ */
+ inert inline void
+ encode_c32(uint32_t value, char **dest);
+
+ /** Encode a C32 at the space pointed to by <code>dest</code>, but add
+ * "leading zeroes" so that the space consumed will always be 5 bytes. As
+ * a side effect, <code>dest</code> will be advanced to immediately after
+ * the end of the C32.
+ */
+ inert inline void
+ encode_padded_c32(uint32_t value, char **dest);
+
+ /** Encode a C64 at the space pointed to by <code>dest</code>. As a side
+ * effect, <code>dest</code> will be advanced to immediately after the end
+ * of the C64.
+ */
+ inert inline void
+ encode_c64(uint64_t value, char **dest);
+
+ /** Read a C32 from the buffer pointed to by <code>source</code>. As a
+ * side effect, advance the pointer, consuming the bytes occupied by the
+ * C32.
+ */
+ inert inline uint32_t
+ decode_c32(char **source);
+
+ /** Read a C64 from the buffer pointed to by <code>source</code>. As a
+ * side effect, advance the pointer, consuming the bytes occupied by the
+ * C64.
+ */
+ inert inline uint64_t
+ decode_c64(char **source);
+
+ /** Advance <code>source</code> past one encoded C32 or C64.
+ */
+ inert inline void
+ skip_cint(char **source);
+
+ /** Interpret <code>array</code> as an array of bits; return true if the
+ * bit at <code>tick</code> is set, false otherwise.
+ */
+ inert inline bool_t
+ u1get(void *array, uint32_t tick);
+
+ /** Interpret <code>array</code> as an array of bits; set the bit at
+ * <code>tick</code>.
+ */
+ inert inline void
+ u1set(void *array, uint32_t tick);
+
+ /** Interpret <code>array</code> as an array of bits; clear the bit at
+ * <code>tick</code>.
+ */
+ inert inline void
+ u1clear(void *array, uint32_t tick);
+
+ /** Interpret <code>array</code> as an array of bits; flip the bit at
+ * <code>tick</code>.
+ */
+ inert inline void
+ u1flip(void *array, uint32_t tick);
+
+ /** Interpret <code>array</code> as an array of two-bit integers; return
+ * the value at <code>tick</code>.
+ */
+ inert inline uint8_t
+ u2get(void *array, uint32_t tick);
+
+ /** Interpret <code>array</code> as an array of two-bit integers; set the
+ * element at <code>tick</code> to <code>value</code>.
+ */
+ inert inline void
+ u2set(void *array, uint32_t tick, uint8_t value);
+
+ /** Interpret <code>array</code> as an array of four-bit integers; return
+ * the value at <code>tick</code>.
+ */
+ inert inline uint8_t
+ u4get(void *array, uint32_t tick);
+
+ /** Interpret <code>array</code> as an array of four-bit integers; set the
+ * element at <code>tick</code> to <code>value</code>.
+ */
+ inert inline void
+ u4set(void *array, uint32_t tick, uint8_t value);
+}
+
+__C__
+
+static CHY_INLINE void
+lucy_NumUtil_encode_bigend_u16(uint16_t value, void *dest_ptr) {
+ uint8_t *dest = *(uint8_t**)dest_ptr;
+#ifdef CHY_BIG_END
+ memcpy(dest, &value, sizeof(uint16_t));
+#else // little endian
+ uint8_t *source = (uint8_t*)&value;
+ dest[0] = source[1];
+ dest[1] = source[0];
+#endif // CHY_BIG_END (and little endian)
+}
+
+static CHY_INLINE void
+lucy_NumUtil_encode_bigend_u32(uint32_t value, void *dest_ptr) {
+ uint8_t *dest = *(uint8_t**)dest_ptr;
+#ifdef CHY_BIG_END
+ memcpy(dest, &value, sizeof(uint32_t));
+#else // little endian
+ uint8_t *source = (uint8_t*)&value;
+ dest[0] = source[3];
+ dest[1] = source[2];
+ dest[2] = source[1];
+ dest[3] = source[0];
+#endif // CHY_BIG_END (and little endian)
+}
+
+static CHY_INLINE void
+lucy_NumUtil_encode_bigend_u64(uint64_t value, void *dest_ptr) {
+ uint8_t *dest = *(uint8_t**)dest_ptr;
+#ifdef CHY_BIG_END
+ memcpy(dest, &value, sizeof(uint64_t));
+#else // little endian
+ uint8_t *source = (uint8_t*)&value;
+ dest[0] = source[7];
+ dest[1] = source[6];
+ dest[2] = source[5];
+ dest[3] = source[4];
+ dest[4] = source[3];
+ dest[5] = source[2];
+ dest[6] = source[1];
+ dest[7] = source[0];
+#endif // CHY_BIG_END (and little endian)
+}
+
+static CHY_INLINE uint16_t
+lucy_NumUtil_decode_bigend_u16(void *source) {
+ uint8_t *const buf = (uint8_t*)source;
+ return (buf[0] << 8) |
+ (buf[1]);
+}
+
+static CHY_INLINE uint32_t
+lucy_NumUtil_decode_bigend_u32(void *source) {
+ uint8_t *const buf = (uint8_t*)source;
+ return (buf[0] << 24) |
+ (buf[1] << 16) |
+ (buf[2] << 8) |
+ (buf[3]);
+}
+
+static CHY_INLINE uint64_t
+lucy_NumUtil_decode_bigend_u64(void *source) {
+ uint8_t *const buf = (uint8_t*)source;
+ uint64_t high_bits = (buf[0] << 24) |
+ (buf[1] << 16) |
+ (buf[2] << 8) |
+ (buf[3]);
+ uint32_t low_bits = (buf[4] << 24) |
+ (buf[5] << 16) |
+ (buf[6] << 8) |
+ (buf[7]);
+ uint64_t retval = high_bits << 32;
+ retval |= low_bits;
+ return retval;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_encode_bigend_f32(float value, void *dest_ptr) {
+ uint8_t *dest = *(uint8_t**)dest_ptr;
+#ifdef CHY_BIG_END
+ memcpy(dest, &value, sizeof(float));
+#else
+ union { float f; uint32_t u32; } duo;
+ duo.f = value;
+ lucy_NumUtil_encode_bigend_u32(duo.u32, &dest);
+#endif
+}
+
+static CHY_INLINE void
+lucy_NumUtil_encode_bigend_f64(double value, void *dest_ptr) {
+ uint8_t *dest = *(uint8_t**)dest_ptr;
+#ifdef CHY_BIG_END
+ memcpy(dest, &value, sizeof(double));
+#else
+ union { double d; uint64_t u64; } duo;
+ duo.d = value;
+ lucy_NumUtil_encode_bigend_u64(duo.u64, &dest);
+#endif
+}
+
+static CHY_INLINE float
+lucy_NumUtil_decode_bigend_f32(void *source) {
+ union { float f; uint32_t u32; } duo;
+ memcpy(&duo, source, sizeof(float));
+#ifdef CHY_LITTLE_END
+ duo.u32 = lucy_NumUtil_decode_bigend_u32(&duo.u32);
+#endif
+ return duo.f;
+}
+
+static CHY_INLINE double
+lucy_NumUtil_decode_bigend_f64(void *source) {
+ union { double d; uint64_t u64; } duo;
+ memcpy(&duo, source, sizeof(double));
+#ifdef CHY_LITTLE_END
+ duo.u64 = lucy_NumUtil_decode_bigend_u64(&duo.u64);
+#endif
+ return duo.d;
+}
+
+#define LUCY_NUMUTIL_C32_MAX_BYTES ((sizeof(uint32_t) * 8 / 7) + 1) // 5
+#define LUCY_NUMUTIL_C64_MAX_BYTES ((sizeof(uint64_t) * 8 / 7) + 1) // 10
+
+static CHY_INLINE void
+lucy_NumUtil_encode_c32(uint32_t value, char **out_buf) {
+ uint8_t buf[LUCY_NUMUTIL_C32_MAX_BYTES];
+ uint8_t *const limit = buf + sizeof(buf);
+ uint8_t *ptr = limit - 1;
+ int num_bytes;
+ // Write last byte first, which has no continue bit.
+ *ptr = value & 0x7f;
+ value >>= 7;
+ while (value) {
+ // Work backwards, writing bytes with continue bits set.
+ *--ptr = ((value & 0x7f) | 0x80);
+ value >>= 7;
+ }
+ num_bytes = limit - ptr;
+ memcpy(*out_buf, ptr, num_bytes);
+ *out_buf += num_bytes;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_encode_c64(uint64_t value, char **out_buf) {
+ uint8_t buf[LUCY_NUMUTIL_C64_MAX_BYTES];
+ uint8_t *const limit = buf + sizeof(buf);
+ uint8_t *ptr = limit - 1;
+ int num_bytes;
+ // Write last byte first, which has no continue bit.
+ *ptr = value & 0x7f;
+ value >>= 7;
+ while (value) {
+ // Work backwards, writing bytes with continue bits set.
+ *--ptr = ((value & 0x7f) | 0x80);
+ value >>= 7;
+ }
+ num_bytes = limit - ptr;
+ memcpy(*out_buf, ptr, num_bytes);
+ *out_buf += num_bytes;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_encode_padded_c32(uint32_t value, char **out_buf) {
+ uint8_t buf[LUCY_NUMUTIL_C32_MAX_BYTES]
+ = { 0x80, 0x80, 0x80, 0x80, 0x80 };
+ uint8_t *const limit = buf + sizeof(buf);
+ uint8_t *ptr = limit - 1;
+ // Write last byte first, which has no continue bit.
+ *ptr = value & 0x7f;
+ value >>= 7;
+ while (value) {
+ // Work backwards, writing bytes with continue bits set.
+ *--ptr = ((value & 0x7f) | 0x80);
+ value >>= 7;
+ }
+ memcpy(*out_buf, buf, LUCY_NUMUTIL_C32_MAX_BYTES);
+ *out_buf += sizeof(buf);
+}
+
+// Decode a compressed integer up to size of 'var', advancing 'source'
+#define LUCY_NUMUTIL_DECODE_CINT(var, source) \
+ do { \
+ var = (*source & 0x7f); \
+ while (*source++ & 0x80) { \
+ var = (*source & 0x7f) | (var << 7); \
+ } \
+ } while (0)
+
+static CHY_INLINE uint32_t
+lucy_NumUtil_decode_c32(char **source_ptr) {
+ char *source = *source_ptr;
+ uint32_t decoded;
+ LUCY_NUMUTIL_DECODE_CINT(decoded, source);
+ *source_ptr = source;
+ return decoded;
+}
+
+static CHY_INLINE uint64_t
+lucy_NumUtil_decode_c64(char **source_ptr) {
+ char *source = *source_ptr;
+ uint64_t decoded;
+ LUCY_NUMUTIL_DECODE_CINT(decoded, source);
+ *source_ptr = source;
+ return decoded;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_skip_cint(char **source_ptr) {
+ uint8_t *ptr = *(uint8_t**)source_ptr;
+ while ((*ptr++ & 0x80) != 0) { }
+ *source_ptr = (char*)ptr;
+}
+
+static CHY_INLINE chy_bool_t
+lucy_NumUtil_u1get(void *array, uint32_t tick) {
+ uint8_t *const u8bits = (uint8_t*)array;
+ const uint32_t byte_offset = tick >> 3;
+ const uint8_t mask = lucy_NumUtil_u1masks[tick & 0x7];
+ return !((u8bits[byte_offset] & mask) == 0);
+}
+
+static CHY_INLINE void
+lucy_NumUtil_u1set(void *array, uint32_t tick) {
+ uint8_t *const u8bits = (uint8_t*)array;
+ const uint32_t byte_offset = tick >> 3;
+ const uint8_t mask = lucy_NumUtil_u1masks[tick & 0x7];
+ u8bits[byte_offset] |= mask;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_u1clear(void *array, uint32_t tick) {
+ uint8_t *const u8bits = (uint8_t*)array;
+ const uint32_t byte_offset = tick >> 3;
+ const uint8_t mask = lucy_NumUtil_u1masks[tick & 0x7];
+ u8bits[byte_offset] &= ~mask;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_u1flip(void *array, uint32_t tick) {
+ uint8_t *const u8bits = (uint8_t*)array;
+ const uint32_t byte_offset = tick >> 3;
+ const uint8_t mask = lucy_NumUtil_u1masks[tick & 0x7];
+ u8bits[byte_offset] ^= mask;
+}
+
+static CHY_INLINE uint8_t
+lucy_NumUtil_u2get(void *array, uint32_t tick) {
+ uint8_t *ints = (uint8_t*)array;
+ uint8_t byte = ints[(tick >> 2)];
+ int shift = lucy_NumUtil_u2shifts[tick & 0x3];
+ return (byte >> shift) & 0x3;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_u2set(void *array, uint32_t tick, uint8_t value) {
+ uint8_t *ints = (uint8_t*)array;
+ unsigned sub_tick = tick & 0x3;
+ int shift = lucy_NumUtil_u2shifts[sub_tick];
+ uint8_t mask = lucy_NumUtil_u2masks[sub_tick];
+ uint8_t new_val = value & 0x3;
+ uint8_t new_bits = new_val << shift;
+ ints[(tick >> 2)] = (ints[(tick >> 2)] & ~mask) | new_bits;
+}
+
+
+static CHY_INLINE uint8_t
+lucy_NumUtil_u4get(void *array, uint32_t tick) {
+ uint8_t *ints = (uint8_t*)array;
+ uint8_t byte = ints[(tick >> 1)];
+ int shift = lucy_NumUtil_u4shifts[(tick & 1)];
+ return (byte >> shift) & 0xF;
+}
+
+static CHY_INLINE void
+lucy_NumUtil_u4set(void *array, uint32_t tick, uint8_t value) {
+ uint8_t *ints = (uint8_t*)array;
+ unsigned sub_tick = tick & 0x1;
+ int shift = lucy_NumUtil_u4shifts[sub_tick];
+ uint8_t mask = lucy_NumUtil_u4masks[sub_tick];
+ uint8_t new_val = value & 0xF;
+ uint8_t new_bits = new_val << shift;
+ ints[(tick >> 1)] = (ints[(tick >> 1)] & ~mask) | new_bits;
+}
+
+#ifdef LUCY_USE_SHORT_NAMES
+ #define C32_MAX_BYTES LUCY_NUMUTIL_C32_MAX_BYTES
+ #define C64_MAX_BYTES LUCY_NUMUTIL_C64_MAX_BYTES
+#endif
+
+__END_C__
+
+
diff --git a/core/Lucy/Util/PriorityQueue.c b/core/Lucy/Util/PriorityQueue.c
new file mode 100644
index 0000000..a1a53df
--- /dev/null
+++ b/core/Lucy/Util/PriorityQueue.c
@@ -0,0 +1,231 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_PRIORITYQUEUE
+#include "Lucy/Util/ToolSet.h"
+
+#include <string.h>
+
+#include "Lucy/Util/PriorityQueue.h"
+
+// Add an element to the heap. Throw an error if too many elements
+// are added.
+static void
+S_put(PriorityQueue *self, Obj *element);
+
+// Free all the elements in the heap and set size to 0.
+static void
+S_clear(PriorityQueue *self);
+
+// Heap adjuster.
+static void
+S_up_heap(PriorityQueue *self);
+
+// Heap adjuster. Should be called when the item at the top changes.
+static void
+S_down_heap(PriorityQueue *self);
+
+PriorityQueue*
+PriQ_init(PriorityQueue *self, uint32_t max_size) {
+ if (max_size == U32_MAX) {
+ THROW(ERR, "max_size too large: %u32", max_size);
+ }
+ uint32_t heap_size = max_size + 1;
+
+ // Init.
+ self->size = 0;
+
+ // Assign.
+ self->max_size = max_size;
+
+ // Allocate space for the heap, assign all slots to NULL.
+ self->heap = (Obj**)CALLOCATE(heap_size, sizeof(Obj*));
+
+ ABSTRACT_CLASS_CHECK(self, PRIORITYQUEUE);
+ return self;
+}
+
+void
+PriQ_destroy(PriorityQueue *self) {
+ if (self->heap) {
+ S_clear(self);
+ FREEMEM(self->heap);
+ }
+ SUPER_DESTROY(self, PRIORITYQUEUE);
+}
+
+uint32_t
+PriQ_get_size(PriorityQueue *self) {
+ return self->size;
+}
+
+static void
+S_put(PriorityQueue *self, Obj *element) {
+ // Increment size.
+ if (self->size >= self->max_size) {
+ THROW(ERR, "PriorityQueue exceeded max_size: %u32 %u32", self->size,
+ self->max_size);
+ }
+ self->size++;
+
+ // Put element into heap.
+ self->heap[self->size] = element;
+
+ // Adjust heap.
+ S_up_heap(self);
+}
+
+bool_t
+PriQ_insert(PriorityQueue *self, Obj *element) {
+ Obj *least = PriQ_Jostle(self, element);
+ DECREF(least);
+ if (element == least) { return false; }
+ else { return true; }
+}
+
+Obj*
+PriQ_jostle(PriorityQueue *self, Obj *element) {
+ // Absorb element if there's a vacancy.
+ if (self->size < self->max_size) {
+ S_put(self, element);
+ return NULL;
+ }
+ // Otherwise, compete for the slot.
+ else if (self->size == 0) {
+ return element;
+ }
+ else {
+ Obj *scratch = PriQ_Peek(self);
+ if (!PriQ_Less_Than(self, element, scratch)) {
+ // If the new element belongs in the queue, replace something.
+ Obj *retval = self->heap[1];
+ self->heap[1] = element;
+ S_down_heap(self);
+ return retval;
+ }
+ else {
+ return element;
+ }
+ }
+}
+
+Obj*
+PriQ_pop(PriorityQueue *self) {
+ if (self->size > 0) {
+ // Save the first value.
+ Obj *result = self->heap[1];
+
+ // Move last to first and adjust heap.
+ self->heap[1] = self->heap[self->size];
+ self->heap[self->size] = NULL;
+ self->size--;
+ S_down_heap(self);
+
+ // Return the value, leaving a refcount for the caller.
+ return result;
+ }
+ else {
+ return NULL;
+ }
+}
+
+VArray*
+PriQ_pop_all(PriorityQueue *self) {
+ VArray *retval = VA_new(self->size);
+
+ // Map the queue nodes onto the array in reverse order.
+ if (self->size) {
+ uint32_t i;
+ for (i = self->size; i--;) {
+ Obj *const elem = PriQ_Pop(self);
+ VA_Store(retval, i, elem);
+ }
+ }
+
+ return retval;
+}
+
+Obj*
+PriQ_peek(PriorityQueue *self) {
+ if (self->size > 0) {
+ return self->heap[1];
+ }
+ else {
+ return NULL;
+ }
+}
+
+static void
+S_clear(PriorityQueue *self) {
+ uint32_t i;
+ Obj **elem_ptr = (self->heap + 1);
+
+ // Node 0 is held empty, to make the algo clearer.
+ for (i = 1; i <= self->size; i++) {
+ DECREF(*elem_ptr);
+ *elem_ptr = NULL;
+ elem_ptr++;
+ }
+ self->size = 0;
+}
+
+static void
+S_up_heap(PriorityQueue *self) {
+ uint32_t i = self->size;
+ uint32_t j = i >> 1;
+ Obj *const node = self->heap[i]; // save bottom node
+
+ while (j > 0
+ && PriQ_Less_Than(self, node, self->heap[j])
+ ) {
+ self->heap[i] = self->heap[j];
+ i = j;
+ j = j >> 1;
+ }
+ self->heap[i] = node;
+}
+
+static void
+S_down_heap(PriorityQueue *self) {
+ uint32_t i = 1;
+ uint32_t j = i << 1;
+ uint32_t k = j + 1;
+ Obj *node = self->heap[i]; // save top node
+
+ // Find smaller child.
+ if (k <= self->size
+ && PriQ_Less_Than(self, self->heap[k], self->heap[j])
+ ) {
+ j = k;
+ }
+
+ while (j <= self->size
+ && PriQ_Less_Than(self, self->heap[j], node)
+ ) {
+ self->heap[i] = self->heap[j];
+ i = j;
+ j = i << 1;
+ k = j + 1;
+ if (k <= self->size
+ && PriQ_Less_Than(self, self->heap[k], self->heap[j])
+ ) {
+ j = k;
+ }
+ }
+ self->heap[i] = node;
+}
+
+
diff --git a/core/Lucy/Util/PriorityQueue.cfh b/core/Lucy/Util/PriorityQueue.cfh
new file mode 100644
index 0000000..d41f88f
--- /dev/null
+++ b/core/Lucy/Util/PriorityQueue.cfh
@@ -0,0 +1,87 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Heap sort / priority queue.
+ *
+ * PriorityQueue implements a textbook heap sort / priority queue algorithm.
+ *
+ * Subclasses must define the abstract method Less_Than.
+ */
+
+class Lucy::Util::PriorityQueue cnick PriQ
+ inherits Lucy::Object::Obj {
+ uint32_t size;
+ uint32_t max_size;
+
+ /* This particular priority queue variant leaves slot 0 open in order to
+ * keep the relationship between node rank and index clear in the up_heap
+ * and down_heap routines.
+ */
+ Obj **heap;
+
+ /**
+ * @param max_size Max elements the queue can hold.
+ */
+ inert PriorityQueue*
+ init(PriorityQueue *self, uint32_t max_size);
+
+ /** Compare queue elements.
+ */
+ abstract bool_t
+ Less_Than(PriorityQueue *self, Obj *a, Obj *b);
+
+ /** Possibly insert an element. Add to the Queue if either...
+ * a) the queue isn't full, or
+ * b) the element belongs in the queue and should displace another.
+ */
+ bool_t
+ Insert(PriorityQueue *self, decremented Obj *element);
+
+ /** Equivalent to Insert(), except for the return value. If the Queue has
+ * room, the element is inserted and Jostle() returns NULL. If not, then
+ * the item which falls out of the bottom of the Queue is returned.
+ */
+ incremented nullable Obj*
+ Jostle(PriorityQueue *self, decremented Obj *element);
+
+ /** Pop the *least* item off of the priority queue.
+ */
+ incremented nullable Obj*
+ Pop(PriorityQueue *self);
+
+ /** Empty out the PriorityQueue into a sorted array.
+ */
+ incremented VArray*
+ Pop_All(PriorityQueue *self);
+
+ /** Return the least item in the queue, but don't remove it.
+ */
+ nullable Obj*
+ Peek(PriorityQueue *self);
+
+ /** Accessor for "size" member.
+ */
+ uint32_t
+ Get_Size(PriorityQueue *self);
+
+ public void
+ Destroy(PriorityQueue *self);
+}
+
+
diff --git a/core/Lucy/Util/ProcessID.c b/core/Lucy/Util/ProcessID.c
new file mode 100644
index 0000000..b875b4b
--- /dev/null
+++ b/core/Lucy/Util/ProcessID.c
@@ -0,0 +1,81 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ProcessID.h"
+
+/********************************* WINDOWS ********************************/
+#if (defined(CHY_HAS_WINDOWS_H) && defined(CHY_HAS_PROCESS_H) && !defined(__CYGWIN__))
+
+#include <Windows.h>
+#include <process.h>
+
+int
+lucy_PID_getpid(void) {
+ return GetCurrentProcessId();
+}
+
+chy_bool_t
+lucy_PID_active(int pid) {
+ // Attempt to open a handle to the process with permissions to terminate
+ // -- but don't actually terminate.
+ HANDLE handle = OpenProcess(PROCESS_TERMINATE, false, pid);
+ if (handle != NULL) {
+ // Successful open, therefore process is active.
+ CloseHandle(handle);
+ return true;
+ }
+ // If the opening attempt fails because we were denied permission, assume
+ // that the process is active.
+ if (GetLastError() == ERROR_ACCESS_DENIED) {
+ return true;
+ }
+
+ // Can't find any trace of the process, so return false.
+ return false;
+}
+
+
+/********************************* UNIXEN *********************************/
+#elif (defined(CHY_HAS_UNISTD_H) && defined(CHY_HAS_SIGNAL_H))
+
+#include <sys/types.h>
+#include <unistd.h>
+#include <signal.h>
+#include <errno.h>
+
+int
+lucy_PID_getpid(void) {
+ return getpid();
+}
+
+chy_bool_t
+lucy_PID_active(int pid) {
+ if (kill(pid, 0) == 0) {
+ return true; // signal succeeded, therefore pid active
+ }
+
+ if (errno != ESRCH) {
+ return true; // an error other than "pid not found", thus active
+ }
+
+ return false;
+}
+
+#else
+ #error "Can't find a known process ID API."
+#endif // OS switch.
+
+
diff --git a/core/Lucy/Util/ProcessID.cfh b/core/Lucy/Util/ProcessID.cfh
new file mode 100644
index 0000000..335e01b
--- /dev/null
+++ b/core/Lucy/Util/ProcessID.cfh
@@ -0,0 +1,35 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Provide platform-compatible process ID functions.
+ */
+inert class Lucy::Util::ProcessID cnick PID {
+
+ /** Return the ID for the current process.
+ */
+ inert int
+ getpid(void);
+
+ /** Return true if the supplied process ID is associated with an active
+ * process.
+ */
+ inert bool_t
+ active(int pid);
+}
+
+
diff --git a/core/Lucy/Util/Sleep.c b/core/Lucy/Util/Sleep.c
new file mode 100644
index 0000000..a556e78
--- /dev/null
+++ b/core/Lucy/Util/Sleep.c
@@ -0,0 +1,58 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SLEEP
+#include "Lucy/Util/Sleep.h"
+
+/********************************* WINDOWS ********************************/
+#ifdef CHY_HAS_WINDOWS_H
+
+#include <windows.h>
+
+void
+lucy_Sleep_sleep(uint32_t seconds) {
+ Sleep(seconds * 1000);
+}
+
+void
+lucy_Sleep_millisleep(uint32_t milliseconds) {
+ Sleep(milliseconds);
+}
+
+/********************************* UNIXEN *********************************/
+#elif defined(CHY_HAS_UNISTD_H)
+
+#include <unistd.h>
+
+void
+lucy_Sleep_sleep(uint32_t seconds) {
+ sleep(seconds);
+}
+
+void
+lucy_Sleep_millisleep(uint32_t milliseconds) {
+ uint32_t seconds = milliseconds / 1000;
+ milliseconds = milliseconds % 1000;
+ sleep(seconds);
+ // TODO: probe for usleep.
+ usleep(milliseconds * 1000);
+}
+
+#else
+ #error "Can't find a known sleep API."
+#endif // OS switch.
+
+
diff --git a/core/Lucy/Util/Sleep.cfh b/core/Lucy/Util/Sleep.cfh
new file mode 100644
index 0000000..fa5bb3a
--- /dev/null
+++ b/core/Lucy/Util/Sleep.cfh
@@ -0,0 +1,34 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Provide platform-compatible sleep() functions.
+ */
+inert class Lucy::Util::Sleep {
+
+ /** Sleep for <code>seconds</code> seconds.
+ */
+ inert void
+ sleep(uint32_t seconds);
+
+ /** Sleep for <code>milliseconds</code> milliseconds.
+ */
+ inert void
+ millisleep(uint32_t milliseconds);
+}
+
+
diff --git a/core/Lucy/Util/SortExternal.c b/core/Lucy/Util/SortExternal.c
new file mode 100644
index 0000000..e0c6a83
--- /dev/null
+++ b/core/Lucy/Util/SortExternal.c
@@ -0,0 +1,339 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTEXTERNAL
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/SortExternal.h"
+
+// Refill the main cache, drawing from the caches of all runs.
+static void
+S_refill_cache(SortExternal *self);
+
+// Absorb all the items which are "in-range" from all the Runs into the main
+// cache.
+static void
+S_absorb_slices(SortExternal *self, uint8_t *endpost);
+
+// Return the address for the item in one of the runs' caches which is the
+// highest in sort order, but which we can guarantee is lower in sort order
+// than any item which has yet to enter a run cache.
+static uint8_t*
+S_find_endpost(SortExternal *self);
+
+// Determine how many cache items are less than or equal to [endpost].
+static uint32_t
+S_find_slice_size(SortExternal *self, uint8_t *endpost);
+
+SortExternal*
+SortEx_init(SortExternal *self, size_t width) {
+ // Assign.
+ self->width = width;
+
+ // Init.
+ self->mem_thresh = U32_MAX;
+ self->cache = NULL;
+ self->cache_cap = 0;
+ self->cache_max = 0;
+ self->cache_tick = 0;
+ self->scratch = NULL;
+ self->scratch_cap = 0;
+ self->runs = VA_new(0);
+ self->slice_sizes = NULL;
+ self->slice_starts = NULL;
+ self->num_slices = 0;
+ self->flipped = false;
+
+ ABSTRACT_CLASS_CHECK(self, SORTEXTERNAL);
+ return self;
+}
+
+void
+SortEx_destroy(SortExternal *self) {
+ FREEMEM(self->scratch);
+ FREEMEM(self->slice_sizes);
+ FREEMEM(self->slice_starts);
+ if (self->cache) {
+ SortEx_Clear_Cache(self);
+ FREEMEM(self->cache);
+ }
+ DECREF(self->runs);
+ SUPER_DESTROY(self, SORTEXTERNAL);
+}
+
+void
+SortEx_clear_cache(SortExternal *self) {
+ self->cache_max = 0;
+ self->cache_tick = 0;
+}
+
+void
+SortEx_feed(SortExternal *self, void *data) {
+ const size_t width = self->width;
+ if (self->cache_max == self->cache_cap) {
+ size_t amount = Memory_oversize(self->cache_max + 1, width);
+ SortEx_Grow_Cache(self, amount);
+ }
+ uint8_t *target = self->cache + self->cache_max * width;
+ memcpy(target, data, width);
+ self->cache_max++;
+}
+
+static INLINE void*
+SI_peek(SortExternal *self) {
+ if (self->cache_tick >= self->cache_max) {
+ S_refill_cache(self);
+ }
+
+ if (self->cache_max > 0) {
+ return self->cache + self->cache_tick * self->width;
+ }
+ else {
+ return NULL;
+ }
+}
+
+void*
+SortEx_fetch(SortExternal *self) {
+ void *address = SI_peek(self);
+ self->cache_tick++;
+ return address;
+}
+
+void*
+SortEx_peek(SortExternal *self) {
+ return SI_peek(self);
+}
+
+void
+SortEx_sort_cache(SortExternal *self) {
+ if (self->cache_tick != 0) {
+ THROW(ERR, "Cant Sort_Cache() after fetching %u32 items", self->cache_tick);
+ }
+ if (self->cache_max != 0) {
+ VTable *vtable = SortEx_Get_VTable(self);
+ lucy_Sort_compare_t compare
+ = (lucy_Sort_compare_t)METHOD(vtable, SortEx, Compare);
+ if (self->scratch_cap < self->cache_cap) {
+ self->scratch_cap = self->cache_cap;
+ self->scratch = (uint8_t*)REALLOCATE(
+ self->scratch,
+ self->scratch_cap * self->width);
+ }
+ Sort_mergesort(self->cache, self->scratch, self->cache_max,
+ self->width, compare, self);
+ }
+}
+
+void
+SortEx_flip(SortExternal *self) {
+ SortEx_Flush(self);
+ self->flipped = true;
+}
+
+void
+SortEx_add_run(SortExternal *self, SortExternal *run) {
+ VA_Push(self->runs, (Obj*)run);
+ uint32_t num_runs = VA_Get_Size(self->runs);
+ self->slice_sizes = (uint32_t*)REALLOCATE(
+ self->slice_sizes,
+ num_runs * sizeof(uint32_t));
+ self->slice_starts = (uint8_t**)REALLOCATE(
+ self->slice_starts,
+ num_runs * sizeof(uint8_t*));
+}
+
+static void
+S_refill_cache(SortExternal *self) {
+ // Reset cache vars.
+ SortEx_Clear_Cache(self);
+
+ // Make sure all runs have at least one item in the cache.
+ uint32_t i = 0;
+ while (i < VA_Get_Size(self->runs)) {
+ SortExternal *const run = (SortExternal*)VA_Fetch(self->runs, i);
+ if (SortEx_Cache_Count(run) > 0 || SortEx_Refill(run) > 0) {
+ i++; // Run has some elements, so keep.
+ }
+ else {
+ VA_Excise(self->runs, i, 1);
+ }
+ }
+
+ // Absorb as many elems as possible from all runs into main cache.
+ if (VA_Get_Size(self->runs)) {
+ uint8_t *endpost = S_find_endpost(self);
+ S_absorb_slices(self, endpost);
+ }
+}
+
+static uint8_t*
+S_find_endpost(SortExternal *self) {
+ uint8_t *endpost = NULL;
+ const size_t width = self->width;
+
+ for (uint32_t i = 0, max = VA_Get_Size(self->runs); i < max; i++) {
+ // Get a run and retrieve the last item in its cache.
+ SortExternal *const run = (SortExternal*)VA_Fetch(self->runs, i);
+ const uint32_t tick = run->cache_max - 1;
+ if (tick >= run->cache_cap || run->cache_max < 1) {
+ THROW(ERR, "Invalid SortExternal cache access: %u32 %u32 %u32", tick,
+ run->cache_max, run->cache_cap);
+ }
+ else {
+ // Cache item with the highest sort value currently held in memory
+ // by the run.
+ uint8_t *candidate = run->cache + tick * width;
+
+ // If it's the first run, item is automatically the new endpost.
+ if (i == 0) {
+ endpost = candidate;
+ }
+ // If it's less than the current endpost, it's the new endpost.
+ else if (SortEx_Compare(self, candidate, endpost) < 0) {
+ endpost = candidate;
+ }
+ }
+ }
+
+ return endpost;
+}
+
+static void
+S_absorb_slices(SortExternal *self, uint8_t *endpost) {
+ size_t width = self->width;
+ uint32_t num_runs = VA_Get_Size(self->runs);
+ uint8_t **slice_starts = self->slice_starts;
+ uint32_t *slice_sizes = self->slice_sizes;
+ VTable *vtable = SortEx_Get_VTable(self);
+ lucy_Sort_compare_t compare
+ = (lucy_Sort_compare_t)METHOD(vtable, SortEx, Compare);
+
+ if (self->cache_max != 0) { THROW(ERR, "Can't refill unless empty"); }
+
+ // Move all the elements in range into the main cache as slices.
+ for (uint32_t i = 0; i < num_runs; i++) {
+ SortExternal *const run = (SortExternal*)VA_Fetch(self->runs, i);
+ uint32_t slice_size = S_find_slice_size(run, endpost);
+
+ if (slice_size) {
+ // Move slice content from run cache to main cache.
+ if (self->cache_max + slice_size > self->cache_cap) {
+ size_t cap = Memory_oversize(self->cache_max + slice_size,
+ width);
+ SortEx_Grow_Cache(self, cap);
+ }
+ memcpy(self->cache + self->cache_max * width,
+ run->cache + run->cache_tick * width,
+ slice_size * width);
+ run->cache_tick += slice_size;
+ self->cache_max += slice_size;
+
+ // Track number of slices and slice sizes.
+ slice_sizes[self->num_slices++] = slice_size;
+ }
+ }
+
+ // Transform slice starts from ticks to pointers.
+ uint32_t total = 0;
+ for (uint32_t i = 0; i < self->num_slices; i++) {
+ slice_starts[i] = self->cache + total * width;
+ total += slice_sizes[i];
+ }
+
+ // The main cache now consists of several slices. Sort the main cache,
+ // but exploit the fact that each slice is already sorted.
+ if (self->scratch_cap < self->cache_cap) {
+ self->scratch_cap = self->cache_cap;
+ self->scratch = (uint8_t*)REALLOCATE(
+ self->scratch, self->scratch_cap * width);
+ }
+
+ // Exploit previous sorting, rather than sort cache naively.
+ // Leave the first slice intact if the number of slices is odd. */
+ while (self->num_slices > 1) {
+ uint32_t i = 0;
+ uint32_t j = 0;
+
+ while (i < self->num_slices) {
+ if (self->num_slices - i >= 2) {
+ // Merge two consecutive slices.
+ const uint32_t merged_size = slice_sizes[i] + slice_sizes[i + 1];
+ Sort_merge(slice_starts[i], slice_sizes[i],
+ slice_starts[i + 1], slice_sizes[i + 1], self->scratch,
+ self->width, compare, self);
+ slice_sizes[j] = merged_size;
+ slice_starts[j] = slice_starts[i];
+ memcpy(slice_starts[j], self->scratch, merged_size * width);
+ i += 2;
+ j += 1;
+ }
+ else if (self->num_slices - i >= 1) {
+ // Move single slice pointer.
+ slice_sizes[j] = slice_sizes[i];
+ slice_starts[j] = slice_starts[i];
+ i += 1;
+ j += 1;
+ }
+ }
+ self->num_slices = j;
+ }
+
+ self->num_slices = 0;
+}
+
+void
+SortEx_grow_cache(SortExternal *self, uint32_t size) {
+ if (size > self->cache_cap) {
+ self->cache = (uint8_t*)REALLOCATE(self->cache, size * self->width);
+ self->cache_cap = size;
+ }
+}
+
+static uint32_t
+S_find_slice_size(SortExternal *self, uint8_t *endpost) {
+ int32_t lo = self->cache_tick - 1;
+ int32_t hi = self->cache_max;
+ uint8_t *const cache = self->cache;
+ const size_t width = self->width;
+ SortEx_compare_t compare
+ = (SortEx_compare_t)METHOD(SortEx_Get_VTable(self), SortEx, Compare);
+
+ // Binary search.
+ while (hi - lo > 1) {
+ const int32_t mid = lo + ((hi - lo) / 2);
+ const int32_t delta = compare(self, cache + mid * width, endpost);
+ if (delta > 0) { hi = mid; }
+ else { lo = mid; }
+ }
+
+ // If lo is still -1, we didn't find anything.
+ return lo == -1
+ ? 0
+ : (lo - self->cache_tick) + 1;
+}
+
+void
+SortEx_set_mem_thresh(SortExternal *self, uint32_t mem_thresh) {
+ self->mem_thresh = mem_thresh;
+}
+
+uint32_t
+SortEx_cache_count(SortExternal *self) {
+ return self->cache_max - self->cache_tick;
+}
+
+
diff --git a/core/Lucy/Util/SortExternal.cfh b/core/Lucy/Util/SortExternal.cfh
new file mode 100644
index 0000000..dd7ea44
--- /dev/null
+++ b/core/Lucy/Util/SortExternal.cfh
@@ -0,0 +1,149 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+#include <stddef.h>
+#include "Lucy/Object/Obj.h"
+#include "Lucy/Util/SortUtils.h"
+
+#define LUCY_SORTEX_DEFAULT_MEM_THRESHOLD 0x1000000
+#ifdef LUCY_USE_SHORT_NAMES
+ #define SORTEX_DEFAULT_MEM_THRESHOLD LUCY_SORTEX_DEFAULT_MEM_THRESHOLD
+#endif
+__END_C__
+
+/** Abstract external sorter.
+ *
+ * SortExternal objects are sort pools which allow you to sort large amounts
+ * of data. To achieve this, you Feed() all values into the SortExternal
+ * object, Flip() the object from write mode to read mode, then Fetch() the
+ * values one at a time in sorted order.
+ *
+ * It's expected that the total memory footprint of the sortable objects will
+ * eventually exceed a specified threshold; at that point, the SortExternal
+ * object will call the abstract method Flush(). It's expected that Flush()
+ * implementations will empty out the current sort cache, write a sorted "run"
+ * to external storage, and add a new child SortExternal object to the top
+ * level object's "runs" array to represent the flushed content.
+ *
+ * During the read phase, the child objects retrieve values from external
+ * storage by calling the abstract method Refill(). The top-level
+ * SortExternal object then interleaves multiple sorted streams to produce a
+ * single unified stream of sorted values.
+ */
+abstract class Lucy::Util::SortExternal cnick SortEx
+ inherits Lucy::Object::Obj {
+
+ uint8_t *cache;
+ uint32_t cache_cap;
+ uint32_t cache_max;
+ uint32_t cache_tick;
+ uint8_t *scratch;
+ uint32_t scratch_cap;
+ VArray *runs;
+ uint32_t num_slices;
+ uint8_t **slice_starts;
+ uint32_t *slice_sizes;
+ uint32_t mem_thresh;
+ size_t width;
+ bool_t flipped;
+
+ inert SortExternal*
+ init(SortExternal *self, size_t width);
+
+ /** Compare two sortable elements.
+ */
+ abstract int
+ Compare(SortExternal *self, void *va, void *vb);
+
+ /** Flush all elements currently in the cache.
+ *
+ * Presumably this entails sorting everything, writing the sorted elements
+ * to disk, spawning a child object to represent those elements, and
+ * adding that child to the top level object via Add_Run().
+ */
+ abstract void
+ Flush(SortExternal *self);
+
+ /** Add data to the sort pool.
+ *
+ * @param data Pointer to the data being added, which must be exactly
+ * <code>width</code> bytes in size.
+ */
+ void
+ Feed(SortExternal *self, void *data);
+
+ /** Flip the sortex from write mode to read mode.
+ */
+ void
+ Flip(SortExternal *self);
+
+ /** Fetch the next sorted item from the sort pool. Invalid prior to
+ * calling Flip(). Returns NULL when all elements have been exhausted.
+ */
+ nullable void*
+ Fetch(SortExternal *self);
+
+ /** Preview the next item that Fetch will return, but don't pop it.
+ * Invalid prior to calling Flip().
+ */
+ nullable void*
+ Peek(SortExternal *self);
+
+ /** Add a run to the sortex's collection.
+ */
+ void
+ Add_Run(SortExternal *self, decremented SortExternal *run);
+
+ /** Refill the cache of a run. Will only be called on child objects, not
+ * the main object.
+ */
+ abstract uint32_t
+ Refill(SortExternal *self);
+
+ /** Sort all items currently in the main cache.
+ */
+ void
+ Sort_Cache(SortExternal *self);
+
+ /** Reset cache variables so that cache gives the appearance of having
+ * been initialized.
+ *
+ * Subclasses may take steps to release items held in cache, if any.
+ */
+ void
+ Clear_Cache(SortExternal *self);
+
+ /** Return the number of items presently in the cache.
+ */
+ uint32_t
+ Cache_Count(SortExternal *self);
+
+ /** Allocate more memory to the cache.
+ */
+ void
+ Grow_Cache(SortExternal *self, uint32_t new_cache_cap);
+
+ void
+ Set_Mem_Thresh(SortExternal *self, uint32_t mem_thresh);
+
+ public void
+ Destroy(SortExternal *self);
+}
+
+
diff --git a/core/Lucy/Util/SortUtils.c b/core/Lucy/Util/SortUtils.c
new file mode 100644
index 0000000..449937f
--- /dev/null
+++ b/core/Lucy/Util/SortUtils.c
@@ -0,0 +1,466 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_SORTUTILS
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include <string.h>
+#include "Lucy/Util/SortUtils.h"
+#include "Lucy/Object/Err.h"
+
+// Define four-byte and eight-byte types so that we can dereference void
+// pointers like integer pointers. The only significance of using int32_t and
+// int64_t is that they are 4 and 8 bytes.
+#define FOUR_BYTE_TYPE int32_t
+#define EIGHT_BYTE_TYPE int64_t
+
+/***************************** mergesort ************************************/
+
+// Recursive merge sorting functions.
+static void
+S_msort4(void *velems, void *vscratch, uint32_t left, uint32_t right,
+ lucy_Sort_compare_t compare, void *context);
+static void
+S_msort8(void *velems, void *vscratch, uint32_t left, uint32_t right,
+ lucy_Sort_compare_t compare, void *context);
+static void
+S_msort_any(void *velems, void *vscratch, uint32_t left, uint32_t right,
+ lucy_Sort_compare_t compare, void *context, size_t width);
+
+static INLINE void
+SI_merge(void *left_vptr, uint32_t left_size,
+ void *right_vptr, uint32_t right_size,
+ void *vdest, size_t width, lucy_Sort_compare_t compare, void *context);
+
+void
+Sort_mergesort(void *elems, void *scratch, uint32_t num_elems, uint32_t width,
+ lucy_Sort_compare_t compare, void *context) {
+ // Arrays of 0 or 1 items are already sorted.
+ if (num_elems < 2) { return; }
+
+ // Validate.
+ if (num_elems >= I32_MAX) {
+ THROW(ERR, "Provided %u64 elems, but can't handle more than %i32",
+ (uint64_t)num_elems, I32_MAX);
+ }
+
+ // Dispatch by element size.
+ switch (width) {
+ case 0:
+ THROW(ERR, "Parameter 'width' cannot be 0");
+ break;
+ case 4:
+ S_msort4(elems, scratch, 0, num_elems - 1, compare, context);
+ break;
+ case 8:
+ S_msort8(elems, scratch, 0, num_elems - 1, compare, context);
+ break;
+ default:
+ S_msort_any(elems, scratch, 0, num_elems - 1, compare,
+ context, width);
+ break;
+ }
+}
+
+void
+Sort_merge(void *left_ptr, uint32_t left_size,
+ void *right_ptr, uint32_t right_size,
+ void *dest, size_t width, lucy_Sort_compare_t compare,
+ void *context) {
+ switch (width) {
+ case 0:
+ THROW(ERR, "Parameter 'width' cannot be 0");
+ break;
+ case 4:
+ SI_merge(left_ptr, left_size, right_ptr, right_size,
+ dest, 4, compare, context);
+ break;
+ case 8:
+ SI_merge(left_ptr, left_size, right_ptr, right_size,
+ dest, 8, compare, context);
+ break;
+ default:
+ SI_merge(left_ptr, left_size, right_ptr, right_size,
+ dest, width, compare, context);
+ break;
+ }
+}
+
+#define WIDTH 4
+static void
+S_msort4(void *velems, void *vscratch, uint32_t left, uint32_t right,
+ lucy_Sort_compare_t compare, void *context) {
+ uint8_t *elems = (uint8_t*)velems;
+ uint8_t *scratch = (uint8_t*)vscratch;
+ if (right > left) {
+ const uint32_t mid = ((right + left) / 2) + 1;
+ S_msort4(elems, scratch, left, mid - 1, compare, context);
+ S_msort4(elems, scratch, mid, right, compare, context);
+ SI_merge((elems + left * WIDTH), (mid - left),
+ (elems + mid * WIDTH), (right - mid + 1),
+ scratch, WIDTH, compare, context);
+ memcpy((elems + left * WIDTH), scratch, ((right - left + 1) * WIDTH));
+ }
+}
+
+#undef WIDTH
+#define WIDTH 8
+static void
+S_msort8(void *velems, void *vscratch, uint32_t left, uint32_t right,
+ lucy_Sort_compare_t compare, void *context) {
+ uint8_t *elems = (uint8_t*)velems;
+ uint8_t *scratch = (uint8_t*)vscratch;
+ if (right > left) {
+ const uint32_t mid = ((right + left) / 2) + 1;
+ S_msort8(elems, scratch, left, mid - 1, compare, context);
+ S_msort8(elems, scratch, mid, right, compare, context);
+ SI_merge((elems + left * WIDTH), (mid - left),
+ (elems + mid * WIDTH), (right - mid + 1),
+ scratch, WIDTH, compare, context);
+ memcpy((elems + left * WIDTH), scratch, ((right - left + 1) * WIDTH));
+ }
+}
+
+#undef WIDTH
+static void
+S_msort_any(void *velems, void *vscratch, uint32_t left, uint32_t right,
+ lucy_Sort_compare_t compare, void *context, size_t width) {
+ uint8_t *elems = (uint8_t*)velems;
+ uint8_t *scratch = (uint8_t*)vscratch;
+ if (right > left) {
+ const uint32_t mid = ((right + left) / 2) + 1;
+ S_msort_any(elems, scratch, left, mid - 1, compare, context, width);
+ S_msort_any(elems, scratch, mid, right, compare, context, width);
+ SI_merge((elems + left * width), (mid - left),
+ (elems + mid * width), (right - mid + 1),
+ scratch, width, compare, context);
+ memcpy((elems + left * width), scratch, ((right - left + 1) * width));
+ }
+}
+
+static INLINE void
+SI_merge(void *left_vptr, uint32_t left_size,
+ void *right_vptr, uint32_t right_size,
+ void *vdest, size_t width, lucy_Sort_compare_t compare,
+ void *context) {
+ uint8_t *left_ptr = (uint8_t*)left_vptr;
+ uint8_t *right_ptr = (uint8_t*)right_vptr;
+ uint8_t *left_limit = left_ptr + left_size * width;
+ uint8_t *right_limit = right_ptr + right_size * width;
+ uint8_t *dest = (uint8_t*)vdest;
+
+ while (left_ptr < left_limit && right_ptr < right_limit) {
+ if (compare(context, left_ptr, right_ptr) < 1) {
+ memcpy(dest, left_ptr, width);
+ dest += width;
+ left_ptr += width;
+ }
+ else {
+ memcpy(dest, right_ptr, width);
+ dest += width;
+ right_ptr += width;
+ }
+ }
+
+ const size_t left_remaining = left_limit - left_ptr;
+ memcpy(dest, left_ptr, left_remaining);
+ dest += left_remaining;
+ const size_t right_remaining = right_limit - right_ptr;
+ memcpy(dest, right_ptr, right_remaining);
+}
+
+/***************************** quicksort ************************************/
+
+// Quicksort implementations optimized for four-byte and eight-byte elements.
+static void
+S_qsort4(FOUR_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context);
+static void
+S_qsort8(EIGHT_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context);
+
+// Swap two elements.
+static INLINE void
+SI_exchange4(FOUR_BYTE_TYPE *elems, int32_t left, int32_t right);
+static INLINE void
+SI_exchange8(EIGHT_BYTE_TYPE *elems, int32_t left, int32_t right);
+
+/* Select a pivot by choosing the median of three values, guarding against
+ * the worst-case behavior of quicksort. Place the pivot in the rightmost
+ * slot.
+ *
+ * Possible states:
+ *
+ * abc => abc => abc => acb
+ * acb => acb => acb => acb
+ * bac => abc => abc => acb
+ * bca => bca => acb => acb
+ * cba => bca => acb => acb
+ * cab => acb => acb => acb
+ * aab => aab => aab => aba
+ * aba => aba => aba => aba
+ * baa => aba => aba => aba
+ * bba => bba => abb => abb
+ * bab => abb => abb => abb
+ * abb => abb => abb => abb
+ * aaa => aaa => aaa => aaa
+ */
+static INLINE FOUR_BYTE_TYPE*
+SI_choose_pivot4(FOUR_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context);
+static INLINE EIGHT_BYTE_TYPE*
+SI_choose_pivot8(EIGHT_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context);
+
+void
+Sort_quicksort(void *elems, size_t num_elems, size_t width,
+ lucy_Sort_compare_t compare, void *context) {
+ // Arrays of 0 or 1 items are already sorted.
+ if (num_elems < 2) { return; }
+
+ // Validate.
+ if (num_elems >= I32_MAX) {
+ THROW(ERR, "Provided %u64 elems, but can't handle more than %i32",
+ (uint64_t)num_elems, I32_MAX);
+ }
+
+ if (width == 4) {
+ S_qsort4((FOUR_BYTE_TYPE*)elems, 0, num_elems - 1, compare, context);
+ }
+ else if (width == 8) {
+ S_qsort8((EIGHT_BYTE_TYPE*)elems, 0, num_elems - 1, compare, context);
+ }
+ else {
+ THROW(ERR, "Unsupported width: %i64", (int64_t)width);
+ }
+}
+
+/************************* quicksort 4 byte *********************************/
+
+static INLINE void
+SI_exchange4(FOUR_BYTE_TYPE *elems, int32_t left, int32_t right) {
+ FOUR_BYTE_TYPE saved = elems[left];
+ elems[left] = elems[right];
+ elems[right] = saved;
+}
+
+static INLINE FOUR_BYTE_TYPE*
+SI_choose_pivot4(FOUR_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context) {
+ if (right - left > 1) {
+ int32_t mid = left + (right - left) / 2;
+ if (compare(context, elems + left, elems + mid) > 0) {
+ SI_exchange4(elems, left, mid);
+ }
+ if (compare(context, elems + left, elems + right) > 0) {
+ SI_exchange4(elems, left, right);
+ }
+ if (compare(context, elems + right, elems + mid) > 0) {
+ SI_exchange4(elems, right, mid);
+ }
+ }
+ return elems + right;
+}
+
+static void
+S_qsort4(FOUR_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context) {
+ FOUR_BYTE_TYPE *const pivot
+ = SI_choose_pivot4(elems, left, right, compare, context);
+ int32_t i = left - 1;
+ int32_t j = right;
+ int32_t p = left - 1;
+ int32_t q = right;
+
+ if (right <= left) { return; }
+
+ /* TODO: A standard optimization for quicksort is to fall back to an
+ * insertion sort when the the number of elements to be sorted becomes
+ * small enough. */
+
+ while (1) {
+ int comparison1;
+ int comparison2;
+
+ // Find an element from the left that is greater than or equal to the
+ // pivot (i.e. that should move to the right).
+ while (1) {
+ i++;
+ comparison1 = compare(context, elems + i, pivot);
+ if (comparison1 >= 0) { break; }
+ }
+
+ // Find an element from the right that is less than or equal to the
+ // pivot (i.e. that should move to the left).
+ while (1) {
+ j--;
+ comparison2 = compare(context, elems + j, pivot);
+ if (comparison2 <= 0) { break; }
+ if (j == left) { break; }
+ }
+
+ // Bail out of loop when we meet in the middle.
+ if (i >= j) { break; }
+
+ // Swap the elements we found, so the lesser element moves left and
+ // the greater element moves right.
+ SI_exchange4(elems, i, j);
+
+ // Move any elements which test as "equal" to the pivot to the outside
+ // edges of the array.
+ if (comparison2 == 0) {
+ p++;
+ SI_exchange4(elems, p, i);
+ }
+ if (comparison1 == 0) {
+ q--;
+ SI_exchange4(elems, j, q);
+ }
+ }
+
+ /* Move "equal" elements from the outside edges to the center.
+ *
+ * Before:
+ *
+ * equal | less_than | greater_than | equal
+ *
+ * After:
+ *
+ * less_than | equal | greater_than
+ */
+ {
+ int32_t k;
+ SI_exchange4(elems, i, right);
+ j = i - 1;
+ i++;
+ for (k = left; k < p; k++, j--) { SI_exchange4(elems, k, j); }
+ for (k = right - 1; k > q; k--, i++) { SI_exchange4(elems, i, k); }
+ }
+
+ // Recurse.
+ S_qsort4(elems, left, j, compare, context); // Sort less_than.
+ S_qsort4(elems, i, right, compare, context); // Sort greater_than.
+}
+
+/************************* quicksort 8 byte *********************************/
+
+static INLINE void
+SI_exchange8(EIGHT_BYTE_TYPE *elems, int32_t left, int32_t right) {
+ EIGHT_BYTE_TYPE saved = elems[left];
+ elems[left] = elems[right];
+ elems[right] = saved;
+}
+
+static INLINE EIGHT_BYTE_TYPE*
+SI_choose_pivot8(EIGHT_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context) {
+ if (right - left > 1) {
+ int32_t mid = left + (right - left) / 2;
+ if (compare(context, elems + left, elems + mid) > 0) {
+ SI_exchange8(elems, left, mid);
+ }
+ if (compare(context, elems + left, elems + right) > 0) {
+ SI_exchange8(elems, left, right);
+ }
+ if (compare(context, elems + right, elems + mid) > 0) {
+ SI_exchange8(elems, right, mid);
+ }
+ }
+ return elems + right;
+}
+
+static void
+S_qsort8(EIGHT_BYTE_TYPE *elems, int32_t left, int32_t right,
+ lucy_Sort_compare_t compare, void *context) {
+ EIGHT_BYTE_TYPE *const pivot
+ = SI_choose_pivot8(elems, left, right, compare, context);
+ int32_t i = left - 1;
+ int32_t j = right;
+ int32_t p = left - 1;
+ int32_t q = right;
+
+ if (right <= left) { return; }
+
+ /* TODO: A standard optimization for quicksort is to fall back to an
+ * insertion sort when the the number of elements to be sorted becomes
+ * small enough. */
+
+ while (1) {
+ int comparison1;
+ int comparison2;
+
+ // Find an element from the left that is greater than or equal to the
+ // pivot (i.e. that should move to the right).
+ while (1) {
+ i++;
+ comparison1 = compare(context, elems + i, pivot);
+ if (comparison1 >= 0) { break; }
+ }
+
+ // Find an element from the right that is less than or equal to the
+ // pivot (i.e. that should move to the left).
+ while (1) {
+ j--;
+ comparison2 = compare(context, elems + j, pivot);
+ if (comparison2 <= 0) { break; }
+ if (j == left) { break; }
+ }
+
+ // Bail out of loop when we meet in the middle.
+ if (i >= j) { break; }
+
+ // Swap the elements we found, so the lesser element moves left and
+ // the greater element moves right.
+ SI_exchange8(elems, i, j);
+
+ // Move any elements which test as "equal" to the pivot to the outside
+ // edges of the array.
+ if (comparison2 == 0) {
+ p++;
+ SI_exchange8(elems, p, i);
+ }
+ if (comparison1 == 0) {
+ q--;
+ SI_exchange8(elems, j, q);
+ }
+ }
+
+ /* Move "equal" elements from the outside edges to the center.
+ *
+ * Before:
+ *
+ * equal | less_than | greater_than | equal
+ *
+ * After:
+ *
+ * less_than | equal | greater_than
+ */
+ {
+ int32_t k;
+ SI_exchange8(elems, i, right);
+ j = i - 1;
+ i++;
+ for (k = left; k < p; k++, j--) { SI_exchange8(elems, k, j); }
+ for (k = right - 1; k > q; k--, i++) { SI_exchange8(elems, i, k); }
+ }
+
+ // Recurse.
+ S_qsort8(elems, left, j, compare, context); // Sort less_than.
+ S_qsort8(elems, i, right, compare, context); // Sort greater_than.
+}
+
+
diff --git a/core/Lucy/Util/SortUtils.cfh b/core/Lucy/Util/SortUtils.cfh
new file mode 100644
index 0000000..e116567
--- /dev/null
+++ b/core/Lucy/Util/SortUtils.cfh
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+__C__
+typedef int
+(*lucy_Sort_compare_t)(void *context, const void *va, const void *vb);
+__END_C__
+
+/** Specialized sorting routines.
+ *
+ * SortUtils provides a merge sort algorithm which allows access to its
+ * internals, enabling specialized functions to jump in and only execute part
+ * of the sort.
+ *
+ * SortUtils also provides a quicksort with an additional context argument.
+ */
+inert class Lucy::Util::SortUtils cnick Sort {
+
+ /** Perform a mergesort. In addition to providing a contiguous array of
+ * elements to be sorted and their count, the caller must also provide a
+ * scratch buffer with room for at least as many elements as are to be
+ * sorted.
+ */
+ inert void
+ mergesort(void *elems, void *scratch, uint32_t num_elems, uint32_t width,
+ lucy_Sort_compare_t compare, void *context);
+
+ /** Merge two source arrays together using the classic mergesort merge
+ * algorithm, storing the result in <code>dest</code>.
+ *
+ * Most merge functions operate on a single contiguous array and copy the
+ * merged results results back into the source array before returning.
+ * These two differ in that it is possible to operate on two discontiguous
+ * source arrays. Copying the results back into the source array is the
+ * responsibility of the caller.
+ *
+ * Lucy's external sort takes advantage of this when it is reading
+ * back pre-sorted runs from disk and merging the streams into a
+ * consolidated buffer.
+ */
+ inert void
+ merge(void *left_ptr, uint32_t left_num_elems,
+ void *right_ptr, uint32_t right_num_elems,
+ void *dest, size_t width, lucy_Sort_compare_t compare, void *context);
+
+ /** Quicksort.
+ */
+ inert void
+ quicksort(void *elems, size_t num_elems, size_t width,
+ lucy_Sort_compare_t compare, void *context);
+}
+
+
diff --git a/core/Lucy/Util/Stepper.c b/core/Lucy/Util/Stepper.c
new file mode 100644
index 0000000..91ecfd2
--- /dev/null
+++ b/core/Lucy/Util/Stepper.c
@@ -0,0 +1,28 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_STEPPER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/Stepper.h"
+
+Stepper*
+Stepper_init(Stepper *self) {
+ ABSTRACT_CLASS_CHECK(self, STEPPER);
+ return self;
+}
+
+
diff --git a/core/Lucy/Util/Stepper.cfh b/core/Lucy/Util/Stepper.cfh
new file mode 100644
index 0000000..e3f2f98
--- /dev/null
+++ b/core/Lucy/Util/Stepper.cfh
@@ -0,0 +1,83 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/**
+ * Abstract encoder/decoder.
+ *
+ * Many Lucy files consist of a single variable length record type
+ * repeated over and over. A Stepper both reads and writes such a file.
+ *
+ * Since the write algorithms for different Stepper types may require
+ * differing argument lists, it is left to the subclass to define the routine.
+ *
+ * Sometimes it is possible to change a file's format by changing only a
+ * Stepper. In that case, a compatibility version of the old class may be
+ * squirreled away as a plugin, to be accessed only when reading files written
+ * to the old format. This cuts down on special-case code in the most current
+ * version.
+ *
+ * Furthermore, isolating I/O code within a Stepper typically clarifies the
+ * logic of the class which calls Stepper_Read_Record.
+ */
+
+class Lucy::Util::Stepper inherits Lucy::Object::Obj {
+
+ inert Stepper*
+ init(Stepper *self);
+
+ public abstract void
+ Reset(Stepper* self);
+
+ /** Update internal state to reflect <code>value</code> and write a frame
+ * to <code>outstream</code> that can be read using Read_Key_Frame().
+ *
+ * @param outstream An OutStream.
+ * @param value State information.
+ */
+ public abstract void
+ Write_Key_Frame(Stepper *self, OutStream *outstream, Obj *value);
+
+ /** Update internal state to reflect <code>value</code> and write a frame
+ * to <code>outstream</code> that can be read using Read_Delta().
+ *
+ * @param outstream An OutStream.
+ * @param value State information.
+ */
+ public abstract void
+ Write_Delta(Stepper *self, OutStream *outstream, Obj *value);
+
+ /** Update intern state using information read from <code>instream</code>.
+ *
+ * @param instream An InStream.
+ */
+ public abstract void
+ Read_Key_Frame(Stepper *self, InStream *instream);
+
+ /** Update state using a combination of information from
+ * <code>instream</code> and the current internal state.
+ */
+ public abstract void
+ Read_Delta(Stepper *self, InStream *instream);
+
+ /** Read the next record from the instream, storing state in [self].
+ */
+ abstract void
+ Read_Record(Stepper *self, InStream *instream);
+}
+
+
diff --git a/core/Lucy/Util/StringHelper.c b/core/Lucy/Util/StringHelper.c
new file mode 100644
index 0000000..86f1825
--- /dev/null
+++ b/core/Lucy/Util/StringHelper.c
@@ -0,0 +1,135 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_STRINGHELPER
+#include <string.h>
+
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "Lucy/Util/StringHelper.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Util/Memory.h"
+
+int32_t
+StrHelp_overlap(const char *a, const char *b, size_t a_len, size_t b_len) {
+ size_t i;
+ const size_t len = a_len <= b_len ? a_len : b_len;
+
+ for (i = 0; i < len; i++) {
+ if (*a++ != *b++) { break; }
+ }
+ return i;
+}
+
+static const char base36_chars[] = "0123456789abcdefghijklmnopqrstuvwxyz";
+
+uint32_t
+StrHelp_to_base36(uint64_t num, void *buffer) {
+ char my_buf[StrHelp_MAX_BASE36_BYTES];
+ char *buf = my_buf + StrHelp_MAX_BASE36_BYTES - 1;
+ char *end = buf;
+
+ // Null terminate.
+ *buf = '\0';
+
+ // Convert to base 36 characters.
+ do {
+ *(--buf) = base36_chars[num % 36];
+ num /= 36;
+ } while (num > 0);
+
+ {
+ uint32_t size = end - buf;
+ memcpy(buffer, buf, size + 1);
+ return size;
+ }
+}
+
+uint32_t
+StrHelp_encode_utf8_char(uint32_t code_point, void *buffer) {
+ uint8_t *buf = (uint8_t*)buffer;
+ if (code_point <= 0x7F) { // ASCII
+ buf[0] = (uint8_t)code_point;
+ return 1;
+ }
+ else if (code_point <= 0x07FF) { // 2 byte range
+ buf[0] = (uint8_t)(0xC0 | (code_point >> 6));
+ buf[1] = (uint8_t)(0x80 | (code_point & 0x3f));
+ return 2;
+ }
+ else if (code_point <= 0xFFFF) { // 3 byte range
+ buf[0] = (uint8_t)(0xE0 | (code_point >> 12));
+ buf[1] = (uint8_t)(0x80 | ((code_point >> 6) & 0x3F));
+ buf[2] = (uint8_t)(0x80 | (code_point & 0x3f));
+ return 3;
+ }
+ else if (code_point <= 0x10FFFF) { // 4 byte range
+ buf[0] = (uint8_t)(0xF0 | (code_point >> 18));
+ buf[1] = (uint8_t)(0x80 | ((code_point >> 12) & 0x3F));
+ buf[2] = (uint8_t)(0x80 | ((code_point >> 6) & 0x3F));
+ buf[3] = (uint8_t)(0x80 | (code_point & 0x3f));
+ return 4;
+ }
+ else {
+ THROW(ERR, "Illegal Unicode code point: %u32", code_point);
+ UNREACHABLE_RETURN(uint32_t);
+ }
+}
+
+uint32_t
+StrHelp_decode_utf8_char(const char *ptr) {
+ const uint8_t *const string = (const uint8_t*)ptr;
+ uint32_t retval = *string;
+ int bytes = StrHelp_UTF8_COUNT[retval];
+
+ switch (bytes & 0x7) {
+ case 1:
+ break;
+
+ case 2:
+ retval = ((retval & 0x1F) << 6)
+ | (string[1] & 0x3F);
+ break;
+
+ case 3:
+ retval = ((retval & 0x0F) << 12)
+ | ((string[1] & 0x3F) << 6)
+ | (string[2] & 0x3F);
+ break;
+
+ case 4:
+ retval = ((retval & 0x07) << 18)
+ | ((string[1] & 0x3F) << 12)
+ | ((string[2] & 0x3F) << 6)
+ | (string[3] & 0x3F);
+ break;
+
+ default:
+ THROW(ERR, "Invalid UTF-8 header byte: %x32", retval);
+ }
+
+ return retval;
+}
+
+const char*
+StrHelp_back_utf8_char(const char *ptr, char *start) {
+ while (--ptr >= start) {
+ if ((*ptr & 0xC0) != 0x80) { return ptr; }
+ }
+ return NULL;
+}
+
diff --git a/core/Lucy/Util/StringHelper.cfh b/core/Lucy/Util/StringHelper.cfh
new file mode 100644
index 0000000..2c4a112
--- /dev/null
+++ b/core/Lucy/Util/StringHelper.cfh
@@ -0,0 +1,83 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+inert class Lucy::Util::StringHelper cnick StrHelp {
+
+ /* A table where the values indicate the number of bytes in a UTF-8
+ * sequence implied by the leading utf8 byte.
+ */
+ inert const uint8_t[] UTF8_COUNT;
+
+ /** Return the number of bytes that two strings have in common.
+ */
+ inert int32_t
+ overlap(const char *a, const char *b, size_t a_len, size_t b_len);
+
+ /** Encode a NULL-terminated string representation of a value in base 36
+ * into <code>buffer</code>.
+ *
+ * @param value The number to be encoded.
+ * @param buffer A buffer at least MAX_BASE36_BYTES bytes long.
+ * @return the number of digits encoded (not including the terminating
+ * NULL).
+ */
+ inert uint32_t
+ to_base36(uint64_t value, void *buffer);
+
+ /** Return true if the string is valid UTF-8, false otherwise.
+ */
+ inert bool_t
+ utf8_valid(const char *ptr, size_t len);
+
+ /** Returns true if the code point qualifies as Unicode whitespace.
+ */
+ inert bool_t
+ is_whitespace(uint32_t code_point);
+
+ /** Encode a Unicode code point to a UTF-8 sequence.
+ *
+ * @param code_point A legal unicode code point.
+ * @param buffer Write buffer which must hold at least 4 bytes (the
+ * maximum legal length for a UTF-8 char).
+ */
+ inert uint32_t
+ encode_utf8_char(uint32_t code_point, void *buffer);
+
+ /** Decode a UTF-8 sequence to a Unicode code point. Assumes valid UTF-8.
+ */
+ inert uint32_t
+ decode_utf8_char(const char *utf8);
+
+ /** Return the first non-continuation byte before the supplied pointer.
+ * If backtracking progresses beyond the supplied start, return NULL.
+ */
+ inert nullable const char*
+ back_utf8_char(const char *utf8, char *start);
+}
+
+__C__
+/** The maximum number of bytes encoded by to_base36(), including the
+ * terminating NULL.
+ */
+#define lucy_StrHelp_MAX_BASE36_BYTES 14
+#ifdef LUCY_USE_SHORT_NAMES
+ #define StrHelp_MAX_BASE36_BYTES lucy_StrHelp_MAX_BASE36_BYTES
+#endif
+__END_C__
+
+
diff --git a/core/Lucy/Util/StringHelper2.c b/core/Lucy/Util/StringHelper2.c
new file mode 100644
index 0000000..7f5eff1
--- /dev/null
+++ b/core/Lucy/Util/StringHelper2.c
@@ -0,0 +1,68 @@
+#include <charmony.h>
+#include "Lucy/Util/StringHelper.h"
+
+// The content of the following table is derived from the
+// "utf8_countTrailBytes" table in ICU4C 4.4.1.
+
+/*
+ICU License - ICU 1.8.1 and later
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright (c) 1995-2010 International Business Machines Corporation and others
+
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell
+copies of the Software, and to permit persons
+to whom the Software is furnished to do so, provided that the above
+copyright notice(s) and this permission notice appear in all copies
+of the Software and that both the above copyright notice(s) and this
+permission notice appear in supporting documentation.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. IN NO EVENT SHALL
+THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM,
+OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER
+RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
+NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE
+USE OR PERFORMANCE OF THIS SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall not be
+used in advertising or otherwise to promote the sale, use or other dealings in
+this Software without prior written authorization of the copyright holder.
+
+All trademarks and registered trademarks mentioned herein are the property of their respective owners.
+*/
+
+const uint8_t lucy_StrHelp_UTF8_COUNT[] = {
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+
+ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+ 4, 4, 4, 4, 4,
+ 4, 4, 4, /* illegal in Unicode */
+ 5, 5, 5, 5, /* illegal in Unicode */
+ 6, 6, /* illegal in Unicode */
+ 7, 7 /* illegal bytes 0xfe and 0xff */
+};
+
diff --git a/core/Lucy/Util/StringHelper3.c b/core/Lucy/Util/StringHelper3.c
new file mode 100644
index 0000000..154d5e7
--- /dev/null
+++ b/core/Lucy/Util/StringHelper3.c
@@ -0,0 +1,90 @@
+#include "charmony.h"
+#include "Lucy/Util/StringHelper.h"
+
+// The list of code points in this function is chosen based on the White_Space
+// property in http://www.unicode.org/Public/UNIDATA/PropList.txt
+
+/*
+
+EXHIBIT 1 UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE
+
+Unicode Data Files include all data files under the directories
+http://www.unicode.org/Public/, http://www.unicode.org/reports/, and
+http://www.unicode.org/cldr/data/ . Unicode Software includes any source code
+published in the Unicode Standard or under the directories
+http://www.unicode.org/Public/, http://www.unicode.org/reports/, and
+http://www.unicode.org/cldr/data/.
+
+NOTICE TO USER: Carefully read the following legal agreement. BY DOWNLOADING,
+INSTALLING, COPYING OR OTHERWISE USING UNICODE INC.'S DATA FILES ("DATA
+FILES"), AND/OR SOFTWARE ("SOFTWARE"), YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO
+BE BOUND BY, ALL OF THE TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT
+AGREE, DO NOT DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR
+SOFTWARE.
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright 1991-2010 Unicode, Inc. All rights reserved. Distributed under the
+Terms of Use in http://www.unicode.org/copyright.html.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+the Unicode data files and any associated documentation (the "Data Files") or
+Unicode software and any associated documentation (the "Software") to deal in
+the Data Files or Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, and/or sell copies
+of the Data Files or Software, and to permit persons to whom the Data Files or
+Software are furnished to do so, provided that (a) the above copyright
+notice(s) and this permission notice appear with all copies of the Data Files
+or Software, (b) both the above copyright notice(s) and this permission notice
+appear in associated documentation, and (c) there is clear notice in each
+modified Data File or in the Software as well as in the documentation
+associated with the Data File(s) or Software that the data or software has been
+modified.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD
+PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN
+THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA FILES OR
+SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder shall not be
+used in advertising or otherwise to promote the sale, use or other dealings in
+these Data Files or Software without prior written authorization of the
+copyright holder.
+
+Unicode and the Unicode logo are trademarks of Unicode, Inc., and may be
+registered in some jurisdictions. All other trademarks and registered
+trademarks mentioned herein are the property of their respective owners.
+
+*/
+
+chy_bool_t
+lucy_StrHelp_is_whitespace(uint32_t code_point) {
+ switch (code_point) {
+ // <control-0009>..<control-000D>
+ case 0x0009: case 0x000A: case 0x000B: case 0x000C: case 0x000D:
+ case 0x0020: // SPACE
+ case 0x0085: // <control-0085>
+ case 0x00A0: // NO-BREAK SPACE
+ case 0x1680: // OGHAM SPACE MARK
+ case 0x180E: // MONGOLIAN VOWEL SEPARATOR
+ // EN QUAD..HAIR SPACE
+ case 0x2000: case 0x2001: case 0x2002: case 0x2003: case 0x2004:
+ case 0x2005: case 0x2006: case 0x2007: case 0x2008: case 0x2009:
+ case 0x200A:
+ case 0x2028: // LINE SEPARATOR
+ case 0x2029: // PARAGRAPH SEPARATOR
+ case 0x202F: // NARROW NO-BREAK SPACE
+ case 0x205F: // MEDIUM MATHEMATICAL SPACE
+ case 0x3000: // IDEOGRAPHIC SPACE
+ return true;
+
+ default:
+ return false;
+ }
+}
+
diff --git a/core/Lucy/Util/ToolSet.h b/core/Lucy/Util/ToolSet.h
new file mode 100644
index 0000000..453873e
--- /dev/null
+++ b/core/Lucy/Util/ToolSet.h
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef H_CFISH_TOOLSET
+#define H_CFISH_TOOLSET 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** ToolSet groups together several commonly used header files, so that only
+ * one pound-include directive is needed for them.
+ *
+ * It should only be used internally, and only included in C files rather than
+ * header files, so that the header files remain as sparse as possible.
+ */
+
+#define LUCY_USE_SHORT_NAMES
+#define CHY_USE_SHORT_NAMES
+
+#include "charmony.h"
+#include <limits.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include "Lucy/Object/Obj.h"
+#include "Lucy/Object/BitVector.h"
+#include "Lucy/Object/ByteBuf.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/Hash.h"
+#include "Lucy/Object/I32Array.h"
+#include "Lucy/Object/Num.h"
+#include "Lucy/Object/VArray.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Util/NumberUtils.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/StringHelper.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // H_CFISH_TOOLSET
+
+
diff --git a/core/LucyX/Search/FilterMatcher.c b/core/LucyX/Search/FilterMatcher.c
new file mode 100644
index 0000000..a510ca9
--- /dev/null
+++ b/core/LucyX/Search/FilterMatcher.c
@@ -0,0 +1,76 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_FILTERMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "LucyX/Search/FilterMatcher.h"
+
+FilterMatcher*
+FilterMatcher_new(BitVector *bits, int32_t doc_max) {
+ FilterMatcher *self = (FilterMatcher*)VTable_Make_Obj(FILTERMATCHER);
+ return FilterMatcher_init(self, bits, doc_max);
+}
+
+FilterMatcher*
+FilterMatcher_init(FilterMatcher *self, BitVector *bits, int32_t doc_max) {
+ Matcher_init((Matcher*)self);
+
+ // Init.
+ self->doc_id = 0;
+
+ // Assign.
+ self->bits = (BitVector*)INCREF(bits);
+ self->doc_max = doc_max;
+
+ return self;
+}
+
+void
+FilterMatcher_destroy(FilterMatcher *self) {
+ DECREF(self->bits);
+ SUPER_DESTROY(self, FILTERMATCHER);
+}
+
+int32_t
+FilterMatcher_next(FilterMatcher* self) {
+ do {
+ if (++self->doc_id > self->doc_max) {
+ self->doc_id--;
+ return 0;
+ }
+ } while (!BitVec_Get(self->bits, self->doc_id));
+ return self->doc_id;
+}
+
+int32_t
+FilterMatcher_skip_to(FilterMatcher* self, int32_t target) {
+ self->doc_id = target - 1;
+ return FilterMatcher_next(self);
+}
+
+float
+FilterMatcher_score(FilterMatcher* self) {
+ UNUSED_VAR(self);
+ return 0.0f;
+}
+
+int32_t
+FilterMatcher_get_doc_id(FilterMatcher* self) {
+ return self->doc_id;
+}
+
+
diff --git a/core/LucyX/Search/FilterMatcher.cfh b/core/LucyX/Search/FilterMatcher.cfh
new file mode 100644
index 0000000..1232e72
--- /dev/null
+++ b/core/LucyX/Search/FilterMatcher.cfh
@@ -0,0 +1,52 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class LucyX::Search::FilterMatcher inherits Lucy::Search::Matcher {
+
+ BitVector *bits;
+ int32_t doc_max;
+ int32_t doc_id;
+
+ /**
+ * @param bits A BitVector with each doc id that should match set to
+ * true.
+ * @param doc_max The largest doc id that could possibly match.
+ */
+ inert incremented FilterMatcher*
+ new(BitVector *bits, int32_t doc_max);
+
+ inert FilterMatcher*
+ init(FilterMatcher *self, BitVector *bits, int32_t doc_max);
+
+ public void
+ Destroy(FilterMatcher *self);
+
+ public int32_t
+ Next(FilterMatcher* self);
+
+ public int32_t
+ Skip_To(FilterMatcher* self, int32_t target);
+
+ public float
+ Score(FilterMatcher* self);
+
+ public int32_t
+ Get_Doc_ID(FilterMatcher* self);
+}
+
+
diff --git a/core/LucyX/Search/MockMatcher.c b/core/LucyX/Search/MockMatcher.c
new file mode 100644
index 0000000..9153cae
--- /dev/null
+++ b/core/LucyX/Search/MockMatcher.c
@@ -0,0 +1,68 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_MOCKMATCHER
+#include "Lucy/Util/ToolSet.h"
+
+#include "LucyX/Search/MockMatcher.h"
+
+MockMatcher*
+MockMatcher_new(I32Array *doc_ids, ByteBuf *scores) {
+ MockMatcher *self = (MockMatcher*)VTable_Make_Obj(MOCKMATCHER);
+ return MockMatcher_init(self, doc_ids, scores);
+}
+
+MockMatcher*
+MockMatcher_init(MockMatcher *self, I32Array *doc_ids, ByteBuf *scores) {
+ Matcher_init((Matcher*)self);
+ self->tick = -1;
+ self->size = I32Arr_Get_Size(doc_ids);
+ self->doc_ids = (I32Array*)INCREF(doc_ids);
+ self->scores = (ByteBuf*)INCREF(scores);
+ return self;
+}
+
+void
+MockMatcher_destroy(MockMatcher *self) {
+ DECREF(self->doc_ids);
+ DECREF(self->scores);
+ SUPER_DESTROY(self, MOCKMATCHER);
+}
+
+int32_t
+MockMatcher_next(MockMatcher* self) {
+ if (++self->tick >= (int32_t)self->size) {
+ self->tick--;
+ return 0;
+ }
+ return I32Arr_Get(self->doc_ids, self->tick);
+}
+
+float
+MockMatcher_score(MockMatcher* self) {
+ if (!self->scores) {
+ THROW(ERR, "Can't call Score() unless scores supplied");
+ }
+ float *raw_scores = (float*)BB_Get_Buf(self->scores);
+ return raw_scores[self->tick];
+}
+
+int32_t
+MockMatcher_get_doc_id(MockMatcher* self) {
+ return I32Arr_Get(self->doc_ids, self->tick);
+}
+
+
diff --git a/core/LucyX/Search/MockMatcher.cfh b/core/LucyX/Search/MockMatcher.cfh
new file mode 100644
index 0000000..27c23c6
--- /dev/null
+++ b/core/LucyX/Search/MockMatcher.cfh
@@ -0,0 +1,50 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+class LucyX::Search::MockMatcher inherits Lucy::Search::Matcher {
+
+ size_t size;
+ I32Array *doc_ids;
+ ByteBuf *scores;
+ int32_t tick;
+
+ inert incremented MockMatcher*
+ new(I32Array *doc_ids, ByteBuf *scores = NULL);
+
+ /**
+ * @param doc_ids An array of matching doc ids.
+ * @param scores Float scores corresponding to the doc ids array. If not
+ * supplied, calling Score() will throw an exception.
+ */
+ inert incremented MockMatcher*
+ init(MockMatcher *self, I32Array *doc_ids, ByteBuf *scores = NULL);
+
+ public void
+ Destroy(MockMatcher *self);
+
+ public int32_t
+ Next(MockMatcher* self);
+
+ public float
+ Score(MockMatcher* self);
+
+ public int32_t
+ Get_Doc_ID(MockMatcher* self);
+}
+
+
diff --git a/core/LucyX/Search/ProximityMatcher.c b/core/LucyX/Search/ProximityMatcher.c
new file mode 100644
index 0000000..92f9162
--- /dev/null
+++ b/core/LucyX/Search/ProximityMatcher.c
@@ -0,0 +1,324 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_PROXIMITYMATCHER
+#define C_LUCY_POSTING
+#define C_LUCY_SCOREPOSTING
+#include "Lucy/Util/ToolSet.h"
+
+#include "LucyX/Search/ProximityMatcher.h"
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Search/Compiler.h"
+
+
+ProximityMatcher*
+ProximityMatcher_new(Similarity *sim, VArray *plists, Compiler *compiler,
+ uint32_t within) {
+ ProximityMatcher *self =
+ (ProximityMatcher*)VTable_Make_Obj(PROXIMITYMATCHER);
+ return ProximityMatcher_init(self, sim, plists, compiler, within);
+
+}
+
+ProximityMatcher*
+ProximityMatcher_init(ProximityMatcher *self, Similarity *similarity,
+ VArray *plists, Compiler *compiler, uint32_t within) {
+ Matcher_init((Matcher*)self);
+
+ // Init.
+ self->anchor_set = BB_new(0);
+ self->proximity_freq = 0.0;
+ self->proximity_boost = 0.0;
+ self->first_time = true;
+ self->more = true;
+ self->within = within;
+
+ // Extract PostingLists out of VArray into local C array for quick access.
+ self->num_elements = VA_Get_Size(plists);
+ self->plists = (PostingList**)MALLOCATE(
+ self->num_elements * sizeof(PostingList*));
+ for (size_t i = 0; i < self->num_elements; i++) {
+ PostingList *const plist
+ = (PostingList*)CERTIFY(VA_Fetch(plists, i), POSTINGLIST);
+ if (plist == NULL) {
+ THROW(ERR, "Missing element %u32", i);
+ }
+ self->plists[i] = (PostingList*)INCREF(plist);
+ }
+
+ // Assign.
+ self->sim = (Similarity*)INCREF(similarity);
+ self->compiler = (Compiler*)INCREF(compiler);
+ self->weight = Compiler_Get_Weight(compiler);
+
+ return self;
+}
+
+void
+ProximityMatcher_destroy(ProximityMatcher *self) {
+ if (self->plists) {
+ for (size_t i = 0; i < self->num_elements; i++) {
+ DECREF(self->plists[i]);
+ }
+ FREEMEM(self->plists);
+ }
+ DECREF(self->sim);
+ DECREF(self->anchor_set);
+ DECREF(self->compiler);
+ SUPER_DESTROY(self, PROXIMITYMATCHER);
+}
+
+int32_t
+ProximityMatcher_next(ProximityMatcher *self) {
+ if (self->first_time) {
+ return ProximityMatcher_Advance(self, 1);
+ }
+ else if (self->more) {
+ const int32_t target = PList_Get_Doc_ID(self->plists[0]) + 1;
+ return ProximityMatcher_Advance(self, target);
+ }
+ else {
+ return 0;
+ }
+}
+
+int32_t
+ProximityMatcher_advance(ProximityMatcher *self, int32_t target) {
+ PostingList **const plists = self->plists;
+ const uint32_t num_elements = self->num_elements;
+ int32_t highest = 0;
+
+ // Reset match variables to indicate no match. New values will be
+ // assigned if a match succeeds.
+ self->proximity_freq = 0.0;
+ self->doc_id = 0;
+
+ // Find the lowest possible matching doc ID greater than the current doc
+ // ID. If any one of the PostingLists is exhausted, we're done.
+ if (self->first_time) {
+ self->first_time = false;
+
+ // On the first call to Advance(), advance all PostingLists.
+ for (size_t i = 0, max = self->num_elements; i < max; i++) {
+ int32_t candidate = PList_Advance(plists[i], target);
+ if (!candidate) {
+ self->more = false;
+ return 0;
+ }
+ else if (candidate > highest) {
+ // Remember the highest doc ID so far.
+ highest = candidate;
+ }
+ }
+ }
+ else {
+ // On subsequent iters, advance only one PostingList. Its new doc ID
+ // becomes the minimum target which all the others must move up to.
+ highest = PList_Advance(plists[0], target);
+ if (highest == 0) {
+ self->more = false;
+ return 0;
+ }
+ }
+
+ // Find a doc which contains all the terms.
+ while (1) {
+ bool_t agreement = true;
+
+ // Scoot all posting lists up to at least the current minimum.
+ for (uint32_t i = 0; i < num_elements; i++) {
+ PostingList *const plist = plists[i];
+ int32_t candidate = PList_Get_Doc_ID(plist);
+
+ // Is this PostingList already beyond the minimum? Then raise the
+ // bar for everyone else.
+ if (highest < candidate) { highest = candidate; }
+ if (target < highest) { target = highest; }
+
+ // Scoot this posting list up.
+ if (candidate < target) {
+ candidate = PList_Advance(plist, target);
+
+ // If this PostingList is exhausted, we're done.
+ if (candidate == 0) {
+ self->more = false;
+ return 0;
+ }
+
+ // After calling PList_Advance(), we are guaranteed to be
+ // either at or beyond the minimum, so we can assign without
+ // checking and the minumum will either go up or stay the
+ // same.
+ highest = candidate;
+ }
+ }
+
+ // See whether all the PostingLists have managed to converge on a
+ // single doc ID.
+ for (uint32_t i = 0; i < num_elements; i++) {
+ const int32_t candidate = PList_Get_Doc_ID(plists[i]);
+ if (candidate != highest) { agreement = false; }
+ }
+
+ // If we've found a doc with all terms in it, see if they form a
+ // phrase.
+ if (agreement && highest >= target) {
+ self->proximity_freq = ProximityMatcher_Calc_Proximity_Freq(self);
+ if (self->proximity_freq == 0.0) {
+ // No phrase. Move on to another doc.
+ target += 1;
+ }
+ else {
+ // Success!
+ self->doc_id = highest;
+ return highest;
+ }
+ }
+ }
+}
+
+
+static INLINE uint32_t
+SI_winnow_anchors(uint32_t *anchors_start, const uint32_t *const anchors_end,
+ const uint32_t *candidates, const uint32_t *const candidates_end,
+ uint32_t offset, uint32_t within) {
+ uint32_t *anchors = anchors_start;
+ uint32_t *anchors_found = anchors_start;
+ uint32_t target_anchor;
+ uint32_t target_candidate;
+
+ // Safety check, so there's no chance of a bad dereference.
+ if (anchors_start == anchors_end || candidates == candidates_end) {
+ return 0;
+ }
+
+ /* This function is a loop that finds terms that can continue a phrase.
+ * It overwrites the anchors in place, and returns the number remaining.
+ * The basic algorithm is to alternately increment the candidates' pointer
+ * until it is at or beyond its target position, and then increment the
+ * anchors' pointer until it is at or beyond its target. The non-standard
+ * form is to avoid unnecessary comparisons. This loop has not been
+ * tested for speed, but glancing at the object code produced (objdump -S)
+ * it appears to be significantly faster than the nested loop alternative.
+ * But given the vagaries of modern processors, it merits actual
+ * testing.*/
+
+SPIN_CANDIDATES:
+ target_candidate = *anchors + offset;
+ while (*candidates < target_candidate) {
+ if (++candidates == candidates_end) { goto DONE; }
+ }
+ if ((*candidates - target_candidate) < within) { goto MATCH; }
+ goto SPIN_ANCHORS;
+
+SPIN_ANCHORS:
+ target_anchor = *candidates - offset;
+ while (*anchors < target_anchor) {
+ if (++anchors == anchors_end) { goto DONE; }
+ };
+ if (*anchors == target_anchor) { goto MATCH; }
+ goto SPIN_CANDIDATES;
+
+MATCH:
+ *anchors_found++ = *anchors;
+ if (++anchors == anchors_end) { goto DONE; }
+ goto SPIN_CANDIDATES;
+
+DONE:
+ // Return number of anchors remaining.
+ return anchors_found - anchors_start;
+}
+
+float
+ProximityMatcher_calc_proximity_freq(ProximityMatcher *self) {
+ PostingList **const plists = self->plists;
+
+ /* Create a overwriteable "anchor set" from the first posting.
+ *
+ * Each "anchor" is a position, measured in tokens, corresponding to a a
+ * term which might start a phrase. We start off with an "anchor set"
+ * comprised of all positions at which the first term in the phrase occurs
+ * in the field.
+ *
+ * There can never be more proximity matches than instances of this first
+ * term. There may be fewer however, which we will determine by seeing
+ * whether all the other terms line up at subsequent position slots.
+ *
+ * Every time we eliminate an anchor from the anchor set, we splice it out
+ * of the array. So if we begin with an anchor set of (15, 51, 72) and we
+ * discover that matches occur at the first and last instances of the
+ * first term but not the middle one, the final array will be (15, 72).
+ *
+ * The number of elements in the anchor set when we are finished winnowing
+ * is our proximity freq.
+ */
+ ScorePosting *posting = (ScorePosting*)PList_Get_Posting(plists[0]);
+ uint32_t anchors_remaining = posting->freq;
+ if (!anchors_remaining) { return 0.0f; }
+
+ size_t amount = anchors_remaining * sizeof(uint32_t);
+ uint32_t *anchors_start = (uint32_t*)BB_Grow(self->anchor_set, amount);
+ uint32_t *anchors_end = anchors_start + anchors_remaining;
+ memcpy(anchors_start, posting->prox, amount);
+
+ // Match the positions of other terms against the anchor set.
+ for (uint32_t i = 1, max = self->num_elements; i < max; i++) {
+ // Get the array of positions for the next term. Unlike the anchor
+ // set (which is a copy), these won't be overwritten.
+ ScorePosting *posting = (ScorePosting*)PList_Get_Posting(plists[i]);
+ uint32_t *candidates_start = posting->prox;
+ uint32_t *candidates_end = candidates_start + posting->freq;
+
+ // Splice out anchors that don't match the next term. Bail out if
+ // we've eliminated all possible anchors.
+ if (self->within == 1) { // exact phrase match
+ anchors_remaining = SI_winnow_anchors(anchors_start, anchors_end,
+ candidates_start,
+ candidates_end, i, 1);
+ }
+ else { // fuzzy-phrase match
+ anchors_remaining = SI_winnow_anchors(anchors_start, anchors_end,
+ candidates_start,
+ candidates_end, i,
+ self->within);
+ }
+ if (!anchors_remaining) { return 0.0f; }
+
+ // Adjust end for number of anchors that remain.
+ anchors_end = anchors_start + anchors_remaining;
+ }
+
+ // The number of anchors left is the proximity freq.
+ return (float)anchors_remaining;
+}
+
+int32_t
+ProximityMatcher_get_doc_id(ProximityMatcher *self) {
+ return self->doc_id;
+}
+
+float
+ProximityMatcher_score(ProximityMatcher *self) {
+ ScorePosting *posting = (ScorePosting*)PList_Get_Posting(self->plists[0]);
+ float score = Sim_TF(self->sim, self->proximity_freq)
+ * self->weight
+ * posting->weight;
+ return score;
+}
+
+
diff --git a/core/LucyX/Search/ProximityMatcher.cfh b/core/LucyX/Search/ProximityMatcher.cfh
new file mode 100644
index 0000000..131b59d
--- /dev/null
+++ b/core/LucyX/Search/ProximityMatcher.cfh
@@ -0,0 +1,66 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Matcher for a ProximityQuery.
+ */
+
+class LucyX::Search::ProximityMatcher inherits Lucy::Search::Matcher {
+
+ int32_t doc_id;
+ uint32_t num_elements;
+ Similarity *sim;
+ PostingList **plists;
+ ByteBuf *anchor_set;
+ float proximity_freq;
+ float proximity_boost;
+ Compiler *compiler;
+ float weight;
+ bool_t first_time;
+ bool_t more;
+ uint32_t within;
+
+ inert incremented ProximityMatcher*
+ new(Similarity *similarity, VArray *posting_lists, Compiler *compiler,
+ uint32_t within);
+
+ inert ProximityMatcher*
+ init(ProximityMatcher *self, Similarity *similarity, VArray *posting_lists,
+ Compiler *compiler, uint32_t within);
+
+ public void
+ Destroy(ProximityMatcher *self);
+
+ public int32_t
+ Next(ProximityMatcher *self);
+
+ public int32_t
+ Advance(ProximityMatcher *self, int32_t target);
+
+ public int32_t
+ Get_Doc_ID(ProximityMatcher *self);
+
+ public float
+ Score(ProximityMatcher *self);
+
+ /** Calculate how often the phrase occurs in the current document.
+ */
+ float
+ Calc_Proximity_Freq(ProximityMatcher *self);
+}
+
+
diff --git a/core/LucyX/Search/ProximityQuery.c b/core/LucyX/Search/ProximityQuery.c
new file mode 100644
index 0000000..721d53d
--- /dev/null
+++ b/core/LucyX/Search/ProximityQuery.c
@@ -0,0 +1,424 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_PROXIMITYQUERY
+#define C_LUCY_PROXIMITYCOMPILER
+#include <stdarg.h>
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "LucyX/Search/ProximityQuery.h"
+#include "Lucy/Index/DocVector.h"
+#include "Lucy/Index/Posting.h"
+#include "Lucy/Index/Posting/ScorePosting.h"
+#include "Lucy/Index/PostingList.h"
+#include "Lucy/Index/PostingListReader.h"
+#include "Lucy/Index/SegPostingList.h"
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Index/Similarity.h"
+#include "Lucy/Index/TermVector.h"
+#include "Lucy/Plan/Schema.h"
+#include "LucyX/Search/ProximityMatcher.h"
+#include "Lucy/Search/Searcher.h"
+#include "Lucy/Search/Span.h"
+#include "Lucy/Search/TermQuery.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Freezer.h"
+
+// Shared initialization routine which assumes that it's ok to assume control
+// over [field] and [terms], eating their refcounts.
+static ProximityQuery*
+S_do_init(ProximityQuery *self, CharBuf *field, VArray *terms, float boost,
+ uint32_t within);
+
+ProximityQuery*
+ProximityQuery_new(const CharBuf *field, VArray *terms, uint32_t within) {
+ ProximityQuery *self = (ProximityQuery*)VTable_Make_Obj(PROXIMITYQUERY);
+ return ProximityQuery_init(self, field, terms, within);
+}
+
+ProximityQuery*
+ProximityQuery_init(ProximityQuery *self, const CharBuf *field, VArray *terms,
+ uint32_t within) {
+ return S_do_init(self, CB_Clone(field), VA_Clone(terms), 1.0f, within);
+}
+
+void
+ProximityQuery_destroy(ProximityQuery *self) {
+ DECREF(self->terms);
+ DECREF(self->field);
+ SUPER_DESTROY(self, PROXIMITYQUERY);
+}
+
+static ProximityQuery*
+S_do_init(ProximityQuery *self, CharBuf *field, VArray *terms, float boost,
+ uint32_t within) {
+ uint32_t i, max;
+ Query_init((Query*)self, boost);
+ for (i = 0, max = VA_Get_Size(terms); i < max; i++) {
+ CERTIFY(VA_Fetch(terms, i), OBJ);
+ }
+ self->field = field;
+ self->terms = terms;
+ self->within = within;
+ return self;
+}
+
+void
+ProximityQuery_serialize(ProximityQuery *self, OutStream *outstream) {
+ OutStream_Write_F32(outstream, self->boost);
+ CB_Serialize(self->field, outstream);
+ VA_Serialize(self->terms, outstream);
+ OutStream_Write_C32(outstream, self->within);
+}
+
+ProximityQuery*
+ProximityQuery_deserialize(ProximityQuery *self, InStream *instream) {
+ float boost = InStream_Read_F32(instream);
+ CharBuf *field = CB_deserialize(NULL, instream);
+ VArray *terms = VA_deserialize(NULL, instream);
+ uint32_t within = InStream_Read_C32(instream);
+ self = self ? self : (ProximityQuery*)VTable_Make_Obj(PROXIMITYQUERY);
+ return S_do_init(self, field, terms, boost, within);
+}
+
+bool_t
+ProximityQuery_equals(ProximityQuery *self, Obj *other) {
+ ProximityQuery *twin = (ProximityQuery*)other;
+ if (twin == self) { return true; }
+ if (!Obj_Is_A(other, PROXIMITYQUERY)) { return false; }
+ if (self->boost != twin->boost) { return false; }
+ if (self->field && !twin->field) { return false; }
+ if (!self->field && twin->field) { return false; }
+ if (self->field && !CB_Equals(self->field, (Obj*)twin->field)) {
+ return false;
+ }
+ if (!VA_Equals(twin->terms, (Obj*)self->terms)) { return false; }
+ if (self->within != twin->within) { return false; }
+ return true;
+}
+
+CharBuf*
+ProximityQuery_to_string(ProximityQuery *self) {
+ uint32_t i;
+ uint32_t num_terms = VA_Get_Size(self->terms);
+ CharBuf *retval = CB_Clone(self->field);
+ CB_Cat_Trusted_Str(retval, ":\"", 2);
+ for (i = 0; i < num_terms; i++) {
+ Obj *term = VA_Fetch(self->terms, i);
+ CharBuf *term_string = Obj_To_String(term);
+ CB_Cat(retval, term_string);
+ DECREF(term_string);
+ if (i < num_terms - 1) {
+ CB_Cat_Trusted_Str(retval, " ", 1);
+ }
+ }
+ CB_Cat_Trusted_Str(retval, "\"", 1);
+ CB_catf(retval, "~%u32", self->within);
+ return retval;
+}
+
+Compiler*
+ProximityQuery_make_compiler(ProximityQuery *self, Searcher *searcher,
+ float boost) {
+ if (VA_Get_Size(self->terms) == 1) {
+ // Optimize for one-term "phrases".
+ Obj *term = VA_Fetch(self->terms, 0);
+ TermQuery *term_query = TermQuery_new(self->field, term);
+ TermCompiler *term_compiler;
+ TermQuery_Set_Boost(term_query, self->boost);
+ term_compiler
+ = (TermCompiler*)TermQuery_Make_Compiler(term_query, searcher,
+ boost);
+ DECREF(term_query);
+ return (Compiler*)term_compiler;
+ }
+ else {
+ return (Compiler*)ProximityCompiler_new(self, searcher, boost,
+ self->within);
+ }
+}
+
+CharBuf*
+ProximityQuery_get_field(ProximityQuery *self) {
+ return self->field;
+}
+
+VArray*
+ProximityQuery_get_terms(ProximityQuery *self) {
+ return self->terms;
+}
+
+uint32_t
+ProximityQuery_get_within(ProximityQuery *self) {
+ return self->within;
+}
+
+/*********************************************************************/
+
+ProximityCompiler*
+ProximityCompiler_new(ProximityQuery *parent, Searcher *searcher, float boost,
+ uint32_t within) {
+ ProximityCompiler *self =
+ (ProximityCompiler*)VTable_Make_Obj(PROXIMITYCOMPILER);
+ return ProximityCompiler_init(self, parent, searcher, boost, within);
+}
+
+ProximityCompiler*
+ProximityCompiler_init(ProximityCompiler *self, ProximityQuery *parent,
+ Searcher *searcher, float boost, uint32_t within) {
+ Schema *schema = Searcher_Get_Schema(searcher);
+ Similarity *sim = Schema_Fetch_Sim(schema, parent->field);
+ VArray *terms = parent->terms;
+ uint32_t i, max;
+
+ self->within = within;
+
+ // Try harder to find a Similarity if necessary.
+ if (!sim) { sim = Schema_Get_Similarity(schema); }
+
+ // Init.
+ Compiler_init((Compiler*)self, (Query*)parent, searcher, sim, boost);
+
+ // Store IDF for the phrase.
+ self->idf = 0;
+ for (i = 0, max = VA_Get_Size(terms); i < max; i++) {
+ Obj *term = VA_Fetch(terms, i);
+ int32_t doc_max = Searcher_Doc_Max(searcher);
+ int32_t doc_freq = Searcher_Doc_Freq(searcher, parent->field, term);
+ self->idf += Sim_IDF(sim, doc_freq, doc_max);
+ }
+
+ // Calculate raw weight.
+ self->raw_weight = self->idf * self->boost;
+
+ // Make final preparations.
+ ProximityCompiler_Normalize(self);
+
+ return self;
+}
+
+void
+ProximityCompiler_serialize(ProximityCompiler *self, OutStream *outstream) {
+ Compiler_serialize((Compiler*)self, outstream);
+ OutStream_Write_F32(outstream, self->idf);
+ OutStream_Write_F32(outstream, self->raw_weight);
+ OutStream_Write_F32(outstream, self->query_norm_factor);
+ OutStream_Write_F32(outstream, self->normalized_weight);
+ OutStream_Write_C32(outstream, self->within);
+}
+
+ProximityCompiler*
+ProximityCompiler_deserialize(ProximityCompiler *self, InStream *instream) {
+ self = self ? self : (ProximityCompiler*)VTable_Make_Obj(PROXIMITYCOMPILER);
+ Compiler_deserialize((Compiler*)self, instream);
+ self->idf = InStream_Read_F32(instream);
+ self->raw_weight = InStream_Read_F32(instream);
+ self->query_norm_factor = InStream_Read_F32(instream);
+ self->normalized_weight = InStream_Read_F32(instream);
+ self->within = InStream_Read_C32(instream);
+ return self;
+}
+
+bool_t
+ProximityCompiler_equals(ProximityCompiler *self, Obj *other) {
+ ProximityCompiler *twin = (ProximityCompiler*)other;
+ if (!Obj_Is_A(other, PROXIMITYCOMPILER)) { return false; }
+ if (!Compiler_equals((Compiler*)self, other)) { return false; }
+ if (self->idf != twin->idf) { return false; }
+ if (self->raw_weight != twin->raw_weight) { return false; }
+ if (self->query_norm_factor != twin->query_norm_factor) { return false; }
+ if (self->normalized_weight != twin->normalized_weight) { return false; }
+ if (self->within != twin->within) { return false; }
+ return true;
+}
+
+float
+ProximityCompiler_get_weight(ProximityCompiler *self) {
+ return self->normalized_weight;
+}
+
+float
+ProximityCompiler_sum_of_squared_weights(ProximityCompiler *self) {
+ return self->raw_weight * self->raw_weight;
+}
+
+void
+ProximityCompiler_apply_norm_factor(ProximityCompiler *self, float factor) {
+ self->query_norm_factor = factor;
+ self->normalized_weight = self->raw_weight * self->idf * factor;
+}
+
+Matcher*
+ProximityCompiler_make_matcher(ProximityCompiler *self, SegReader *reader,
+ bool_t need_score) {
+ UNUSED_VAR(need_score);
+ ProximityQuery *const parent = (ProximityQuery*)self->parent;
+ VArray *const terms = parent->terms;
+ uint32_t num_terms = VA_Get_Size(terms);
+
+ // Bail if there are no terms.
+ if (!num_terms) { return NULL; }
+
+ // Bail unless field is valid and posting type supports positions.
+ Similarity *sim = ProximityCompiler_Get_Similarity(self);
+ Posting *posting = Sim_Make_Posting(sim);
+ if (posting == NULL || !Obj_Is_A((Obj*)posting, SCOREPOSTING)) {
+ DECREF(posting);
+ return NULL;
+ }
+ DECREF(posting);
+
+ // Bail if there's no PostingListReader for this segment.
+ PostingListReader *const plist_reader
+ = (PostingListReader*)SegReader_Fetch(
+ reader, VTable_Get_Name(POSTINGLISTREADER));
+ if (!plist_reader) { return NULL; }
+
+ // Look up each term.
+ VArray *plists = VA_new(num_terms);
+ for (uint32_t i = 0; i < num_terms; i++) {
+ Obj *term = VA_Fetch(terms, i);
+ PostingList *plist
+ = PListReader_Posting_List(plist_reader, parent->field, term);
+
+ // Bail if any one of the terms isn't in the index.
+ if (!plist || !PList_Get_Doc_Freq(plist)) {
+ DECREF(plist);
+ DECREF(plists);
+ return NULL;
+ }
+ VA_Push(plists, (Obj*)plist);
+ }
+
+ Matcher *retval
+ = (Matcher*)ProximityMatcher_new(sim, plists, (Compiler*)self, self->within);
+ DECREF(plists);
+ return retval;
+}
+
+VArray*
+ProximityCompiler_highlight_spans(ProximityCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field) {
+ ProximityQuery *const parent = (ProximityQuery*)self->parent;
+ VArray *const terms = parent->terms;
+ VArray *const spans = VA_new(0);
+ VArray *term_vectors;
+ BitVector *posit_vec;
+ BitVector *other_posit_vec;
+ uint32_t i;
+ const uint32_t num_terms = VA_Get_Size(terms);
+ uint32_t num_tvs;
+ UNUSED_VAR(searcher);
+
+ // Bail if no terms or field doesn't match.
+ if (!num_terms) { return spans; }
+ if (!CB_Equals(field, (Obj*)parent->field)) { return spans; }
+
+ term_vectors = VA_new(num_terms);
+ posit_vec = BitVec_new(0);
+ other_posit_vec = BitVec_new(0);
+ for (i = 0; i < num_terms; i++) {
+ Obj *term = VA_Fetch(terms, i);
+ TermVector *term_vector
+ = DocVec_Term_Vector(doc_vec, field, (CharBuf*)term);
+
+ // Bail if any term is missing.
+ if (!term_vector) {
+ break;
+ }
+
+ VA_Push(term_vectors, (Obj*)term_vector);
+
+ if (i == 0) {
+ // Set initial positions from first term.
+ uint32_t j;
+ I32Array *positions = TV_Get_Positions(term_vector);
+ for (j = I32Arr_Get_Size(positions); j > 0; j--) {
+ BitVec_Set(posit_vec, I32Arr_Get(positions, j - 1));
+ }
+ }
+ else {
+ // Filter positions using logical "and".
+ uint32_t j;
+ I32Array *positions = TV_Get_Positions(term_vector);
+
+ BitVec_Clear_All(other_posit_vec);
+ for (j = I32Arr_Get_Size(positions); j > 0; j--) {
+ int32_t pos = I32Arr_Get(positions, j - 1) - i;
+ if (pos >= 0) {
+ BitVec_Set(other_posit_vec, pos);
+ }
+ }
+ BitVec_And(posit_vec, other_posit_vec);
+ }
+ }
+
+ // Proceed only if all terms are present.
+ num_tvs = VA_Get_Size(term_vectors);
+ if (num_tvs == num_terms) {
+ TermVector *first_tv = (TermVector*)VA_Fetch(term_vectors, 0);
+ TermVector *last_tv
+ = (TermVector*)VA_Fetch(term_vectors, num_tvs - 1);
+ I32Array *tv_start_positions = TV_Get_Positions(first_tv);
+ I32Array *tv_end_positions = TV_Get_Positions(last_tv);
+ I32Array *tv_start_offsets = TV_Get_Start_Offsets(first_tv);
+ I32Array *tv_end_offsets = TV_Get_End_Offsets(last_tv);
+ uint32_t terms_max = num_terms - 1;
+ I32Array *valid_posits = BitVec_To_Array(posit_vec);
+ uint32_t num_valid_posits = I32Arr_Get_Size(valid_posits);
+ uint32_t j = 0;
+ uint32_t posit_tick;
+ float weight = ProximityCompiler_Get_Weight(self);
+ i = 0;
+
+ // Add only those starts/ends that belong to a valid position.
+ for (posit_tick = 0; posit_tick < num_valid_posits; posit_tick++) {
+ int32_t valid_start_posit = I32Arr_Get(valid_posits, posit_tick);
+ int32_t valid_end_posit = valid_start_posit + terms_max;
+ int32_t start_offset = 0, end_offset = 0;
+ uint32_t max;
+
+ for (max = I32Arr_Get_Size(tv_start_positions); i < max; i++) {
+ if (I32Arr_Get(tv_start_positions, i) == valid_start_posit) {
+ start_offset = I32Arr_Get(tv_start_offsets, i);
+ break;
+ }
+ }
+ for (max = I32Arr_Get_Size(tv_end_positions); j < max; j++) {
+ if (I32Arr_Get(tv_end_positions, j) == valid_end_posit) {
+ end_offset = I32Arr_Get(tv_end_offsets, j);
+ break;
+ }
+ }
+
+ VA_Push(spans, (Obj*)Span_new(start_offset,
+ end_offset - start_offset, weight));
+
+ i++, j++;
+ }
+
+ DECREF(valid_posits);
+ }
+
+ DECREF(other_posit_vec);
+ DECREF(posit_vec);
+ DECREF(term_vectors);
+ return spans;
+}
+
+
diff --git a/core/LucyX/Search/ProximityQuery.cfh b/core/LucyX/Search/ProximityQuery.cfh
new file mode 100644
index 0000000..349477e
--- /dev/null
+++ b/core/LucyX/Search/ProximityQuery.cfh
@@ -0,0 +1,118 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+parcel Lucy;
+
+/** Query matching an ordered list of terms.
+ *
+ * ProximityQuery is a subclass of L<Lucy::Search::Query> for matching
+ * against an ordered sequence of terms.
+ */
+
+class LucyX::Search::ProximityQuery inherits Lucy::Search::Query
+ : dumpable {
+
+ CharBuf *field;
+ VArray *terms;
+ uint32_t within;
+
+ inert incremented ProximityQuery*
+ new(const CharBuf *field, VArray *terms, uint32_t within);
+
+ /**
+ * @param field The field that the phrase must occur in.
+ * @param terms The ordered array of terms that must match.
+ */
+ public inert ProximityQuery*
+ init(ProximityQuery *self, const CharBuf *field, VArray *terms, uint32_t within);
+
+ /** Accessor for object's field attribute.
+ */
+ public CharBuf*
+ Get_Field(ProximityQuery *self);
+
+ /** Accessor for object's array of terms.
+ */
+ public VArray*
+ Get_Terms(ProximityQuery *self);
+
+ /** Accessor for object's within attribute.
+ */
+ public uint32_t
+ Get_Within(ProximityQuery *self);
+
+ public incremented Compiler*
+ Make_Compiler(ProximityQuery *self, Searcher *searcher, float boost);
+
+ public bool_t
+ Equals(ProximityQuery *self, Obj *other);
+
+ public incremented CharBuf*
+ To_String(ProximityQuery *self);
+
+ public void
+ Serialize(ProximityQuery *self, OutStream *outstream);
+
+ public incremented ProximityQuery*
+ Deserialize(ProximityQuery *self, InStream *instream);
+
+ public void
+ Destroy(ProximityQuery *self);
+}
+
+class LucyX::Search::ProximityCompiler
+ inherits Lucy::Search::Compiler {
+
+ float idf;
+ float raw_weight;
+ float query_norm_factor;
+ float normalized_weight;
+ uint32_t within;
+
+ inert incremented ProximityCompiler*
+ new(ProximityQuery *parent, Searcher *searcher, float boost, uint32_t within);
+
+ inert ProximityCompiler*
+ init(ProximityCompiler *self, ProximityQuery *parent, Searcher *searcher,
+ float boost, uint32_t within);
+
+ public incremented nullable Matcher*
+ Make_Matcher(ProximityCompiler *self, SegReader *reader, bool_t need_score);
+
+ public float
+ Get_Weight(ProximityCompiler *self);
+
+ public float
+ Sum_Of_Squared_Weights(ProximityCompiler *self);
+
+ public void
+ Apply_Norm_Factor(ProximityCompiler *self, float factor);
+
+ public incremented VArray*
+ Highlight_Spans(ProximityCompiler *self, Searcher *searcher,
+ DocVector *doc_vec, const CharBuf *field);
+
+ public bool_t
+ Equals(ProximityCompiler *self, Obj *other);
+
+ public void
+ Serialize(ProximityCompiler *self, OutStream *outstream);
+
+ public incremented ProximityCompiler*
+ Deserialize(ProximityCompiler *self, InStream *instream);
+}
+
+
diff --git a/devel/benchmarks/README.txt b/devel/benchmarks/README.txt
new file mode 100755
index 0000000..9746350
--- /dev/null
+++ b/devel/benchmarks/README.txt
@@ -0,0 +1,39 @@
+Indexing Benchmarks
+
+The purpose of this experiment is to test raw indexing speed, using
+Reuters-21578, Distribution 1.0 as a test corpus. As of this writing,
+Reuters-21578 is available at:
+
+ http://www.daviddlewis.com/resources/testcollections/reuters21578
+
+The corpus comes packaged in SGML, which means we need to preprocess it so
+that our results are not infected by differences between SGML parsers. A
+simple perl script, "./extract_reuters.plx" is supplied, which expands the
+Reuters articles out into the file system, 1 article per file, with the title
+as the first line of text. It takes one command line argument: the location
+of the un-tarred Reuters collection.
+
+ ./extract_reuters.plx /path/to/reuters_collection
+
+Filepaths are hard-coded, and the assumption is that the apps will be run from
+within the benchmarks/ directory. Each of the indexing apps takes four
+optional command line arguments:
+
+ * The number of documents to index.
+ * The number of times to repeat the indexing process.
+ * The increment, or number of docs to add during each index writer instance.
+ * Whether or not the main text should be stored and highlightable.
+
+ $ perl -Mblib indexers/lucy_indexer.plx \
+ > --docs=1000 --reps=6 --increment=10 --store=1
+
+ $ java -server -Xmx500M -XX:CompileThreshold=100 LuceneIndexer \
+ > -docs 1000 -reps 6 -increment 10 -store 1
+
+If no command line args are supplied, the apps will index the entire 19043
+article collection once, using a single index writer, and will neither store
+nor vectorize the main text.
+
+Upon finishing, each app will produce a "truncated mean" report: the slowest
+25% and fastest 25% of reps will be discarded, and the rest will be averaged.
+
diff --git a/devel/benchmarks/extract_reuters.plx b/devel/benchmarks/extract_reuters.plx
new file mode 100755
index 0000000..744afae
--- /dev/null
+++ b/devel/benchmarks/extract_reuters.plx
@@ -0,0 +1,130 @@
+#!/usr/local/bin/perl
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use File::Spec::Functions qw( catfile catdir );
+use Cwd qw( getcwd );
+
+# Ensure call from correct location and with required arg.
+my $source_dir = $ARGV[0];
+die "Usage: ./extract_reuters.plx /path/to/expanded/archive"
+ unless -d $source_dir;
+my $working_dir = getcwd;
+die "Must be run from the benchmarks/ directory"
+ unless ( $working_dir =~ /benchmarks\W*$/ );
+
+# Create the main output directory.
+my $main_out_dir = 'extracted_corpus';
+if ( !-d $main_out_dir ) {
+ mkdir $main_out_dir or die "Couldn't mkdir '$main_out_dir': $!";
+}
+
+# Get a list of the sgm files.
+opendir SOURCE_DIR, $source_dir or die "Couldn't open directory: $!";
+my @sgm_files = grep {/\.sgm$/} readdir SOURCE_DIR;
+closedir SOURCE_DIR or die "Couldn't close directory: $!";
+die "Couldn't find all the sgm files"
+ unless @sgm_files == 22;
+
+# Track number of story docs.
+my $num_files = 0;
+
+for my $sgm_file (@sgm_files) {
+ # Get the sgm file.
+ my $sgm_filepath = catfile( $source_dir, $sgm_file );
+ print "Processing $sgm_filepath\n";
+ open( my $sgm_fh, '<', $sgm_filepath )
+ or die "Couldn't open file '$sgm_filepath': $!";
+
+ # Prepare output directory.
+ $sgm_file =~ /(\d+)\.sgm$/ or die "no match";
+ my $out_dir = catdir( $main_out_dir, "articles$1" );
+ if ( !-d $out_dir ) {
+ mkdir $out_dir or die "Couldn't create directory '$out_dir': $!";
+ }
+
+ my $in_body = 0;
+ my $in_title = 0;
+ my ( $title, $body );
+ while (<$sgm_fh>) {
+ # Start a new story doc.
+ if (/<REUTERS/) {
+ $title = '';
+ $body = '';
+ }
+
+ # Extract title and body.
+ if (s/.*?<TITLE>//) {
+ $in_title = 1;
+ $title = '';
+ }
+ $title .= $_ if $in_title;
+ if (s/.*?<BODY>//) {
+ $in_body = 1;
+ $body = '';
+ }
+ $body .= $_ if $in_body;
+ if (m#</TITLE>.*#) {
+ $in_title = 0;
+ $title =~ s#</TITLE>.*##s;
+ }
+ if (m#</BODY>.*#) {
+ $in_body = 0;
+ $body =~ s#</BODY>.*##s;
+ }
+
+ # Write out a finished article doc.
+ if (m#</REUTERS>#) {
+ die "Malformed data" if ( $in_title or $in_body );
+ if ( length $title and length $body ) {
+ my $out_filename = sprintf( "article%05d.txt", $num_files );
+ my $out_filepath = catfile( $out_dir, $out_filename );
+ open( my $out_fh, '>', $out_filepath )
+ or die "Couldn't open '$out_filepath' for writing: $!";
+ $title =~ s/^\s*//;
+ $title =~ s/\s*$//;
+ print $out_fh "$title\n\n" or die "print failed: $!";
+ print $out_fh $body or die "print failed: $!";
+ close $out_fh or die "Couldn't close '$out_filepath': $!";
+ $num_files++;
+ }
+ }
+ }
+}
+
+print "Total articles extracted: $num_files\n";
+
+__END__
+
+=head1 NAME
+
+extract_reuters.plx - parse Reuters 21578 corpus into individual files
+
+=head1 SYNOPSIS
+
+ ./extract_reuters.plx /path/to/expanded/reuters/archive
+
+=head1 DESCRIPTION
+
+This script will extract TITLE and BODY for each item in the Reuters 21578
+corpus into individual files. It expects to be passed the location of the
+decompressed archive as a command line argument.
+
+=cut
+
diff --git a/devel/benchmarks/indexers/BenchmarkingIndexer.pm b/devel/benchmarks/indexers/BenchmarkingIndexer.pm
new file mode 100644
index 0000000..53522b3
--- /dev/null
+++ b/devel/benchmarks/indexers/BenchmarkingIndexer.pm
@@ -0,0 +1,226 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package BenchmarkingIndexer;
+
+use Carp;
+use Config;
+use File::Spec::Functions qw( catfile catdir );
+use POSIX qw( uname );
+
+sub new {
+ my $either = shift;
+ my $class = ref($either) || $either;
+ return bless {
+ docs => undef,
+ increment => undef,
+ store => undef,
+ engine => undef,
+ version => undef,
+ index_dir => undef,
+ corpus_dir => 'extracted_corpus',
+ article_filepaths => undef,
+ @_,
+ }, $class;
+}
+
+sub init_indexer { confess "abstract method" }
+sub build_index { confess "abstract method" }
+
+sub delayed_init {
+ my $self = shift;
+ my $article_filepaths = $self->{article_filepaths}
+ = $self->build_file_list;
+ $self->{docs} = @$article_filepaths unless defined $self->{docs};
+ $self->{increment} = $self->{docs} + 1 unless defined $self->{increment};
+}
+
+# Return a lexically sorted list of all article files from all subdirs.
+sub build_file_list {
+ my $self = shift;
+ my $corpus_dir = $self->{corpus_dir};
+ my @article_filepaths;
+ opendir CORPUS_DIR, $corpus_dir
+ or confess "Can't opendir '$corpus_dir': $!";
+ my @article_dir_names = grep {/articles/} readdir CORPUS_DIR;
+ for my $article_dir_name (@article_dir_names) {
+ my $article_dir = catdir( $corpus_dir, $article_dir_name );
+ opendir ARTICLE_DIR, $article_dir
+ or die "Can't opendir '$article_dir': $!";
+ push @article_filepaths, map { catfile( $article_dir, $_ ) }
+ grep {m/^article\d+\.txt$/} readdir ARTICLE_DIR;
+ }
+ @article_filepaths = sort @article_filepaths;
+ $self->{article_filepaths} = \@article_filepaths;
+}
+
+# Print out stats for one run.
+sub print_interim_report {
+ my ( $self, %args ) = @_;
+ printf( "%-3d Secs: %.3f Docs: %-4d\n", @args{qw( rep secs count )} );
+}
+
+sub start_report {
+ # Start the output.
+ print '-' x 60 . "\n";
+}
+
+# Print out aggregate stats.
+sub print_final_report {
+ my ( $self, $times ) = @_;
+
+ # Produce mean and truncated mean.
+ my @sorted_times = sort @$times;
+ my $num_to_chop = int( @sorted_times >> 2 );
+ my $mean = 0;
+ my $trunc_mean = 0;
+ my $num_kept = 0;
+ for ( my $i = 0; $i < @sorted_times; $i++ ) {
+ $mean += $sorted_times[$i];
+ # Discard fastest 25% and slowest 25% of runs.
+ next if $i < $num_to_chop;
+ next if $i > ( $#sorted_times - $num_to_chop );
+ $trunc_mean += $sorted_times[$i];
+ $num_kept++;
+ }
+
+ $mean /= @sorted_times;
+ $trunc_mean /= $num_kept;
+ my $num_discarded = @sorted_times - $num_kept;
+ $mean = sprintf( "%.3f", $mean );
+ $trunc_mean = sprintf( "%.3f", $trunc_mean );
+
+ # Get some info about the system.
+ my $thread_support = $Config{usethreads} ? "yes" : "no";
+ my @uname_info = (uname)[ 0, 2, 4 ];
+
+ print <<END_REPORT;
+------------------------------------------------------------
+$self->{engine} $self->{version}
+Perl $Config{version}
+Thread support: $thread_support
+@uname_info
+Mean: $mean secs
+Truncated mean ($num_kept kept, $num_discarded discarded): $trunc_mean secs
+------------------------------------------------------------
+END_REPORT
+}
+
+package BenchSchema::WhiteSpaceTokenizer;
+use base qw( Lucy::Analysis::RegexTokenizer );
+
+sub new { return shift->SUPER::new( pattern => '\S+' ) }
+
+package BenchSchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new;
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => BenchSchema::WhiteSpaceTokenizer->new, );
+ $self->spec_field( name => 'title', type => $type );
+ return $self;
+}
+
+package BenchmarkingIndexer::Lucy;
+use base qw( BenchmarkingIndexer );
+
+use Time::HiRes qw( gettimeofday );
+
+sub new {
+ my $class = shift;
+ my $self = $class->SUPER::new(@_);
+
+ require Lucy;
+ require Lucy::Index::Indexer;
+
+ # Provide runtime flexibility.
+ my $schema = $self->{schema} = BenchSchema->new;
+ my $body_type = Lucy::Plan::FullTextType->new(
+ analyzer => BenchSchema::WhiteSpaceTokenizer->new,
+ highlightable => $self->{store} ? 1 : 0,
+ stored => $self->{store} ? 1 : 0,
+ );
+ $schema->spec_field( name => 'body', type => $body_type );
+
+ $self->{index_dir} = 'lucy_index';
+ $self->{engine} = 'Lucy';
+ $self->{version} = $Lucy::VERSION;
+
+ return $self;
+}
+
+sub init_indexer {
+ my ( $self, $count ) = @_;
+ my $truncate = $count == 0 ? 1 : 0;
+ return Lucy::Index::Indexer->new(
+ schema => $self->{schema},
+ index => $self->{index_dir},
+ truncate => $truncate,
+ create => 1,
+ );
+}
+
+# Build an index, stopping at $max docs if $max > 0.
+sub build_index {
+ my $self = shift;
+ $self->delayed_init;
+ my ( $max, $increment, $article_filepaths )
+ = @{$self}{qw( docs increment article_filepaths )};
+
+ # Start timer.
+ my $start = gettimeofday();
+
+ my $indexer = $self->init_indexer(0);
+
+ my $count = 0;
+ while ( $count < $max ) {
+ for my $article_filepath (@$article_filepaths) {
+ # The title is the first line, the body is the rest.
+ open( my $article_fh, '<', $article_filepath )
+ or die "Can't open file '$article_filepath'";
+
+ my %doc;
+ $doc{title} = <$article_fh>;
+ $doc{body} = do { local $/; <$article_fh> };
+
+ $indexer->add_doc( \%doc );
+
+ # Bail if we've reached spec'd number of docs.
+ $count++;
+ last if $count >= $max;
+ if ( $count % $increment == 0 and $count ) {
+ $indexer->commit;
+ undef $indexer;
+ $indexer = $self->init_indexer($count);
+ }
+ }
+ }
+
+ # Finish index.
+ $indexer->optimize;
+ $indexer->commit;
+
+ # Return elapsed seconds.
+ my $end = gettimeofday();
+ my $secs = $end - $start;
+ return ( $count, $secs );
+}
+
+1;
diff --git a/devel/benchmarks/indexers/LuceneIndexer.java b/devel/benchmarks/indexers/LuceneIndexer.java
new file mode 100755
index 0000000..20078ab
--- /dev/null
+++ b/devel/benchmarks/indexers/LuceneIndexer.java
@@ -0,0 +1,240 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.FSDirectory;
+
+import java.io.File;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.text.DecimalFormat;
+import java.util.Date;
+import java.util.Vector;
+import java.util.Collections;
+import java.util.Arrays;
+
+/**
+ * LuceneIndexer - benchmarking app
+ * usage: java LuceneIndexer [-docs MAX_TO_INDEX] [-reps NUM_REPETITIONS]
+ *
+ * Recommended options: -server -Xmx500M -XX:CompileThreshold=100
+ */
+
+public class LuceneIndexer {
+ static File corpusDir = new File("extracted_corpus");
+ static File indexDir = new File("lucene_index");
+ static String[] fileList;
+
+ public static void main (String[] args) throws Exception {
+ // verify that we're running from the right directory
+ if (!corpusDir.exists())
+ throw new Exception("Can't find 'extracted_corpus' directory");
+
+ // assemble the sorted list of article files
+ fileList = buildFileList();
+
+ // parse command line args
+ int maxToIndex = fileList.length; // default: index all docs
+ int numReps = 1; // default: run once
+ int increment = 0;
+ boolean store = false;
+ String arg;
+ int i = 0;
+ while (i < (args.length - 1) && args[i].startsWith("-")) {
+ arg = args[i++];
+ if (arg.equals("-docs")) {
+ maxToIndex = Integer.parseInt(args[i++]);
+ }
+ else if (arg.equals("-reps")) {
+ numReps = Integer.parseInt(args[i++]);
+ }
+ else if (arg.equals("-increment")) {
+ increment = Integer.parseInt(args[i++]);
+ }
+ else if (arg.equals("-store")) {
+ if (Integer.parseInt(args[i++]) != 0) {
+ store = true;
+ }
+ }
+ else {
+ throw new Exception("Unknown argument: " + arg);
+ }
+ }
+ increment = increment == 0 ? maxToIndex + 1 : increment;
+
+ // start the output
+ System.out.println("---------------------------------------------------");
+
+ // build the index numReps times, then print a final report
+ float[] times = new float[numReps];
+ for (int rep = 1; rep <= numReps; rep++) {
+ // start the clock and build the index
+ long start = new Date().getTime();
+ int numIndexed = buildIndex(fileList, maxToIndex, increment, store);
+
+ // stop the clock and print a report
+ long end = new Date().getTime();
+ float secs = (float)(end - start) / 1000;
+ times[rep - 1] = secs;
+ printInterimReport(rep, secs, numIndexed);
+ }
+ printFinalReport(times);
+ }
+
+ // Return a lexically sorted list of all article files from all subdirs.
+ static String[] buildFileList () throws Exception {
+ File[] articleDirs = corpusDir.listFiles();
+ Vector<String> filePaths = new Vector<String>();
+ for (int i = 0; i < articleDirs.length; i++) {
+ File[] articles = articleDirs[i].listFiles();
+ for (int j = 0; j < articles.length; j++) {
+ String path = articles[j].getPath();
+ if (path.indexOf("article") == -1)
+ continue;
+ filePaths.add(path);
+ }
+ }
+ Collections.sort(filePaths);
+ return (String[])filePaths.toArray(new String[filePaths.size()]);
+ }
+
+ // Initialize an IndexWriter
+ static IndexWriter initWriter (int count) throws Exception {
+ boolean create = count > 0 ? false : true;
+ FSDirectory directory = FSDirectory.getDirectory(indexDir);
+ IndexWriter writer = new IndexWriter(directory, true,
+ new WhitespaceAnalyzer(), create);
+ // writer.setMaxBufferedDocs(1000);
+ writer.setUseCompoundFile(false);
+
+ return writer;
+ }
+
+ // Build an index, stopping at maxToIndex docs if maxToIndex > 0.
+ static int buildIndex (String[] fileList, int maxToIndex, int increment,
+ boolean store) throws Exception {
+ IndexWriter writer = initWriter(0);
+ int docsSoFar = 0;
+
+ while (docsSoFar < maxToIndex) {
+ for (int i = 0; i < fileList.length; i++) {
+ // add content to index
+ File f = new File(fileList[i]);
+ Document doc = new Document();
+ BufferedReader br = new BufferedReader(new FileReader(f));
+
+ try {
+ // the title is the first line
+ String title;
+ if ( (title = br.readLine()) == null)
+ throw new Exception("Failed to read title");
+ Field titleField = new Field("title", title, Field.Store.YES,
+ Field.Index.TOKENIZED, Field.TermVector.NO);
+ doc.add(titleField);
+
+ // the body is the rest
+ if (store) {
+ StringBuffer buf = new StringBuffer();
+ String str;
+ while ( (str = br.readLine()) != null )
+ buf.append( str );
+ String body = buf.toString();
+ Field bodyField = new Field("body", body, Field.Store.YES,
+ Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
+ doc.add(bodyField);
+ } else {
+ Field bodyField = new Field("body", br);
+ doc.add(bodyField);
+ }
+
+ writer.addDocument(doc);
+ } finally {
+ br.close();
+ }
+
+ docsSoFar++;
+ if (docsSoFar >= maxToIndex) {
+ break;
+ }
+ if (docsSoFar % increment == 0) {
+ writer.close();
+ writer = initWriter(docsSoFar);
+ }
+ }
+ }
+
+ // finish index
+ int numIndexed = writer.docCount();
+ writer.optimize();
+ writer.close();
+
+ return numIndexed;
+ }
+
+ // Print out stats for one run.
+ private static void printInterimReport(int rep, float secs,
+ int numIndexed) {
+ DecimalFormat secsFormat = new DecimalFormat("#,##0.00");
+ String secString = secsFormat.format(secs);
+ System.out.println(rep + " Secs: " + secString +
+ " Docs: " + numIndexed);
+ }
+
+ // Print out aggregate stats
+ private static void printFinalReport(float[] times) {
+ // produce mean and truncated mean
+ Arrays.sort(times);
+ float meanTime = 0.0f;
+ float truncatedMeanTime = 0.0f;
+ int numToChop = times.length >> 2;
+ int numKept = 0;
+ for (int i = 0; i < times.length; i++) {
+ meanTime += times[i];
+ // discard fastest 25% and slowest 25% of reps
+ if (i < numToChop || i >= (times.length - numToChop))
+ continue;
+ truncatedMeanTime += times[i];
+ numKept++;
+ }
+ meanTime /= times.length;
+ truncatedMeanTime /= numKept;
+ int numDiscarded = times.length - numKept;
+ DecimalFormat format = new DecimalFormat("#,##0.00");
+ String meanString = format.format(meanTime);
+ String truncatedMeanString = format.format(truncatedMeanTime);
+
+ // get the Lucene version
+ Package lucenePackage = org.apache.lucene.LucenePackage.get();
+ String luceneVersion = lucenePackage.getSpecificationVersion();
+
+ System.out.println("---------------------------------------------------");
+ System.out.println("Lucene " + luceneVersion);
+ System.out.println("JVM " + System.getProperty("java.version") +
+ " (" + System.getProperty("java.vendor") + ")");
+ System.out.println(System.getProperty("os.name") + " " +
+ System.getProperty("os.version") + " " +
+ System.getProperty("os.arch"));
+ System.out.println("Mean: " + meanString + " secs");
+ System.out.println("Truncated mean (" +
+ numKept + " kept, " +
+ numDiscarded + " discarded): " +
+ truncatedMeanString + " secs");
+ System.out.println("---------------------------------------------------");
+ }
+}
diff --git a/devel/benchmarks/indexers/lucy_indexer.plx b/devel/benchmarks/indexers/lucy_indexer.plx
new file mode 100755
index 0000000..8ec1f0a
--- /dev/null
+++ b/devel/benchmarks/indexers/lucy_indexer.plx
@@ -0,0 +1,87 @@
+#!/usr/local/bin/perl
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib '../devel/benchmarks/indexers';
+use lib 'devel/benchmarks/indexers';
+
+use Getopt::Long;
+use Cwd qw( getcwd );
+use BenchmarkingIndexer;
+
+# Index all docs and run one iter unless otherwise spec'd.
+my ( $num_reps, $max_to_index, $increment, $store, $build_index );
+GetOptions(
+ 'reps=s' => \$num_reps,
+ 'docs=s' => \$max_to_index,
+ 'increment=s' => \$increment,
+ 'store=s' => \$store,
+ 'build_index=s' => \$build_index,
+);
+$num_reps = 1 unless defined $num_reps;
+
+my $bencher = BenchmarkingIndexer::Lucy->new(
+ docs => $max_to_index,
+ increment => $increment,
+ store => $store,
+);
+
+if ($build_index) {
+ my ( $count, $secs ) = $bencher->build_index;
+ print "docs: $count elapsed: $secs\n";
+ exit;
+}
+else {
+ $bencher->start_report;
+
+ my @times;
+ for my $rep ( 1 .. $num_reps ) {
+
+ # Spawn an index-building child process.
+ my $command = "$^X ";
+ # Try to figure out if this program was called with -Mblib.
+ for (@INC) {
+ next unless /\bblib\b/;
+ # Propagate -Mblib to the child.
+ $command .= "-Mblib ";
+ last;
+ }
+ $command .= "$0 --build_index=1 ";
+ $command .= "--docs=$max_to_index " if $max_to_index;
+ $command .= "--store=$store " if $store;
+ $command .= "--increment=$increment " if $increment;
+ my $output = `$command`;
+
+ # Extract elapsed time from the output of the child.
+ $output =~ /^docs: (\d+) elapsed: ([\d.]+)/
+ or die "no match: '$output'";
+ my $docs = $1;
+ my $secs = $2;
+ push @times, $secs;
+
+ $bencher->print_interim_report(
+ rep => $rep,
+ secs => $secs,
+ count => $docs,
+ );
+ }
+
+ $bencher->print_final_report( \@times );
+}
+
diff --git a/devel/bin/lucytidy.pl b/devel/bin/lucytidy.pl
new file mode 100755
index 0000000..cf4ab83
--- /dev/null
+++ b/devel/bin/lucytidy.pl
@@ -0,0 +1,267 @@
+#!/usr/bin/perl
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use File::Spec::Functions qw( catfile rel2abs tmpdir );
+use File::Find qw( find );
+use File::Temp qw( mktemp );
+use Getopt::Long;
+use Fcntl;
+
+my $ignore = qr/(
+ \.svn
+ | \.git
+ | modules.analysis.snowstem.source
+ | perl.sample
+ )/x;
+my $scratch_template = catfile( tmpdir(), 'lucytidy_scratch_XXXXXX' );
+my $scratch = mktemp($scratch_template);
+END { unlink $scratch }
+
+# Parse command-line options.
+my $astyle = 'astyle';
+my $perltidy = 'perltidy';
+my $suffix = 'tdy';
+GetOptions(
+ 'perltidy=s' => \$perltidy,
+ 'astyle=s' => \$astyle,
+ 'suffix:s' => \$suffix
+);
+my $start = $ARGV[0];
+die "Usage: run_astyle.pl [options] PATH" unless $start;
+
+# Find astylerc and perltidyrc files.
+my $top_dir = rel2abs(__FILE__);
+$top_dir =~ s/\Wdevel.*//;
+my $astylerc = catfile( $top_dir, qw( devel conf astylerc ) );
+die "Can't find astylerc file" unless -f $astylerc;
+my $perltidyrc = catfile( $top_dir, qw( devel conf perltidyrc ) );
+die "Can't find perltidyrc file" unless -f $perltidyrc;
+
+# Find/verify astyle and perltidy executables.
+my $astyle_version_output = `$astyle --version 2>&1`;
+my $min_astyle = 2.01;
+my ($astyle_version)
+ = defined $astyle_version_output
+ ? $astyle_version_output =~ /version\s+([\d.]+)/i
+ : (undef);
+if ( !defined $astyle_version || $astyle_version < $min_astyle ) {
+ print "No astyle version >= $min_astyle -- "
+ . "C source files will be skipped...\n";
+ undef $astyle;
+}
+my $perltidy_version_output = `$perltidy --version 2>&1`;
+my $min_perltidy = "20090616";
+my ($perltidy_version)
+ = defined $perltidy_version_output
+ ? $perltidy_version_output =~ /\sv(\d+)\s/i
+ : (undef);
+if ( !defined $perltidy_version || $perltidy_version < $min_perltidy ) {
+ print "No perltidy version >= $min_perltidy -- "
+ . "Perl source files will be skipped...\n";
+ undef $perltidy;
+}
+
+# Process files.
+find( { no_chdir => 1, wanted => \&process_file, }, $start );
+exit;
+
+sub process_file {
+ my $path = $File::Find::name;
+ if ( !-f $path ) {
+ return;
+ }
+ elsif ( $path =~ $ignore ) {
+ return;
+ }
+ elsif ( $path =~ /\.(c|h)$/ ) {
+ process_c($path);
+ }
+ elsif ( $path =~ /\.(pl|plx|PL|pm|t)$/i ) {
+ process_perl($path);
+ }
+}
+
+sub process_c {
+ my $path = shift;
+ if ( !defined $astyle ) {
+ print "No astyle version >= $min_astyle, skipping $path\n";
+ return;
+ }
+ my $content = read_file($path);
+ my $orig_content = $content;
+
+ # Our __cplusplus header wrappers cause AStyle to indent everything
+ # between them. Defeat this behavior by installing placeholders that
+ # don't contain brackets.
+ $content =~ s/(__cplusplus\s+)extern\s*"C"\s*\{/$1EXTERN_C_OPEN/g;
+ $content =~ s/(__cplusplus\s+)\}/$1EXTERN_C_CLOSE/g;
+
+ # Prevent AStyle from forcing all preprocessor directives into column 1.
+ # This hack works by filling leading spaces before preprocessor directives
+ # with '#'. When AStyle sees multiple '#' symbols beginning a line, it
+ # gives up and leaves the line untouched.
+ $content =~ s/^([ ]+)#/'#' x (length($1) + 1)/mge;
+
+ # AStyle has a bug regarding statement continuation lines which break
+ # around the assignment operator. We can fool AStyle into doing the right
+ # thing by tacking on an extra '=' to the line before (a C syntax error).
+ $content =~ s!(\n\s+= )! = // HAKK$1!g;
+
+ # The same AStyle directive which allows for uncuddled elses also has the
+ # undesirable effect of detaching the "while" in a do-while loop from its
+ # closing bracket. We can thwart this behavior by munging the 'while'
+ # symbol.
+ $content =~ s!\}[ ]+while!} wHiLe!g;
+
+ # AStyle sometimes confuses dereference operators for multiplication
+ # operators and pads them. This hack prevents it from turning
+ # '(Foo*volatile*)' into '(Foo * volatile*)'.
+ $content =~ s!\*volatile\*!STARVOLAT*!g;
+
+ # AStyle has a bad interaction with Charmonizer's QUOTE macro, because it
+ # can't handle e.g. implicitly stringified brackets:
+ #
+ # QUOTE( } )
+ #
+ # We solve this problem by swapping in innocuous placeholders for the
+ # contents of all QUOTE macros.
+ my %quote_placeholders;
+ my $counter = 0;
+ while ( $content =~ /\bQUOTE\(/ ) {
+ $counter++;
+ my $placeholder = "KWOTE$counter()";
+ die "Already found placeholder '$placeholder'"
+ if $content =~ /$placeholder/;
+ $content =~ s/(QUOTE\(.*\))(.*?)$/$placeholder$2/m
+ or die "no match ('$path')";
+ $quote_placeholders{$placeholder} = $1;
+ }
+
+ # Write out the prepped file and run AStyle on it.
+ write_file( $scratch, $content );
+ system(
+ "$astyle --options=$astylerc --suffix=none --mode=c --quiet $scratch")
+ and die "astyle failed";
+
+ # Undo all of the transforms.
+ $content = read_file($scratch);
+ $content =~ s/EXTERN_C_OPEN/extern "C" {/g;
+ $content =~ s/EXTERN_C_CLOSE/}/g;
+ $content =~ s/^(#+)#/" " x length($1) . '#'/mge;
+ $content =~ s! = // HAKK!!g;
+ $content =~ s!wHiLe[ ]*\(!while (!g;
+ $content =~ s!STARVOLAT!*volatile!g;
+ while ( my ( $placeholder, $orig ) = each %quote_placeholders ) {
+ $content =~ s/\Q$placeholder/$orig/
+ or die "Can't match placeholder '$placeholder'";
+ }
+
+ # Write out the final file if it's been changed.
+ if ( $content eq $orig_content ) {
+ print "Unchanged: $path\n";
+ }
+ else {
+ print "Tidied: $path\n";
+ my $out = $suffix ? "$path.$suffix" : $path;
+ write_file( $out, $content );
+ }
+}
+
+sub process_perl {
+ my $path = shift;
+ if ( !defined $perltidy ) {
+ print "Skipped: $path\n";
+ return;
+ }
+ unlink $scratch;
+ system("$perltidy -pro=$perltidyrc -o=$scratch $path")
+ and die "perltidy failed";
+
+ # Write out the final file if it's been changed.
+ my $orig_content = read_file($path);
+ my $tidied = read_file($scratch);
+ if ( $tidied eq $orig_content ) {
+ print "Unchanged: $path\n";
+ }
+ else {
+ my $out = $suffix ? "$path.$suffix" : $path;
+ print "Tidied: $out\n";
+ write_file( $out, $tidied );
+ }
+}
+
+sub read_file {
+ my $path = shift;
+ open( my $fh, '<', $path ) or confess("Can't open '$path': $!");
+ local $/;
+ return <$fh>;
+}
+
+sub write_file {
+ my ( $path, $content ) = @_;
+ unlink $path;
+ sysopen( my $fh, $path, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$path': $!");
+ print $fh $content;
+ close $fh or confess("Close failed for '$path': $!");
+}
+
+__END__
+
+=head1 NAME
+
+lucytidy.pl - Auto-format Lucy code.
+
+=head1 SYNOPSIS
+
+ lucytidy.pl [options] PATH
+
+=head1 DESCRIPTION
+
+Lucy uses various automatic code formatters for the different languages it
+supports: AStyle, a.k.a. Artistic Style, for C code, Perltidy for Perl, etc.
+This wrapper script walks the directory structure pointed to by the supplied
+C<PATH> and decides based on file type which formatter to run. It also works
+around some formatter quirks and bugs so that we don't have to live with
+compromises in our actual code.
+
+=head1 OPTIONS
+
+=head2 --suffix
+
+ lucytidy.pl --suffix="" PATH
+
+The suffix for the tidied file. The default is "tdy". Supplying an empty
+string causes the original file to be overwritten.
+
+
+=head2 --astyle
+
+ lucytidy.pl --astyle=/path/to/alternative/astyle PATH
+
+Specify the location of the astyle executable.
+
+=head2 --perltidy
+
+ lucytidy.pl --perltidy=/path/to/alternative/perltidy PATH
+
+Specify the location of the perltidy executable.
+
+=cut
+
diff --git a/devel/bin/smoke.pl b/devel/bin/smoke.pl
new file mode 100755
index 0000000..9df360b
--- /dev/null
+++ b/devel/bin/smoke.pl
@@ -0,0 +1,145 @@
+#!/usr/bin/env perl
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use Carp;
+use FindBin;
+use Sys::Hostname;
+use File::Spec::Functions qw( catdir canonpath );
+use Net::SMTP;
+
+my $config = {
+ src => canonpath( catdir( $FindBin::Bin, '../../' ) ),
+ verbose => 0,
+ email_to => undef,
+ email_from => getpwuid($<) . '@' . hostname(),
+ email_subject => 'Lucy Smoke Test Report ' . localtime(),
+ mailhost => 'localhost',
+ test_target => 'test', # could also be 'test_valgrind' if on Linux
+};
+
+if (@ARGV) {
+ my $config_file = shift @ARGV;
+ open( my $fh, '<', $config_file ) or die "Can't open '$config_file': $!";
+ while ( defined( my $line = <$fh> ) ) {
+ $line =~ s/#.*//;
+ next unless $line =~ /^(.*?)=(.*)$/;
+ my ( $key, $value ) = ( $1, $2 );
+ for ( $key, $value ) {
+ s/^\s*//;
+ s/\s*$//;
+ }
+ $config->{$key} = $value;
+ }
+ close $fh or die "Can't close '$config_file': $!";
+}
+
+if ( !$config->{src} ) {
+ croak "no 'src' in config -- check your syntax";
+}
+if ( !-d $config->{src} ) {
+ croak "no such dir: $config->{src}";
+}
+
+my $test_target = $config->{test_target};
+my $dir = $config->{src};
+my $perl_info = get_out("$^X -V");
+my $sys_info = get_out('uname -a');
+system( 'svn', 'up', $dir ) and croak "can't svn update $dir:\n";
+chdir "$dir" or croak "can't chdir to $dir: $!";
+chdir 'perl' or croak "can't chdir to perl: $!";
+run_quiet("./Build clean") if -f 'Build';
+run_quiet("$^X Build.PL");
+run_quiet("$^X Build");
+my $test_info = get_out("./Build $test_target");
+
+if ( should_send_smoke_signal($test_info) ) {
+
+ my $msg = <<EOF;
+Looks like one or more tests failed:
+$test_info
+$sys_info
+$perl_info
+EOF
+ $msg .= `svn info $dir`;
+
+ if ( $ENV{SMOKE_TEST} ) {
+ print $msg . "\n";
+ }
+ elsif ( $config->{email_to} ) {
+ my $smtp = Net::SMTP->new( $config->{mailhost} );
+ $smtp->mail( $config->{email_from} );
+ $smtp->to( $config->{email_to} );
+ $smtp->data();
+ $smtp->datasend("$msg\n");
+ $smtp->dataend();
+ $smtp->quit;
+ }
+}
+elsif ( $config->{verbose} ) {
+ print "All tests pass.\n";
+ print `svn info $dir`;
+}
+
+exit;
+
+sub should_send_smoke_signal {
+ return 1 if $_[0] =~ m/fail/i;
+ return 1 if $? != 0;
+}
+
+sub run_quiet {
+ my $cmd = shift;
+ system("$cmd 2>/dev/null 1>/dev/null") and croak "$cmd failed: $!";
+}
+
+sub get_out {
+ my $cmd = shift;
+ return join( '', `$cmd 2>&1` );
+}
+
+__END__
+
+=head1 NAME
+
+smoke.pl - Lucy smoke test script
+
+=head1 SYNOPSIS
+
+ perl devel/bin/smoke.pl [path/to/config_file]
+
+=head1 DESCRIPTION
+
+By default, smoke.pl updates to the latest SVN version of the branch within which it resides
+and runs a clean build and test suite. If there are any test failures, a full
+system and test summary is printed to stdout.
+
+You may specify an alternate path to test in an ini-formatted config file.
+Use the 'src' config option to specify a path. Example:
+
+ src = /path/to/checked/out/lucy/branch
+ email_to = me@example.com
+ email_from = me@example.com
+
+By default, smoke.pl will only print output if there are errors. To see output
+if all tests pass, specify a true 'verbose' flag in your config file.
+
+ verbose = 1
+
+=cut
+
diff --git a/devel/bin/update_version b/devel/bin/update_version
new file mode 100755
index 0000000..771e9a2
--- /dev/null
+++ b/devel/bin/update_version
@@ -0,0 +1,106 @@
+#!/usr/bin/env perl
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use FindBin qw( $Bin );
+
+# make sure we are at the top-level dir
+if ( !-d 'perl' and !-d 'core' ) {
+ chdir("$Bin/../../");
+}
+
+my $usage = "$0 version\n";
+my $version = shift(@ARGV) or die $usage;
+
+# standardize version strings
+my $x_y_z_version = "";
+my $x_yyyzzz_version = "";
+if ( $version =~ m/^(\d+)\.(\d+)\.(\d+)$/ ) {
+ $x_yyyzzz_version = sprintf( "%d.%03d%03d", $1, $2, $3 );
+ $x_y_z_version = $version;
+}
+elsif ( $version =~ m/^(\d+)\.(\d\d\d)(\d\d\d)$/ ) {
+ $x_y_z_version = sprintf( "%d.%d.%d", $1, $2, $3 );
+ $x_yyyzzz_version = $version;
+}
+else {
+ die "Unknown version syntax. Try X.Y.Z or X.YYYZZZ\n";
+}
+
+print "Using version: $x_y_z_version ( $x_yyyzzz_version )\n";
+
+my $buf;
+
+# the actual work
+$buf = read_file('perl/lib/Lucy.pm');
+$buf =~ s/(our \$VERSION\ +=\ +)'.+?'/$1'$x_yyyzzz_version'/
+ or die "no match";
+$buf =~ s/XSLoader::load\( 'Lucy', '(.+?)'/XSLoader::load\( 'Lucy', '$x_yyyzzz_version'/
+ or die "no match";
+write_file( 'perl/lib/Lucy.pm', $buf );
+$buf = read_file('perl/lib/Lucy.pod');
+$buf =~ s/(^=head1\s+VERSION\s+)([\d.]+)/$1$x_y_z_version/m
+ or die "no match";
+write_file( 'perl/lib/Lucy.pod', $buf );
+$buf = read_file('perl/Build.PL');
+$buf =~ s/(dist_version\ +=>\ +)'.+?'/$1'$x_y_z_version'/
+ or die "no match";
+write_file( 'perl/Build.PL', $buf );
+
+# utility functions
+sub read_file {
+ my ($file) = @_;
+ local $/;
+ open( F, "< $file" ) or die "Cannot read $file: $!\n";
+ my $buf = <F>;
+ close(F) or die "Cannot close $file: $!\n";
+ return $buf;
+}
+
+sub write_file {
+ my ( $file, $buf ) = @_;
+ open( F, "> $file" ) or die "Cannot write $file: $!\n";
+ print F $buf;
+ close(F) or die "Cannot close $file: $!\n";
+}
+
+exit();
+
+__END__
+
+=head1 NAME
+
+update_version - update Lucy version strings in source files
+
+=head1 SYNOPSIS
+
+ perl devel/bin/update_version version
+
+=head1 DESCRIPTION
+
+Updates key source files with I<version>, using correct syntax
+depending on the file format and type.
+
+I<version> may be specified in either format:
+
+ X.Y.Z
+ X.YYYZZZ
+
+and update_version will convert the specified string into the
+correct format for each relevant file.
+
+=cut
diff --git a/devel/bin/valgrind_triggers.pl b/devel/bin/valgrind_triggers.pl
new file mode 100644
index 0000000..cdf0b27
--- /dev/null
+++ b/devel/bin/valgrind_triggers.pl
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file is used by the test_valgrind build target to generate a list of
+# suppressions.
+use strict;
+use warnings;
+use Time::HiRes;
+use Data::Dumper; # triggers XSLoader
+
diff --git a/devel/conf/astylerc b/devel/conf/astylerc
new file mode 100644
index 0000000..6141893
--- /dev/null
+++ b/devel/conf/astylerc
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Disallow tabs.
+--convert-tabs
+
+# Attach opening brackets to the previous line rather than place them on a
+# dedicated line.
+--brackets=attach
+
+# Indent switches and case statements.
+--indent-switches
+--indent-cases
+
+# Indent backslash continuations within a preprocessor directive.
+--indent-preprocessor
+
+# Vertically align subsections of multi-part conditionals.
+--min-conditional-indent=0
+
+# Force space around operators, i.e. "foo = 7" not "foo=7".
+--pad-oper
+
+# Don't cuddle elses.
+--break-elseifs
+--break-closing-brackets
+
+# Allow all statements to occupy one line, particularly conditional
+# statements:
+#
+# if (foo) do_stuff();
+#
+--keep-one-line-statements
+
+# Allow single-line blocks:
+#
+# int
+# Foo_get_thing(Foo *self)
+# { return self->thing; }
+#
+--keep-one-line-blocks
+
+# Force space between if/when/while and opening paren.
+--pad-header
+
+# Tighten parens around their contents.
+--unpad-paren
+
+# Allow continuation lines to start at anywhere up to column 79.
+--max-instatement-indent=79
+
diff --git a/devel/conf/lucyperl.supp b/devel/conf/lucyperl.supp
new file mode 100644
index 0000000..d4b9957
--- /dev/null
+++ b/devel/conf/lucyperl.supp
@@ -0,0 +1,149 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{
+ <insert a suppression name here>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_VTable_make_obj
+ fun:lucy_LFReg_new
+ fun:lucy_VTable_init_registry
+ fun:*
+}
+
+{
+ <insert a suppression name here>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_VTable_make_obj
+ fun:S_alt_field_type
+ fun:*
+}
+
+{
+ <insert_a_suppression_name_here>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_LFReg_init
+ fun:lucy_LFReg_new
+ fun:lucy_VTable_init_registry
+ fun:*
+}
+
+{
+ <insert a suppression name here>
+ Memcheck:Leak
+ fun:malloc
+ fun:lucy_Memory_wrapped_malloc
+ fun:lucy_LFReg_register
+ fun:lucy_VTable_add_to_registry
+ fun:*
+}
+
+{
+ <Class name key for VTable_registry (malloc)>
+ Memcheck:Leak
+ fun:malloc
+ fun:lucy_Memory_wrapped_malloc
+ fun:lucy_CB_new_from_trusted_utf8
+ fun:lucy_CB_clone
+ fun:lucy_VTable_add_to_registry
+ fun:*
+}
+
+{
+ <Class name key for VTable_registry (calloc)>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_VTable_make_obj
+ fun:lucy_CB_new_from_trusted_utf8
+ fun:lucy_CB_clone
+ fun:lucy_VTable_add_to_registry
+ fun:*
+}
+
+{
+ <insert a suppression name here>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_VTable_clone
+ fun:lucy_VTable_singleton
+ fun:*
+}
+
+{
+ <Add class name to child VTable (malloc)>
+ Memcheck:Leak
+ fun:malloc
+ fun:lucy_Memory_wrapped_malloc
+ fun:lucy_CB_new_from_trusted_utf8
+ fun:lucy_CB_clone
+ fun:lucy_VTable_singleton
+ fun:*
+}
+
+{
+ <Add class name to child VTable (calloc)>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_VTable_make_obj
+ fun:lucy_CB_new_from_trusted_utf8
+ fun:lucy_CB_clone
+ fun:lucy_VTable_singleton
+ fun:*
+}
+
+{
+ <Add aliased class name for VTable (malloc)>
+ Memcheck:Leak
+ fun:malloc
+ fun:lucy_Memory_wrapped_malloc
+ fun:lucy_CB_new_from_trusted_utf8
+ fun:lucy_CB_clone
+ fun:lucy_VTable_add_alias_to_registry
+ fun:*
+}
+
+{
+ <Add aliased class name for VTable (calloc)>
+ Memcheck:Leak
+ fun:calloc
+ fun:lucy_Memory_wrapped_calloc
+ fun:lucy_VTable_make_obj
+ fun:lucy_CB_new_from_trusted_utf8
+ fun:lucy_CB_clone
+ fun:lucy_VTable_add_alias_to_registry
+ fun:*
+}
+
+{
+ <insert_a_suppression_name_here>
+ Memcheck:Leak
+ fun:malloc
+ fun:lucy_Memory_wrapped_malloc
+ fun:lucy_LFReg_register
+ fun:lucy_VTable_add_alias_to_registry
+ fun:*
+}
+
+
+
+
diff --git a/devel/conf/perltidyrc b/devel/conf/perltidyrc
new file mode 100755
index 0000000..b7e223e
--- /dev/null
+++ b/devel/conf/perltidyrc
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+--perl-best-practices
+--nostandard-output
+--nostandard-error-output
+--noblanks-before-comments
+--noblanks-before-subs
+
diff --git a/devel/conf/rat-excludes b/devel/conf/rat-excludes
new file mode 100644
index 0000000..2e029c5
--- /dev/null
+++ b/devel/conf/rat-excludes
@@ -0,0 +1,19 @@
+CHANGES
+clownfish/MANIFEST
+core/Lucy/Util/StringHelper2.c
+core/Lucy/Util/StringHelper3.c
+devel/conf/rat-excludes
+modules/analysis/snowstem/source/include/libstemmer.h
+modules/analysis/snowstem/source/libstemmer/libstemmer_utf8.c
+modules/analysis/snowstem/source/libstemmer/modules_utf8.h
+modules/analysis/snowstem/source/runtime/api.c
+modules/analysis/snowstem/source/runtime/api.h
+modules/analysis/snowstem/source/runtime/header.h
+modules/analysis/snowstem/source/runtime/utilities.c
+modules/analysis/snowstem/source/src_c/*.c
+modules/analysis/snowstem/source/src_c/*.h
+modules/analysis/snowstem/source/test/tests.json
+modules/analysis/snowstop/source/snowball_stoplists.c
+perl/Changes
+perl/MANIFEST
+perl/sample/us_constitution/*.txt
diff --git a/modules/analysis/snowstem/devel/update_snowstem.pl b/modules/analysis/snowstem/devel/update_snowstem.pl
new file mode 100644
index 0000000..75e5e09
--- /dev/null
+++ b/modules/analysis/snowstem/devel/update_snowstem.pl
@@ -0,0 +1,148 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use File::Spec::Functions qw( catfile catdir no_upwards );
+use File::Copy qw( copy );
+use Cwd qw( getcwd );
+use JSON::XS;
+
+if ( @ARGV != 2 ) {
+ die "Usage: perl update_snowstem.pl SNOWBALL_SVN_CO LUCY_SNOWSTEM_DIR";
+}
+
+my ( $snow_co_dir, $dest_dir ) = @ARGV;
+die("Not a directory: '$snow_co_dir'") unless -d $snow_co_dir;
+
+my $retval = system( "svn", "update", "-r", "541", $snow_co_dir );
+die "svn update failed" if ( $retval >> 8 );
+
+my $oldpwd = getcwd();
+my $snow_build_dir = catdir( $snow_co_dir, 'snowball' );
+chdir($snow_build_dir) or die $!;
+$retval = system("make dist_libstemmer_c");
+die "'make dist_libstemmer_c' failed" if ( $retval >> 8 );
+chdir($oldpwd) or die $!;
+
+# Copy only UTF-8 Stemmer files. Keep directory structure intact so that
+# compilation succeeds.
+copy_dir_contents( 'src_c', qr/UTF/ );
+copy_dir_contents('include');
+copy_dir_contents('runtime');
+copy_dir_contents( 'libstemmer', qr/utf8.[ch]$/ );
+
+# Add include guard to libstemmer.h.
+my $libstemmer_h_path
+ = catfile( $dest_dir, qw( source include libstemmer.h ) );
+open( my $libstemmer_h_fh, '<', $libstemmer_h_path )
+ or die "Can't open '$libstemmer_h_path': $!";
+my $libstemmer_h_content = do { local $/; <$libstemmer_h_fh> };
+close $libstemmer_h_fh or die $!;
+open( $libstemmer_h_fh, '>', $libstemmer_h_path )
+ or die "Can't open '$libstemmer_h_path': $!";
+print $libstemmer_h_fh <<END_STUFF;
+#ifndef H_LIBSTEMMER
+#define H_LIBSTEMMER
+
+$libstemmer_h_content
+
+#endif /* H_LIBSTEMMER */
+
+END_STUFF
+
+# Write tests.json file. Only include 10 sample tests for each language to
+# save space -- we assume that Snowball is thoroughly exercising its tests
+# elsewhere.
+my %languages = (
+ en => 'english',
+ da => 'danish',
+ de => 'german',
+ es => 'spanish',
+ fi => 'finnish',
+ fr => 'french',
+ it => 'italian',
+ nl => 'dutch',
+ hu => 'hungarian',
+ no => 'norwegian',
+ pt => 'portuguese',
+ ro => 'romanian',
+ ru => 'russian',
+ sv => 'swedish',
+ tr => 'turkish',
+);
+my %tests;
+for my $iso ( sort keys %languages ) {
+ my $language = $languages{$iso};
+ my $words_path = catfile( $snow_co_dir, 'data', $language, 'voc.txt' );
+ my $stems_path = catfile( $snow_co_dir, 'data', $language, 'output.txt' );
+ open( my $words_fh, '<:encoding(UTF-8)', $words_path )
+ or die "Can't open '$words_path': $!";
+ open( my $stems_fh, '<:encoding(UTF-8)', $stems_path )
+ or die "Can't open '$stems_path': $!";
+ my @all_words = <$words_fh>;
+ my @all_stems = <$stems_fh>;
+
+ my @some_words;
+ my @some_stems;
+ my $interval = int( @all_words / 10 );
+ for my $i ( 0 .. 9 ) {
+ my $word = $all_words[ $i * $interval ];
+ my $stem = $all_stems[ $i * $interval ];
+ chomp($word);
+ chomp($stem);
+ die unless length($word) && length($stem);
+ push @some_words, $word;
+ push @some_stems, $stem;
+ }
+ $tests{$iso}{words} = \@some_words;
+ $tests{$iso}{stems} = \@some_stems;
+}
+my $json_encoder = JSON::XS->new->pretty(1)->canonical(1);
+my $json = $json_encoder->encode( \%tests );
+my $tests_json_path = catfile( $dest_dir, 'source', 'test', 'tests.json' );
+open( my $json_fh, '>:encoding(UTF-8)', $tests_json_path )
+ or die "Can't open '$tests_json_path': $!";
+print $json_fh $json;
+close $json_fh or die $!;
+
+# Write separate README file describing test.json's contents, since JSON is a
+# commentless format.
+my $readme_path = catfile( $dest_dir, 'source', 'test', 'README' );
+open( my $readme_fh, '>:encoding(UTF-8)', $readme_path )
+ or die "Can't open '$readme_path': $!";
+print $readme_fh <<'END_STUFF';
+The file 'tests.json' and this file were autogenerated by update_snowstem.pl.
+'tests.json' contains materials from the Snowball project. See the LICENSE
+and NOTICE files for more information.
+END_STUFF
+
+sub copy_dir_contents {
+ my ( $dir_name, $pattern ) = @_;
+ my $from_dir = catdir( $snow_build_dir, $dir_name );
+ my $to_dir = catdir( $dest_dir, 'source', $dir_name );
+ opendir( my $dh, $from_dir )
+ or die "Can't opendir '$from_dir': $!";
+ die "Not a directory: '$to_dir'" unless -d $to_dir;
+ for my $file ( no_upwards( readdir $dh ) ) {
+ next if $pattern && $file !~ $pattern;
+ next if $file =~ /\.svn/;
+ my $from = catfile( $from_dir, $file );
+ my $to = catfile( $to_dir, $file );
+ copy( $from, $to ) or die "Can't copy '$from' to '$to': $!";
+ }
+ closedir $dh or die $!;
+}
+
diff --git a/modules/analysis/snowstem/source/include/libstemmer.h b/modules/analysis/snowstem/source/include/libstemmer.h
new file mode 100644
index 0000000..575c341
--- /dev/null
+++ b/modules/analysis/snowstem/source/include/libstemmer.h
@@ -0,0 +1,86 @@
+#ifndef H_LIBSTEMMER
+#define H_LIBSTEMMER
+
+
+/* Make header file work when included from C++ */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct sb_stemmer;
+typedef unsigned char sb_symbol;
+
+/* FIXME - should be able to get a version number for each stemming
+ * algorithm (which will be incremented each time the output changes). */
+
+/** Returns an array of the names of the available stemming algorithms.
+ * Note that these are the canonical names - aliases (ie, other names for
+ * the same algorithm) will not be included in the list.
+ * The list is terminated with a null pointer.
+ *
+ * The list must not be modified in any way.
+ */
+const char ** sb_stemmer_list(void);
+
+/** Create a new stemmer object, using the specified algorithm, for the
+ * specified character encoding.
+ *
+ * All algorithms will usually be available in UTF-8, but may also be
+ * available in other character encodings.
+ *
+ * @param algorithm The algorithm name. This is either the english
+ * name of the algorithm, or the 2 or 3 letter ISO 639 codes for the
+ * language. Note that case is significant in this parameter - the
+ * value should be supplied in lower case.
+ *
+ * @param charenc The character encoding. NULL may be passed as
+ * this value, in which case UTF-8 encoding will be assumed. Otherwise,
+ * the argument may be one of "UTF_8", "ISO_8859_1" (ie, Latin 1),
+ * "CP850" (ie, MS-DOS Latin 1) or "KOI8_R" (Russian). Note that
+ * case is significant in this parameter.
+ *
+ * @return NULL if the specified algorithm is not recognised, or the
+ * algorithm is not available for the requested encoding. Otherwise,
+ * returns a pointer to a newly created stemmer for the requested algorithm.
+ * The returned pointer must be deleted by calling sb_stemmer_delete().
+ *
+ * @note NULL will also be returned if an out of memory error occurs.
+ */
+struct sb_stemmer * sb_stemmer_new(const char * algorithm, const char * charenc);
+
+/** Delete a stemmer object.
+ *
+ * This frees all resources allocated for the stemmer. After calling
+ * this function, the supplied stemmer may no longer be used in any way.
+ *
+ * It is safe to pass a null pointer to this function - this will have
+ * no effect.
+ */
+void sb_stemmer_delete(struct sb_stemmer * stemmer);
+
+/** Stem a word.
+ *
+ * The return value is owned by the stemmer - it must not be freed or
+ * modified, and it will become invalid when the stemmer is called again,
+ * or if the stemmer is freed.
+ *
+ * The length of the return value can be obtained using sb_stemmer_length().
+ *
+ * If an out-of-memory error occurs, this will return NULL.
+ */
+const sb_symbol * sb_stemmer_stem(struct sb_stemmer * stemmer,
+ const sb_symbol * word, int size);
+
+/** Get the length of the result of the last stemmed word.
+ * This should not be called before sb_stemmer_stem() has been called.
+ */
+int sb_stemmer_length(struct sb_stemmer * stemmer);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+
+#endif /* H_LIBSTEMMER */
+
diff --git a/modules/analysis/snowstem/source/libstemmer/libstemmer_utf8.c b/modules/analysis/snowstem/source/libstemmer/libstemmer_utf8.c
new file mode 100644
index 0000000..1cad3e6
--- /dev/null
+++ b/modules/analysis/snowstem/source/libstemmer/libstemmer_utf8.c
@@ -0,0 +1,95 @@
+
+#include <stdlib.h>
+#include <string.h>
+#include "../include/libstemmer.h"
+#include "../runtime/api.h"
+#include "modules_utf8.h"
+
+struct sb_stemmer {
+ struct SN_env * (*create)(void);
+ void (*close)(struct SN_env *);
+ int (*stem)(struct SN_env *);
+
+ struct SN_env * env;
+};
+
+extern const char **
+sb_stemmer_list(void)
+{
+ return algorithm_names;
+}
+
+static stemmer_encoding_t
+sb_getenc(const char * charenc)
+{
+ struct stemmer_encoding * encoding;
+ if (charenc == NULL) return ENC_UTF_8;
+ for (encoding = encodings; encoding->name != 0; encoding++) {
+ if (strcmp(encoding->name, charenc) == 0) break;
+ }
+ if (encoding->name == NULL) return ENC_UNKNOWN;
+ return encoding->enc;
+}
+
+extern struct sb_stemmer *
+sb_stemmer_new(const char * algorithm, const char * charenc)
+{
+ stemmer_encoding_t enc;
+ struct stemmer_modules * module;
+ struct sb_stemmer * stemmer;
+
+ enc = sb_getenc(charenc);
+ if (enc == ENC_UNKNOWN) return NULL;
+
+ for (module = modules; module->name != 0; module++) {
+ if (strcmp(module->name, algorithm) == 0 && module->enc == enc) break;
+ }
+ if (module->name == NULL) return NULL;
+
+ stemmer = (struct sb_stemmer *) malloc(sizeof(struct sb_stemmer));
+ if (stemmer == NULL) return NULL;
+
+ stemmer->create = module->create;
+ stemmer->close = module->close;
+ stemmer->stem = module->stem;
+
+ stemmer->env = stemmer->create();
+ if (stemmer->env == NULL)
+ {
+ sb_stemmer_delete(stemmer);
+ return NULL;
+ }
+
+ return stemmer;
+}
+
+void
+sb_stemmer_delete(struct sb_stemmer * stemmer)
+{
+ if (stemmer == 0) return;
+ if (stemmer->close == 0) return;
+ stemmer->close(stemmer->env);
+ stemmer->close = 0;
+ free(stemmer);
+}
+
+const sb_symbol *
+sb_stemmer_stem(struct sb_stemmer * stemmer, const sb_symbol * word, int size)
+{
+ int ret;
+ if (SN_set_current(stemmer->env, size, (const symbol *)(word)))
+ {
+ stemmer->env->l = 0;
+ return NULL;
+ }
+ ret = stemmer->stem(stemmer->env);
+ if (ret < 0) return NULL;
+ stemmer->env->p[stemmer->env->l] = 0;
+ return (const sb_symbol *)(stemmer->env->p);
+}
+
+int
+sb_stemmer_length(struct sb_stemmer * stemmer)
+{
+ return stemmer->env->l;
+}
diff --git a/modules/analysis/snowstem/source/libstemmer/modules_utf8.h b/modules/analysis/snowstem/source/libstemmer/modules_utf8.h
new file mode 100644
index 0000000..6a7cc92
--- /dev/null
+++ b/modules/analysis/snowstem/source/libstemmer/modules_utf8.h
@@ -0,0 +1,121 @@
+/* libstemmer/modules_utf8.h: List of stemming modules.
+ *
+ * This file is generated by mkmodules.pl from a list of module names.
+ * Do not edit manually.
+ *
+ * Modules included by this file are: danish, dutch, english, finnish, french,
+ * german, hungarian, italian, norwegian, porter, portuguese, romanian,
+ * russian, spanish, swedish, turkish
+ */
+
+#include "../src_c/stem_UTF_8_danish.h"
+#include "../src_c/stem_UTF_8_dutch.h"
+#include "../src_c/stem_UTF_8_english.h"
+#include "../src_c/stem_UTF_8_finnish.h"
+#include "../src_c/stem_UTF_8_french.h"
+#include "../src_c/stem_UTF_8_german.h"
+#include "../src_c/stem_UTF_8_hungarian.h"
+#include "../src_c/stem_UTF_8_italian.h"
+#include "../src_c/stem_UTF_8_norwegian.h"
+#include "../src_c/stem_UTF_8_porter.h"
+#include "../src_c/stem_UTF_8_portuguese.h"
+#include "../src_c/stem_UTF_8_romanian.h"
+#include "../src_c/stem_UTF_8_russian.h"
+#include "../src_c/stem_UTF_8_spanish.h"
+#include "../src_c/stem_UTF_8_swedish.h"
+#include "../src_c/stem_UTF_8_turkish.h"
+
+typedef enum {
+ ENC_UNKNOWN=0,
+ ENC_UTF_8
+} stemmer_encoding_t;
+
+struct stemmer_encoding {
+ const char * name;
+ stemmer_encoding_t enc;
+};
+static struct stemmer_encoding encodings[] = {
+ {"UTF_8", ENC_UTF_8},
+ {0,ENC_UNKNOWN}
+};
+
+struct stemmer_modules {
+ const char * name;
+ stemmer_encoding_t enc;
+ struct SN_env * (*create)(void);
+ void (*close)(struct SN_env *);
+ int (*stem)(struct SN_env *);
+};
+static struct stemmer_modules modules[] = {
+ {"da", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
+ {"dan", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
+ {"danish", ENC_UTF_8, danish_UTF_8_create_env, danish_UTF_8_close_env, danish_UTF_8_stem},
+ {"de", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
+ {"deu", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
+ {"dut", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
+ {"dutch", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
+ {"en", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
+ {"eng", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
+ {"english", ENC_UTF_8, english_UTF_8_create_env, english_UTF_8_close_env, english_UTF_8_stem},
+ {"es", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
+ {"esl", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
+ {"fi", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
+ {"fin", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
+ {"finnish", ENC_UTF_8, finnish_UTF_8_create_env, finnish_UTF_8_close_env, finnish_UTF_8_stem},
+ {"fr", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
+ {"fra", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
+ {"fre", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
+ {"french", ENC_UTF_8, french_UTF_8_create_env, french_UTF_8_close_env, french_UTF_8_stem},
+ {"ger", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
+ {"german", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
+ {"hu", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
+ {"hun", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
+ {"hungarian", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
+ {"it", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
+ {"ita", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
+ {"italian", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
+ {"nl", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
+ {"nld", ENC_UTF_8, dutch_UTF_8_create_env, dutch_UTF_8_close_env, dutch_UTF_8_stem},
+ {"no", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
+ {"nor", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
+ {"norwegian", ENC_UTF_8, norwegian_UTF_8_create_env, norwegian_UTF_8_close_env, norwegian_UTF_8_stem},
+ {"por", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
+ {"porter", ENC_UTF_8, porter_UTF_8_create_env, porter_UTF_8_close_env, porter_UTF_8_stem},
+ {"portuguese", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
+ {"pt", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
+ {"ro", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
+ {"romanian", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
+ {"ron", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
+ {"ru", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
+ {"rum", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
+ {"rus", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
+ {"russian", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
+ {"spa", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
+ {"spanish", ENC_UTF_8, spanish_UTF_8_create_env, spanish_UTF_8_close_env, spanish_UTF_8_stem},
+ {"sv", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
+ {"swe", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
+ {"swedish", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
+ {"tr", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
+ {"tur", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
+ {"turkish", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
+ {0,ENC_UNKNOWN,0,0,0}
+};
+static const char * algorithm_names[] = {
+ "danish",
+ "dutch",
+ "english",
+ "finnish",
+ "french",
+ "german",
+ "hungarian",
+ "italian",
+ "norwegian",
+ "porter",
+ "portuguese",
+ "romanian",
+ "russian",
+ "spanish",
+ "swedish",
+ "turkish",
+ 0
+};
diff --git a/modules/analysis/snowstem/source/runtime/api.c b/modules/analysis/snowstem/source/runtime/api.c
new file mode 100644
index 0000000..40039ef
--- /dev/null
+++ b/modules/analysis/snowstem/source/runtime/api.c
@@ -0,0 +1,66 @@
+
+#include <stdlib.h> /* for calloc, free */
+#include "header.h"
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
+{
+ struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
+ if (z == NULL) return NULL;
+ z->p = create_s();
+ if (z->p == NULL) goto error;
+ if (S_size)
+ {
+ int i;
+ z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
+ if (z->S == NULL) goto error;
+
+ for (i = 0; i < S_size; i++)
+ {
+ z->S[i] = create_s();
+ if (z->S[i] == NULL) goto error;
+ }
+ }
+
+ if (I_size)
+ {
+ z->I = (int *) calloc(I_size, sizeof(int));
+ if (z->I == NULL) goto error;
+ }
+
+ if (B_size)
+ {
+ z->B = (unsigned char *) calloc(B_size, sizeof(unsigned char));
+ if (z->B == NULL) goto error;
+ }
+
+ return z;
+error:
+ SN_close_env(z, S_size);
+ return NULL;
+}
+
+extern void SN_close_env(struct SN_env * z, int S_size)
+{
+ if (z == NULL) return;
+ if (S_size)
+ {
+ int i;
+ for (i = 0; i < S_size; i++)
+ {
+ lose_s(z->S[i]);
+ }
+ free(z->S);
+ }
+ free(z->I);
+ free(z->B);
+ if (z->p) lose_s(z->p);
+ free(z);
+}
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
+{
+ int err = replace_s(z, 0, z->l, size, s, NULL);
+ z->c = 0;
+ return err;
+}
+
diff --git a/modules/analysis/snowstem/source/runtime/api.h b/modules/analysis/snowstem/source/runtime/api.h
new file mode 100644
index 0000000..8b997f0
--- /dev/null
+++ b/modules/analysis/snowstem/source/runtime/api.h
@@ -0,0 +1,26 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+ More precisely, replace 'char' with whatever type guarantees the
+ character width you need. Note however that sizeof(symbol) should divide
+ HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+ there is an alignment problem. In the unlikely event of a problem here,
+ consult Martin Porter.
+
+*/
+
+struct SN_env {
+ symbol * p;
+ int c; int l; int lb; int bra; int ket;
+ symbol * * S;
+ int * I;
+ unsigned char * B;
+};
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
+extern void SN_close_env(struct SN_env * z, int S_size);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+
diff --git a/modules/analysis/snowstem/source/runtime/header.h b/modules/analysis/snowstem/source/runtime/header.h
new file mode 100644
index 0000000..4d3078f
--- /dev/null
+++ b/modules/analysis/snowstem/source/runtime/header.h
@@ -0,0 +1,58 @@
+
+#include <limits.h>
+
+#include "api.h"
+
+#define MAXINT INT_MAX
+#define MININT INT_MIN
+
+#define HEAD 2*sizeof(int)
+
+#define SIZE(p) ((int *)(p))[-1]
+#define SET_SIZE(p, n) ((int *)(p))[-1] = n
+#define CAPACITY(p) ((int *)(p))[-2]
+
+struct among
+{ int s_size; /* number of chars in string */
+ const symbol * s; /* search string */
+ int substring_i;/* index to longest matching substring */
+ int result; /* result of the lookup */
+ int (* function)(struct SN_env *);
+};
+
+extern symbol * create_s(void);
+extern void lose_s(symbol * p);
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_v(struct SN_env * z, const symbol * p);
+extern int eq_v_b(struct SN_env * z, const symbol * p);
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size);
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
+
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
+extern int slice_from_v(struct SN_env * z, const symbol * p);
+extern int slice_del(struct SN_env * z);
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
+
+extern symbol * slice_to(struct SN_env * z, symbol * p);
+extern symbol * assign_to(struct SN_env * z, symbol * p);
+
+extern void debug(struct SN_env * z, int number, int line_count);
+
diff --git a/modules/analysis/snowstem/source/runtime/utilities.c b/modules/analysis/snowstem/source/runtime/utilities.c
new file mode 100644
index 0000000..1840f02
--- /dev/null
+++ b/modules/analysis/snowstem/source/runtime/utilities.c
@@ -0,0 +1,478 @@
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "header.h"
+
+#define unless(C) if(!(C))
+
+#define CREATE_SIZE 1
+
+extern symbol * create_s(void) {
+ symbol * p;
+ void * mem = malloc(HEAD + (CREATE_SIZE + 1) * sizeof(symbol));
+ if (mem == NULL) return NULL;
+ p = (symbol *) (HEAD + (char *) mem);
+ CAPACITY(p) = CREATE_SIZE;
+ SET_SIZE(p, CREATE_SIZE);
+ return p;
+}
+
+extern void lose_s(symbol * p) {
+ if (p == NULL) return;
+ free((char *) p - HEAD);
+}
+
+/*
+ new_p = skip_utf8(p, c, lb, l, n); skips n characters forwards from p + c
+ if n +ve, or n characters backwards from p + c - 1 if n -ve. new_p is the new
+ position, or 0 on failure.
+
+ -- used to implement hop and next in the utf8 case.
+*/
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n) {
+ int b;
+ if (n >= 0) {
+ for (; n > 0; n--) {
+ if (c >= l) return -1;
+ b = p[c++];
+ if (b >= 0xC0) { /* 1100 0000 */
+ while (c < l) {
+ b = p[c];
+ if (b >= 0xC0 || b < 0x80) break;
+ /* break unless b is 10------ */
+ c++;
+ }
+ }
+ }
+ } else {
+ for (; n < 0; n++) {
+ if (c <= lb) return -1;
+ b = p[--c];
+ if (b >= 0x80) { /* 1000 0000 */
+ while (c > lb) {
+ b = p[c];
+ if (b >= 0xC0) break; /* 1100 0000 */
+ c--;
+ }
+ }
+ }
+ }
+ return c;
+}
+
+/* Code for character groupings: utf8 cases */
+
+static int get_utf8(const symbol * p, int c, int l, int * slot) {
+ int b0, b1;
+ if (c >= l) return 0;
+ b0 = p[c++];
+ if (b0 < 0xC0 || c == l) { /* 1100 0000 */
+ * slot = b0; return 1;
+ }
+ b1 = p[c++];
+ if (b0 < 0xE0 || c == l) { /* 1110 0000 */
+ * slot = (b0 & 0x1F) << 6 | (b1 & 0x3F); return 2;
+ }
+ * slot = (b0 & 0xF) << 12 | (b1 & 0x3F) << 6 | (p[c] & 0x3F); return 3;
+}
+
+static int get_b_utf8(const symbol * p, int c, int lb, int * slot) {
+ int b0, b1;
+ if (c <= lb) return 0;
+ b0 = p[--c];
+ if (b0 < 0x80 || c == lb) { /* 1000 0000 */
+ * slot = b0; return 1;
+ }
+ b1 = p[--c];
+ if (b1 >= 0xC0 || c == lb) { /* 1100 0000 */
+ * slot = (b1 & 0x1F) << 6 | (b0 & 0x3F); return 2;
+ }
+ * slot = (p[c] & 0xF) << 12 | (b1 & 0x3F) << 6 | (b0 & 0x3F); return 3;
+}
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_utf8(z->p, z->c, z->l, & ch);
+ unless (w) return -1;
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c += w;
+ } while (repeat);
+ return 0;
+}
+
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+ unless (w) return -1;
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c -= w;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_utf8(z->p, z->c, z->l, & ch);
+ unless (w) return -1;
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c += w;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ int w = get_b_utf8(z->p, z->c, z->lb, & ch);
+ unless (w) return -1;
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return w;
+ z->c -= w;
+ } while (repeat);
+ return 0;
+}
+
+/* Code for character groupings: non-utf8 cases */
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c >= z->l) return -1;
+ ch = z->p[z->c];
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c++;
+ } while (repeat);
+ return 0;
+}
+
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c <= z->lb) return -1;
+ ch = z->p[z->c - 1];
+ if (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c--;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c >= z->l) return -1;
+ ch = z->p[z->c];
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c++;
+ } while (repeat);
+ return 0;
+}
+
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat) {
+ do {
+ int ch;
+ if (z->c <= z->lb) return -1;
+ ch = z->p[z->c - 1];
+ unless (ch > max || (ch -= min) < 0 || (s[ch >> 3] & (0X1 << (ch & 0X7))) == 0)
+ return 1;
+ z->c--;
+ } while (repeat);
+ return 0;
+}
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s) {
+ if (z->l - z->c < s_size || memcmp(z->p + z->c, s, s_size * sizeof(symbol)) != 0) return 0;
+ z->c += s_size; return 1;
+}
+
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s) {
+ if (z->c - z->lb < s_size || memcmp(z->p + z->c - s_size, s, s_size * sizeof(symbol)) != 0) return 0;
+ z->c -= s_size; return 1;
+}
+
+extern int eq_v(struct SN_env * z, const symbol * p) {
+ return eq_s(z, SIZE(p), p);
+}
+
+extern int eq_v_b(struct SN_env * z, const symbol * p) {
+ return eq_s_b(z, SIZE(p), p);
+}
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size) {
+
+ int i = 0;
+ int j = v_size;
+
+ int c = z->c; int l = z->l;
+ symbol * q = z->p + c;
+
+ const struct among * w;
+
+ int common_i = 0;
+ int common_j = 0;
+
+ int first_key_inspected = 0;
+
+ while(1) {
+ int k = i + ((j - i) >> 1);
+ int diff = 0;
+ int common = common_i < common_j ? common_i : common_j; /* smaller */
+ w = v + k;
+ {
+ int i2; for (i2 = common; i2 < w->s_size; i2++) {
+ if (c + common == l) { diff = -1; break; }
+ diff = q[common] - w->s[i2];
+ if (diff != 0) break;
+ common++;
+ }
+ }
+ if (diff < 0) { j = k; common_j = common; }
+ else { i = k; common_i = common; }
+ if (j - i <= 1) {
+ if (i > 0) break; /* v->s has been inspected */
+ if (j == i) break; /* only one item in v */
+
+ /* - but now we need to go round once more to get
+ v->s inspected. This looks messy, but is actually
+ the optimal approach. */
+
+ if (first_key_inspected) break;
+ first_key_inspected = 1;
+ }
+ }
+ while(1) {
+ w = v + i;
+ if (common_i >= w->s_size) {
+ z->c = c + w->s_size;
+ if (w->function == 0) return w->result;
+ {
+ int res = w->function(z);
+ z->c = c + w->s_size;
+ if (res) return w->result;
+ }
+ }
+ i = w->substring_i;
+ if (i < 0) return 0;
+ }
+}
+
+/* find_among_b is for backwards processing. Same comments apply */
+
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size) {
+
+ int i = 0;
+ int j = v_size;
+
+ int c = z->c; int lb = z->lb;
+ symbol * q = z->p + c - 1;
+
+ const struct among * w;
+
+ int common_i = 0;
+ int common_j = 0;
+
+ int first_key_inspected = 0;
+
+ while(1) {
+ int k = i + ((j - i) >> 1);
+ int diff = 0;
+ int common = common_i < common_j ? common_i : common_j;
+ w = v + k;
+ {
+ int i2; for (i2 = w->s_size - 1 - common; i2 >= 0; i2--) {
+ if (c - common == lb) { diff = -1; break; }
+ diff = q[- common] - w->s[i2];
+ if (diff != 0) break;
+ common++;
+ }
+ }
+ if (diff < 0) { j = k; common_j = common; }
+ else { i = k; common_i = common; }
+ if (j - i <= 1) {
+ if (i > 0) break;
+ if (j == i) break;
+ if (first_key_inspected) break;
+ first_key_inspected = 1;
+ }
+ }
+ while(1) {
+ w = v + i;
+ if (common_i >= w->s_size) {
+ z->c = c - w->s_size;
+ if (w->function == 0) return w->result;
+ {
+ int res = w->function(z);
+ z->c = c - w->s_size;
+ if (res) return w->result;
+ }
+ }
+ i = w->substring_i;
+ if (i < 0) return 0;
+ }
+}
+
+
+/* Increase the size of the buffer pointed to by p to at least n symbols.
+ * If insufficient memory, returns NULL and frees the old buffer.
+ */
+static symbol * increase_size(symbol * p, int n) {
+ symbol * q;
+ int new_size = n + 20;
+ void * mem = realloc((char *) p - HEAD,
+ HEAD + (new_size + 1) * sizeof(symbol));
+ if (mem == NULL) {
+ lose_s(p);
+ return NULL;
+ }
+ q = (symbol *) (HEAD + (char *)mem);
+ CAPACITY(q) = new_size;
+ return q;
+}
+
+/* to replace symbols between c_bra and c_ket in z->p by the
+ s_size symbols at s.
+ Returns 0 on success, -1 on error.
+ Also, frees z->p (and sets it to NULL) on error.
+*/
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjptr)
+{
+ int adjustment;
+ int len;
+ if (z->p == NULL) {
+ z->p = create_s();
+ if (z->p == NULL) return -1;
+ }
+ adjustment = s_size - (c_ket - c_bra);
+ len = SIZE(z->p);
+ if (adjustment != 0) {
+ if (adjustment + len > CAPACITY(z->p)) {
+ z->p = increase_size(z->p, adjustment + len);
+ if (z->p == NULL) return -1;
+ }
+ memmove(z->p + c_ket + adjustment,
+ z->p + c_ket,
+ (len - c_ket) * sizeof(symbol));
+ SET_SIZE(z->p, adjustment + len);
+ z->l += adjustment;
+ if (z->c >= c_ket)
+ z->c += adjustment;
+ else
+ if (z->c > c_bra)
+ z->c = c_bra;
+ }
+ unless (s_size == 0) memmove(z->p + c_bra, s, s_size * sizeof(symbol));
+ if (adjptr != NULL)
+ *adjptr = adjustment;
+ return 0;
+}
+
+static int slice_check(struct SN_env * z) {
+
+ if (z->bra < 0 ||
+ z->bra > z->ket ||
+ z->ket > z->l ||
+ z->p == NULL ||
+ z->l > SIZE(z->p)) /* this line could be removed */
+ {
+#if 0
+ fprintf(stderr, "faulty slice operation:\n");
+ debug(z, -1, 0);
+#endif
+ return -1;
+ }
+ return 0;
+}
+
+extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s) {
+ if (slice_check(z)) return -1;
+ return replace_s(z, z->bra, z->ket, s_size, s, NULL);
+}
+
+extern int slice_from_v(struct SN_env * z, const symbol * p) {
+ return slice_from_s(z, SIZE(p), p);
+}
+
+extern int slice_del(struct SN_env * z) {
+ return slice_from_s(z, 0, 0);
+}
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s) {
+ int adjustment;
+ if (replace_s(z, bra, ket, s_size, s, &adjustment))
+ return -1;
+ if (bra <= z->bra) z->bra += adjustment;
+ if (bra <= z->ket) z->ket += adjustment;
+ return 0;
+}
+
+extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p) {
+ int adjustment;
+ if (replace_s(z, bra, ket, SIZE(p), p, &adjustment))
+ return -1;
+ if (bra <= z->bra) z->bra += adjustment;
+ if (bra <= z->ket) z->ket += adjustment;
+ return 0;
+}
+
+extern symbol * slice_to(struct SN_env * z, symbol * p) {
+ if (slice_check(z)) {
+ lose_s(p);
+ return NULL;
+ }
+ {
+ int len = z->ket - z->bra;
+ if (CAPACITY(p) < len) {
+ p = increase_size(p, len);
+ if (p == NULL)
+ return NULL;
+ }
+ memmove(p, z->p + z->bra, len * sizeof(symbol));
+ SET_SIZE(p, len);
+ }
+ return p;
+}
+
+extern symbol * assign_to(struct SN_env * z, symbol * p) {
+ int len = z->l;
+ if (CAPACITY(p) < len) {
+ p = increase_size(p, len);
+ if (p == NULL)
+ return NULL;
+ }
+ memmove(p, z->p, len * sizeof(symbol));
+ SET_SIZE(p, len);
+ return p;
+}
+
+#if 0
+extern void debug(struct SN_env * z, int number, int line_count) {
+ int i;
+ int limit = SIZE(z->p);
+ /*if (number >= 0) printf("%3d (line %4d): '", number, line_count);*/
+ if (number >= 0) printf("%3d (line %4d): [%d]'", number, line_count,limit);
+ for (i = 0; i <= limit; i++) {
+ if (z->lb == i) printf("{");
+ if (z->bra == i) printf("[");
+ if (z->c == i) printf("|");
+ if (z->ket == i) printf("]");
+ if (z->l == i) printf("}");
+ if (i < limit)
+ { int ch = z->p[i];
+ if (ch == 0) ch = '#';
+ printf("%c", ch);
+ }
+ }
+ printf("'\n");
+}
+#endif
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_danish.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_danish.c
new file mode 100644
index 0000000..e470542
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_danish.c
@@ -0,0 +1,339 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int danish_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_undouble(struct SN_env * z);
+static int r_other_suffix(struct SN_env * z);
+static int r_consonant_pair(struct SN_env * z);
+static int r_main_suffix(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * danish_UTF_8_create_env(void);
+extern void danish_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[3] = { 'h', 'e', 'd' };
+static const symbol s_0_1[5] = { 'e', 't', 'h', 'e', 'd' };
+static const symbol s_0_2[4] = { 'e', 'r', 'e', 'd' };
+static const symbol s_0_3[1] = { 'e' };
+static const symbol s_0_4[5] = { 'e', 'r', 'e', 'd', 'e' };
+static const symbol s_0_5[4] = { 'e', 'n', 'd', 'e' };
+static const symbol s_0_6[6] = { 'e', 'r', 'e', 'n', 'd', 'e' };
+static const symbol s_0_7[3] = { 'e', 'n', 'e' };
+static const symbol s_0_8[4] = { 'e', 'r', 'n', 'e' };
+static const symbol s_0_9[3] = { 'e', 'r', 'e' };
+static const symbol s_0_10[2] = { 'e', 'n' };
+static const symbol s_0_11[5] = { 'h', 'e', 'd', 'e', 'n' };
+static const symbol s_0_12[4] = { 'e', 'r', 'e', 'n' };
+static const symbol s_0_13[2] = { 'e', 'r' };
+static const symbol s_0_14[5] = { 'h', 'e', 'd', 'e', 'r' };
+static const symbol s_0_15[4] = { 'e', 'r', 'e', 'r' };
+static const symbol s_0_16[1] = { 's' };
+static const symbol s_0_17[4] = { 'h', 'e', 'd', 's' };
+static const symbol s_0_18[2] = { 'e', 's' };
+static const symbol s_0_19[5] = { 'e', 'n', 'd', 'e', 's' };
+static const symbol s_0_20[7] = { 'e', 'r', 'e', 'n', 'd', 'e', 's' };
+static const symbol s_0_21[4] = { 'e', 'n', 'e', 's' };
+static const symbol s_0_22[5] = { 'e', 'r', 'n', 'e', 's' };
+static const symbol s_0_23[4] = { 'e', 'r', 'e', 's' };
+static const symbol s_0_24[3] = { 'e', 'n', 's' };
+static const symbol s_0_25[6] = { 'h', 'e', 'd', 'e', 'n', 's' };
+static const symbol s_0_26[5] = { 'e', 'r', 'e', 'n', 's' };
+static const symbol s_0_27[3] = { 'e', 'r', 's' };
+static const symbol s_0_28[3] = { 'e', 't', 's' };
+static const symbol s_0_29[5] = { 'e', 'r', 'e', 't', 's' };
+static const symbol s_0_30[2] = { 'e', 't' };
+static const symbol s_0_31[4] = { 'e', 'r', 'e', 't' };
+
+static const struct among a_0[32] =
+{
+/* 0 */ { 3, s_0_0, -1, 1, 0},
+/* 1 */ { 5, s_0_1, 0, 1, 0},
+/* 2 */ { 4, s_0_2, -1, 1, 0},
+/* 3 */ { 1, s_0_3, -1, 1, 0},
+/* 4 */ { 5, s_0_4, 3, 1, 0},
+/* 5 */ { 4, s_0_5, 3, 1, 0},
+/* 6 */ { 6, s_0_6, 5, 1, 0},
+/* 7 */ { 3, s_0_7, 3, 1, 0},
+/* 8 */ { 4, s_0_8, 3, 1, 0},
+/* 9 */ { 3, s_0_9, 3, 1, 0},
+/* 10 */ { 2, s_0_10, -1, 1, 0},
+/* 11 */ { 5, s_0_11, 10, 1, 0},
+/* 12 */ { 4, s_0_12, 10, 1, 0},
+/* 13 */ { 2, s_0_13, -1, 1, 0},
+/* 14 */ { 5, s_0_14, 13, 1, 0},
+/* 15 */ { 4, s_0_15, 13, 1, 0},
+/* 16 */ { 1, s_0_16, -1, 2, 0},
+/* 17 */ { 4, s_0_17, 16, 1, 0},
+/* 18 */ { 2, s_0_18, 16, 1, 0},
+/* 19 */ { 5, s_0_19, 18, 1, 0},
+/* 20 */ { 7, s_0_20, 19, 1, 0},
+/* 21 */ { 4, s_0_21, 18, 1, 0},
+/* 22 */ { 5, s_0_22, 18, 1, 0},
+/* 23 */ { 4, s_0_23, 18, 1, 0},
+/* 24 */ { 3, s_0_24, 16, 1, 0},
+/* 25 */ { 6, s_0_25, 24, 1, 0},
+/* 26 */ { 5, s_0_26, 24, 1, 0},
+/* 27 */ { 3, s_0_27, 16, 1, 0},
+/* 28 */ { 3, s_0_28, 16, 1, 0},
+/* 29 */ { 5, s_0_29, 28, 1, 0},
+/* 30 */ { 2, s_0_30, -1, 1, 0},
+/* 31 */ { 4, s_0_31, 30, 1, 0}
+};
+
+static const symbol s_1_0[2] = { 'g', 'd' };
+static const symbol s_1_1[2] = { 'd', 't' };
+static const symbol s_1_2[2] = { 'g', 't' };
+static const symbol s_1_3[2] = { 'k', 't' };
+
+static const struct among a_1[4] =
+{
+/* 0 */ { 2, s_1_0, -1, -1, 0},
+/* 1 */ { 2, s_1_1, -1, -1, 0},
+/* 2 */ { 2, s_1_2, -1, -1, 0},
+/* 3 */ { 2, s_1_3, -1, -1, 0}
+};
+
+static const symbol s_2_0[2] = { 'i', 'g' };
+static const symbol s_2_1[3] = { 'l', 'i', 'g' };
+static const symbol s_2_2[4] = { 'e', 'l', 'i', 'g' };
+static const symbol s_2_3[3] = { 'e', 'l', 's' };
+static const symbol s_2_4[5] = { 'l', 0xC3, 0xB8, 's', 't' };
+
+static const struct among a_2[5] =
+{
+/* 0 */ { 2, s_2_0, -1, 1, 0},
+/* 1 */ { 3, s_2_1, 0, 1, 0},
+/* 2 */ { 4, s_2_2, 1, 1, 0},
+/* 3 */ { 3, s_2_3, -1, 1, 0},
+/* 4 */ { 5, s_2_4, -1, 2, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 128 };
+
+static const unsigned char g_s_ending[] = { 239, 254, 42, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16 };
+
+static const symbol s_0[] = { 's', 't' };
+static const symbol s_1[] = { 'i', 'g' };
+static const symbol s_2[] = { 'l', 0xC3, 0xB8, 's' };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ { int c_test = z->c; /* test, line 33 */
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, + 3);
+ if (ret < 0) return 0;
+ z->c = ret; /* hop, line 33 */
+ }
+ z->I[1] = z->c; /* setmark x, line 33 */
+ z->c = c_test;
+ }
+ if (out_grouping_U(z, g_v, 97, 248, 1) < 0) return 0; /* goto */ /* grouping v, line 34 */
+ { /* gopast */ /* non v, line 34 */
+ int ret = in_grouping_U(z, g_v, 97, 248, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 34 */
+ /* try, line 35 */
+ if (!(z->I[0] < z->I[1])) goto lab0;
+ z->I[0] = z->I[1];
+lab0:
+ return 1;
+}
+
+static int r_main_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 41 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 41 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 41 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1851440 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_0, 32); /* substring, line 41 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 41 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 48 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ if (in_grouping_b_U(z, g_s_ending, 97, 229, 0)) return 0;
+ { int ret = slice_del(z); /* delete, line 50 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_consonant_pair(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 55 */
+ { int mlimit; /* setlimit, line 56 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 56 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 56 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 100 && z->p[z->c - 1] != 116)) { z->lb = mlimit; return 0; }
+ if (!(find_among_b(z, a_1, 4))) { z->lb = mlimit; return 0; } /* substring, line 56 */
+ z->bra = z->c; /* ], line 56 */
+ z->lb = mlimit;
+ }
+ z->c = z->l - m_test;
+ }
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 62 */
+ }
+ z->bra = z->c; /* ], line 62 */
+ { int ret = slice_del(z); /* delete, line 62 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_other_suffix(struct SN_env * z) {
+ int among_var;
+ { int m1 = z->l - z->c; (void)m1; /* do, line 66 */
+ z->ket = z->c; /* [, line 66 */
+ if (!(eq_s_b(z, 2, s_0))) goto lab0;
+ z->bra = z->c; /* ], line 66 */
+ if (!(eq_s_b(z, 2, s_1))) goto lab0;
+ { int ret = slice_del(z); /* delete, line 66 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = z->l - m1;
+ }
+ { int mlimit; /* setlimit, line 67 */
+ int m2 = z->l - z->c; (void)m2;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 67 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m2;
+ z->ket = z->c; /* [, line 67 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1572992 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_2, 5); /* substring, line 67 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 67 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 70 */
+ if (ret < 0) return ret;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 70 */
+ { int ret = r_consonant_pair(z);
+ if (ret == 0) goto lab1; /* call consonant_pair, line 70 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m3;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 4, s_2); /* <-, line 72 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_undouble(struct SN_env * z) {
+ { int mlimit; /* setlimit, line 76 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 76 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 76 */
+ if (out_grouping_b_U(z, g_v, 97, 248, 0)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 76 */
+ z->S[0] = slice_to(z, z->S[0]); /* -> ch, line 76 */
+ if (z->S[0] == 0) return -1; /* -> ch, line 76 */
+ z->lb = mlimit;
+ }
+ if (!(eq_v_b(z, z->S[0]))) return 0; /* name ch, line 77 */
+ { int ret = slice_del(z); /* delete, line 78 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+extern int danish_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 84 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 84 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 85 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 86 */
+ { int ret = r_main_suffix(z);
+ if (ret == 0) goto lab1; /* call main_suffix, line 86 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 87 */
+ { int ret = r_consonant_pair(z);
+ if (ret == 0) goto lab2; /* call consonant_pair, line 87 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 88 */
+ { int ret = r_other_suffix(z);
+ if (ret == 0) goto lab3; /* call other_suffix, line 88 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = z->l - m4;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 89 */
+ { int ret = r_undouble(z);
+ if (ret == 0) goto lab4; /* call undouble, line 89 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ z->c = z->l - m5;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern struct SN_env * danish_UTF_8_create_env(void) { return SN_create_env(1, 2, 0); }
+
+extern void danish_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 1); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_danish.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_danish.h
new file mode 100644
index 0000000..ed744d4
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_danish.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * danish_UTF_8_create_env(void);
+extern void danish_UTF_8_close_env(struct SN_env * z);
+
+extern int danish_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_dutch.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_dutch.c
new file mode 100644
index 0000000..06cb812
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_dutch.c
@@ -0,0 +1,634 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int dutch_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_standard_suffix(struct SN_env * z);
+static int r_undouble(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_en_ending(struct SN_env * z);
+static int r_e_ending(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * dutch_UTF_8_create_env(void);
+extern void dutch_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_1[2] = { 0xC3, 0xA1 };
+static const symbol s_0_2[2] = { 0xC3, 0xA4 };
+static const symbol s_0_3[2] = { 0xC3, 0xA9 };
+static const symbol s_0_4[2] = { 0xC3, 0xAB };
+static const symbol s_0_5[2] = { 0xC3, 0xAD };
+static const symbol s_0_6[2] = { 0xC3, 0xAF };
+static const symbol s_0_7[2] = { 0xC3, 0xB3 };
+static const symbol s_0_8[2] = { 0xC3, 0xB6 };
+static const symbol s_0_9[2] = { 0xC3, 0xBA };
+static const symbol s_0_10[2] = { 0xC3, 0xBC };
+
+static const struct among a_0[11] =
+{
+/* 0 */ { 0, 0, -1, 6, 0},
+/* 1 */ { 2, s_0_1, 0, 1, 0},
+/* 2 */ { 2, s_0_2, 0, 1, 0},
+/* 3 */ { 2, s_0_3, 0, 2, 0},
+/* 4 */ { 2, s_0_4, 0, 2, 0},
+/* 5 */ { 2, s_0_5, 0, 3, 0},
+/* 6 */ { 2, s_0_6, 0, 3, 0},
+/* 7 */ { 2, s_0_7, 0, 4, 0},
+/* 8 */ { 2, s_0_8, 0, 4, 0},
+/* 9 */ { 2, s_0_9, 0, 5, 0},
+/* 10 */ { 2, s_0_10, 0, 5, 0}
+};
+
+static const symbol s_1_1[1] = { 'I' };
+static const symbol s_1_2[1] = { 'Y' };
+
+static const struct among a_1[3] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 1, s_1_1, 0, 2, 0},
+/* 2 */ { 1, s_1_2, 0, 1, 0}
+};
+
+static const symbol s_2_0[2] = { 'd', 'd' };
+static const symbol s_2_1[2] = { 'k', 'k' };
+static const symbol s_2_2[2] = { 't', 't' };
+
+static const struct among a_2[3] =
+{
+/* 0 */ { 2, s_2_0, -1, -1, 0},
+/* 1 */ { 2, s_2_1, -1, -1, 0},
+/* 2 */ { 2, s_2_2, -1, -1, 0}
+};
+
+static const symbol s_3_0[3] = { 'e', 'n', 'e' };
+static const symbol s_3_1[2] = { 's', 'e' };
+static const symbol s_3_2[2] = { 'e', 'n' };
+static const symbol s_3_3[5] = { 'h', 'e', 'd', 'e', 'n' };
+static const symbol s_3_4[1] = { 's' };
+
+static const struct among a_3[5] =
+{
+/* 0 */ { 3, s_3_0, -1, 2, 0},
+/* 1 */ { 2, s_3_1, -1, 3, 0},
+/* 2 */ { 2, s_3_2, -1, 2, 0},
+/* 3 */ { 5, s_3_3, 2, 1, 0},
+/* 4 */ { 1, s_3_4, -1, 3, 0}
+};
+
+static const symbol s_4_0[3] = { 'e', 'n', 'd' };
+static const symbol s_4_1[2] = { 'i', 'g' };
+static const symbol s_4_2[3] = { 'i', 'n', 'g' };
+static const symbol s_4_3[4] = { 'l', 'i', 'j', 'k' };
+static const symbol s_4_4[4] = { 'b', 'a', 'a', 'r' };
+static const symbol s_4_5[3] = { 'b', 'a', 'r' };
+
+static const struct among a_4[6] =
+{
+/* 0 */ { 3, s_4_0, -1, 1, 0},
+/* 1 */ { 2, s_4_1, -1, 2, 0},
+/* 2 */ { 3, s_4_2, -1, 1, 0},
+/* 3 */ { 4, s_4_3, -1, 3, 0},
+/* 4 */ { 4, s_4_4, -1, 4, 0},
+/* 5 */ { 3, s_4_5, -1, 5, 0}
+};
+
+static const symbol s_5_0[2] = { 'a', 'a' };
+static const symbol s_5_1[2] = { 'e', 'e' };
+static const symbol s_5_2[2] = { 'o', 'o' };
+static const symbol s_5_3[2] = { 'u', 'u' };
+
+static const struct among a_5[4] =
+{
+/* 0 */ { 2, s_5_0, -1, -1, 0},
+/* 1 */ { 2, s_5_1, -1, -1, 0},
+/* 2 */ { 2, s_5_2, -1, -1, 0},
+/* 3 */ { 2, s_5_3, -1, -1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128 };
+
+static const unsigned char g_v_I[] = { 1, 0, 0, 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128 };
+
+static const unsigned char g_v_j[] = { 17, 67, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128 };
+
+static const symbol s_0[] = { 'a' };
+static const symbol s_1[] = { 'e' };
+static const symbol s_2[] = { 'i' };
+static const symbol s_3[] = { 'o' };
+static const symbol s_4[] = { 'u' };
+static const symbol s_5[] = { 'y' };
+static const symbol s_6[] = { 'Y' };
+static const symbol s_7[] = { 'i' };
+static const symbol s_8[] = { 'I' };
+static const symbol s_9[] = { 'y' };
+static const symbol s_10[] = { 'Y' };
+static const symbol s_11[] = { 'y' };
+static const symbol s_12[] = { 'i' };
+static const symbol s_13[] = { 'e' };
+static const symbol s_14[] = { 'g', 'e', 'm' };
+static const symbol s_15[] = { 'h', 'e', 'i', 'd' };
+static const symbol s_16[] = { 'h', 'e', 'i', 'd' };
+static const symbol s_17[] = { 'c' };
+static const symbol s_18[] = { 'e', 'n' };
+static const symbol s_19[] = { 'i', 'g' };
+static const symbol s_20[] = { 'e' };
+static const symbol s_21[] = { 'e' };
+
+static int r_prelude(struct SN_env * z) {
+ int among_var;
+ { int c_test = z->c; /* test, line 42 */
+ while(1) { /* repeat, line 42 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 43 */
+ if (z->c + 1 >= z->l || z->p[z->c + 1] >> 5 != 5 || !((340306450 >> (z->p[z->c + 1] & 0x1f)) & 1)) among_var = 6; else
+ among_var = find_among(z, a_0, 11); /* substring, line 43 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 43 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_0); /* <-, line 45 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_1); /* <-, line 47 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_2); /* <-, line 49 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 1, s_3); /* <-, line 51 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 1, s_4); /* <-, line 53 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 54 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ z->c = c_test;
+ }
+ { int c_keep = z->c; /* try, line 57 */
+ z->bra = z->c; /* [, line 57 */
+ if (!(eq_s(z, 1, s_5))) { z->c = c_keep; goto lab1; }
+ z->ket = z->c; /* ], line 57 */
+ { int ret = slice_from_s(z, 1, s_6); /* <-, line 57 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ ;
+ }
+ while(1) { /* repeat, line 58 */
+ int c2 = z->c;
+ while(1) { /* goto, line 58 */
+ int c3 = z->c;
+ if (in_grouping_U(z, g_v, 97, 232, 0)) goto lab3;
+ z->bra = z->c; /* [, line 59 */
+ { int c4 = z->c; /* or, line 59 */
+ if (!(eq_s(z, 1, s_7))) goto lab5;
+ z->ket = z->c; /* ], line 59 */
+ if (in_grouping_U(z, g_v, 97, 232, 0)) goto lab5;
+ { int ret = slice_from_s(z, 1, s_8); /* <-, line 59 */
+ if (ret < 0) return ret;
+ }
+ goto lab4;
+ lab5:
+ z->c = c4;
+ if (!(eq_s(z, 1, s_9))) goto lab3;
+ z->ket = z->c; /* ], line 60 */
+ { int ret = slice_from_s(z, 1, s_10); /* <-, line 60 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab4:
+ z->c = c3;
+ break;
+ lab3:
+ z->c = c3;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab2;
+ z->c = ret; /* goto, line 58 */
+ }
+ }
+ continue;
+ lab2:
+ z->c = c2;
+ break;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ { /* gopast */ /* grouping v, line 69 */
+ int ret = out_grouping_U(z, g_v, 97, 232, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 69 */
+ int ret = in_grouping_U(z, g_v, 97, 232, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 69 */
+ /* try, line 70 */
+ if (!(z->I[0] < 3)) goto lab0;
+ z->I[0] = 3;
+lab0:
+ { /* gopast */ /* grouping v, line 71 */
+ int ret = out_grouping_U(z, g_v, 97, 232, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 71 */
+ int ret = in_grouping_U(z, g_v, 97, 232, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p2, line 71 */
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 75 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 77 */
+ if (z->c >= z->l || (z->p[z->c + 0] != 73 && z->p[z->c + 0] != 89)) among_var = 3; else
+ among_var = find_among(z, a_1, 3); /* substring, line 77 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 77 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_11); /* <-, line 78 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_12); /* <-, line 79 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 80 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_undouble(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 91 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1050640 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_2, 3))) return 0; /* among, line 91 */
+ z->c = z->l - m_test;
+ }
+ z->ket = z->c; /* [, line 91 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 91 */
+ }
+ z->bra = z->c; /* ], line 91 */
+ { int ret = slice_del(z); /* delete, line 91 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_e_ending(struct SN_env * z) {
+ z->B[0] = 0; /* unset e_found, line 95 */
+ z->ket = z->c; /* [, line 96 */
+ if (!(eq_s_b(z, 1, s_13))) return 0;
+ z->bra = z->c; /* ], line 96 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 96 */
+ if (ret < 0) return ret;
+ }
+ { int m_test = z->l - z->c; /* test, line 96 */
+ if (out_grouping_b_U(z, g_v, 97, 232, 0)) return 0;
+ z->c = z->l - m_test;
+ }
+ { int ret = slice_del(z); /* delete, line 96 */
+ if (ret < 0) return ret;
+ }
+ z->B[0] = 1; /* set e_found, line 97 */
+ { int ret = r_undouble(z);
+ if (ret == 0) return 0; /* call undouble, line 98 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_en_ending(struct SN_env * z) {
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 102 */
+ if (ret < 0) return ret;
+ }
+ { int m1 = z->l - z->c; (void)m1; /* and, line 102 */
+ if (out_grouping_b_U(z, g_v, 97, 232, 0)) return 0;
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* not, line 102 */
+ if (!(eq_s_b(z, 3, s_14))) goto lab0;
+ return 0;
+ lab0:
+ z->c = z->l - m2;
+ }
+ }
+ { int ret = slice_del(z); /* delete, line 102 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_undouble(z);
+ if (ret == 0) return 0; /* call undouble, line 103 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ { int m1 = z->l - z->c; (void)m1; /* do, line 107 */
+ z->ket = z->c; /* [, line 108 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((540704 >> (z->p[z->c - 1] & 0x1f)) & 1)) goto lab0;
+ among_var = find_among_b(z, a_3, 5); /* substring, line 108 */
+ if (!(among_var)) goto lab0;
+ z->bra = z->c; /* ], line 108 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = r_R1(z);
+ if (ret == 0) goto lab0; /* call R1, line 110 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 4, s_15); /* <-, line 110 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_en_ending(z);
+ if (ret == 0) goto lab0; /* call en_ending, line 113 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = r_R1(z);
+ if (ret == 0) goto lab0; /* call R1, line 116 */
+ if (ret < 0) return ret;
+ }
+ if (out_grouping_b_U(z, g_v_j, 97, 232, 0)) goto lab0;
+ { int ret = slice_del(z); /* delete, line 116 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab0:
+ z->c = z->l - m1;
+ }
+ { int m2 = z->l - z->c; (void)m2; /* do, line 120 */
+ { int ret = r_e_ending(z);
+ if (ret == 0) goto lab1; /* call e_ending, line 120 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 122 */
+ z->ket = z->c; /* [, line 122 */
+ if (!(eq_s_b(z, 4, s_16))) goto lab2;
+ z->bra = z->c; /* ], line 122 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab2; /* call R2, line 122 */
+ if (ret < 0) return ret;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* not, line 122 */
+ if (!(eq_s_b(z, 1, s_17))) goto lab3;
+ goto lab2;
+ lab3:
+ z->c = z->l - m4;
+ }
+ { int ret = slice_del(z); /* delete, line 122 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 123 */
+ if (!(eq_s_b(z, 2, s_18))) goto lab2;
+ z->bra = z->c; /* ], line 123 */
+ { int ret = r_en_ending(z);
+ if (ret == 0) goto lab2; /* call en_ending, line 123 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 126 */
+ z->ket = z->c; /* [, line 127 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((264336 >> (z->p[z->c - 1] & 0x1f)) & 1)) goto lab4;
+ among_var = find_among_b(z, a_4, 6); /* substring, line 127 */
+ if (!(among_var)) goto lab4;
+ z->bra = z->c; /* ], line 127 */
+ switch(among_var) {
+ case 0: goto lab4;
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab4; /* call R2, line 129 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 129 */
+ if (ret < 0) return ret;
+ }
+ { int m6 = z->l - z->c; (void)m6; /* or, line 130 */
+ z->ket = z->c; /* [, line 130 */
+ if (!(eq_s_b(z, 2, s_19))) goto lab6;
+ z->bra = z->c; /* ], line 130 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab6; /* call R2, line 130 */
+ if (ret < 0) return ret;
+ }
+ { int m7 = z->l - z->c; (void)m7; /* not, line 130 */
+ if (!(eq_s_b(z, 1, s_20))) goto lab7;
+ goto lab6;
+ lab7:
+ z->c = z->l - m7;
+ }
+ { int ret = slice_del(z); /* delete, line 130 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = z->l - m6;
+ { int ret = r_undouble(z);
+ if (ret == 0) goto lab4; /* call undouble, line 130 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab4; /* call R2, line 133 */
+ if (ret < 0) return ret;
+ }
+ { int m8 = z->l - z->c; (void)m8; /* not, line 133 */
+ if (!(eq_s_b(z, 1, s_21))) goto lab8;
+ goto lab4;
+ lab8:
+ z->c = z->l - m8;
+ }
+ { int ret = slice_del(z); /* delete, line 133 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab4; /* call R2, line 136 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 136 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_e_ending(z);
+ if (ret == 0) goto lab4; /* call e_ending, line 136 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab4; /* call R2, line 139 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 139 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab4; /* call R2, line 142 */
+ if (ret < 0) return ret;
+ }
+ if (!(z->B[0])) goto lab4; /* Boolean test e_found, line 142 */
+ { int ret = slice_del(z); /* delete, line 142 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab4:
+ z->c = z->l - m5;
+ }
+ { int m9 = z->l - z->c; (void)m9; /* do, line 146 */
+ if (out_grouping_b_U(z, g_v_I, 73, 232, 0)) goto lab9;
+ { int m_test = z->l - z->c; /* test, line 148 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((2129954 >> (z->p[z->c - 1] & 0x1f)) & 1)) goto lab9;
+ if (!(find_among_b(z, a_5, 4))) goto lab9; /* among, line 149 */
+ if (out_grouping_b_U(z, g_v, 97, 232, 0)) goto lab9;
+ z->c = z->l - m_test;
+ }
+ z->ket = z->c; /* [, line 152 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) goto lab9;
+ z->c = ret; /* next, line 152 */
+ }
+ z->bra = z->c; /* ], line 152 */
+ { int ret = slice_del(z); /* delete, line 152 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = z->l - m9;
+ }
+ return 1;
+}
+
+extern int dutch_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 159 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab0; /* call prelude, line 159 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 160 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab1; /* call mark_regions, line 160 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = c2;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 161 */
+
+ { int m3 = z->l - z->c; (void)m3; /* do, line 162 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab2; /* call standard_suffix, line 162 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ z->c = z->lb;
+ { int c4 = z->c; /* do, line 163 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab3; /* call postlude, line 163 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = c4;
+ }
+ return 1;
+}
+
+extern struct SN_env * dutch_UTF_8_create_env(void) { return SN_create_env(0, 2, 1); }
+
+extern void dutch_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_dutch.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_dutch.h
new file mode 100644
index 0000000..a996464
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_dutch.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * dutch_UTF_8_create_env(void);
+extern void dutch_UTF_8_close_env(struct SN_env * z);
+
+extern int dutch_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_english.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_english.c
new file mode 100644
index 0000000..a00f721
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_english.c
@@ -0,0 +1,1125 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int english_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_exception2(struct SN_env * z);
+static int r_exception1(struct SN_env * z);
+static int r_Step_5(struct SN_env * z);
+static int r_Step_4(struct SN_env * z);
+static int r_Step_3(struct SN_env * z);
+static int r_Step_2(struct SN_env * z);
+static int r_Step_1c(struct SN_env * z);
+static int r_Step_1b(struct SN_env * z);
+static int r_Step_1a(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_shortv(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * english_UTF_8_create_env(void);
+extern void english_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[5] = { 'a', 'r', 's', 'e', 'n' };
+static const symbol s_0_1[6] = { 'c', 'o', 'm', 'm', 'u', 'n' };
+static const symbol s_0_2[5] = { 'g', 'e', 'n', 'e', 'r' };
+
+static const struct among a_0[3] =
+{
+/* 0 */ { 5, s_0_0, -1, -1, 0},
+/* 1 */ { 6, s_0_1, -1, -1, 0},
+/* 2 */ { 5, s_0_2, -1, -1, 0}
+};
+
+static const symbol s_1_0[1] = { '\'' };
+static const symbol s_1_1[3] = { '\'', 's', '\'' };
+static const symbol s_1_2[2] = { '\'', 's' };
+
+static const struct among a_1[3] =
+{
+/* 0 */ { 1, s_1_0, -1, 1, 0},
+/* 1 */ { 3, s_1_1, 0, 1, 0},
+/* 2 */ { 2, s_1_2, -1, 1, 0}
+};
+
+static const symbol s_2_0[3] = { 'i', 'e', 'd' };
+static const symbol s_2_1[1] = { 's' };
+static const symbol s_2_2[3] = { 'i', 'e', 's' };
+static const symbol s_2_3[4] = { 's', 's', 'e', 's' };
+static const symbol s_2_4[2] = { 's', 's' };
+static const symbol s_2_5[2] = { 'u', 's' };
+
+static const struct among a_2[6] =
+{
+/* 0 */ { 3, s_2_0, -1, 2, 0},
+/* 1 */ { 1, s_2_1, -1, 3, 0},
+/* 2 */ { 3, s_2_2, 1, 2, 0},
+/* 3 */ { 4, s_2_3, 1, 1, 0},
+/* 4 */ { 2, s_2_4, 1, -1, 0},
+/* 5 */ { 2, s_2_5, 1, -1, 0}
+};
+
+static const symbol s_3_1[2] = { 'b', 'b' };
+static const symbol s_3_2[2] = { 'd', 'd' };
+static const symbol s_3_3[2] = { 'f', 'f' };
+static const symbol s_3_4[2] = { 'g', 'g' };
+static const symbol s_3_5[2] = { 'b', 'l' };
+static const symbol s_3_6[2] = { 'm', 'm' };
+static const symbol s_3_7[2] = { 'n', 'n' };
+static const symbol s_3_8[2] = { 'p', 'p' };
+static const symbol s_3_9[2] = { 'r', 'r' };
+static const symbol s_3_10[2] = { 'a', 't' };
+static const symbol s_3_11[2] = { 't', 't' };
+static const symbol s_3_12[2] = { 'i', 'z' };
+
+static const struct among a_3[13] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 2, s_3_1, 0, 2, 0},
+/* 2 */ { 2, s_3_2, 0, 2, 0},
+/* 3 */ { 2, s_3_3, 0, 2, 0},
+/* 4 */ { 2, s_3_4, 0, 2, 0},
+/* 5 */ { 2, s_3_5, 0, 1, 0},
+/* 6 */ { 2, s_3_6, 0, 2, 0},
+/* 7 */ { 2, s_3_7, 0, 2, 0},
+/* 8 */ { 2, s_3_8, 0, 2, 0},
+/* 9 */ { 2, s_3_9, 0, 2, 0},
+/* 10 */ { 2, s_3_10, 0, 1, 0},
+/* 11 */ { 2, s_3_11, 0, 2, 0},
+/* 12 */ { 2, s_3_12, 0, 1, 0}
+};
+
+static const symbol s_4_0[2] = { 'e', 'd' };
+static const symbol s_4_1[3] = { 'e', 'e', 'd' };
+static const symbol s_4_2[3] = { 'i', 'n', 'g' };
+static const symbol s_4_3[4] = { 'e', 'd', 'l', 'y' };
+static const symbol s_4_4[5] = { 'e', 'e', 'd', 'l', 'y' };
+static const symbol s_4_5[5] = { 'i', 'n', 'g', 'l', 'y' };
+
+static const struct among a_4[6] =
+{
+/* 0 */ { 2, s_4_0, -1, 2, 0},
+/* 1 */ { 3, s_4_1, 0, 1, 0},
+/* 2 */ { 3, s_4_2, -1, 2, 0},
+/* 3 */ { 4, s_4_3, -1, 2, 0},
+/* 4 */ { 5, s_4_4, 3, 1, 0},
+/* 5 */ { 5, s_4_5, -1, 2, 0}
+};
+
+static const symbol s_5_0[4] = { 'a', 'n', 'c', 'i' };
+static const symbol s_5_1[4] = { 'e', 'n', 'c', 'i' };
+static const symbol s_5_2[3] = { 'o', 'g', 'i' };
+static const symbol s_5_3[2] = { 'l', 'i' };
+static const symbol s_5_4[3] = { 'b', 'l', 'i' };
+static const symbol s_5_5[4] = { 'a', 'b', 'l', 'i' };
+static const symbol s_5_6[4] = { 'a', 'l', 'l', 'i' };
+static const symbol s_5_7[5] = { 'f', 'u', 'l', 'l', 'i' };
+static const symbol s_5_8[6] = { 'l', 'e', 's', 's', 'l', 'i' };
+static const symbol s_5_9[5] = { 'o', 'u', 's', 'l', 'i' };
+static const symbol s_5_10[5] = { 'e', 'n', 't', 'l', 'i' };
+static const symbol s_5_11[5] = { 'a', 'l', 'i', 't', 'i' };
+static const symbol s_5_12[6] = { 'b', 'i', 'l', 'i', 't', 'i' };
+static const symbol s_5_13[5] = { 'i', 'v', 'i', 't', 'i' };
+static const symbol s_5_14[6] = { 't', 'i', 'o', 'n', 'a', 'l' };
+static const symbol s_5_15[7] = { 'a', 't', 'i', 'o', 'n', 'a', 'l' };
+static const symbol s_5_16[5] = { 'a', 'l', 'i', 's', 'm' };
+static const symbol s_5_17[5] = { 'a', 't', 'i', 'o', 'n' };
+static const symbol s_5_18[7] = { 'i', 'z', 'a', 't', 'i', 'o', 'n' };
+static const symbol s_5_19[4] = { 'i', 'z', 'e', 'r' };
+static const symbol s_5_20[4] = { 'a', 't', 'o', 'r' };
+static const symbol s_5_21[7] = { 'i', 'v', 'e', 'n', 'e', 's', 's' };
+static const symbol s_5_22[7] = { 'f', 'u', 'l', 'n', 'e', 's', 's' };
+static const symbol s_5_23[7] = { 'o', 'u', 's', 'n', 'e', 's', 's' };
+
+static const struct among a_5[24] =
+{
+/* 0 */ { 4, s_5_0, -1, 3, 0},
+/* 1 */ { 4, s_5_1, -1, 2, 0},
+/* 2 */ { 3, s_5_2, -1, 13, 0},
+/* 3 */ { 2, s_5_3, -1, 16, 0},
+/* 4 */ { 3, s_5_4, 3, 12, 0},
+/* 5 */ { 4, s_5_5, 4, 4, 0},
+/* 6 */ { 4, s_5_6, 3, 8, 0},
+/* 7 */ { 5, s_5_7, 3, 14, 0},
+/* 8 */ { 6, s_5_8, 3, 15, 0},
+/* 9 */ { 5, s_5_9, 3, 10, 0},
+/* 10 */ { 5, s_5_10, 3, 5, 0},
+/* 11 */ { 5, s_5_11, -1, 8, 0},
+/* 12 */ { 6, s_5_12, -1, 12, 0},
+/* 13 */ { 5, s_5_13, -1, 11, 0},
+/* 14 */ { 6, s_5_14, -1, 1, 0},
+/* 15 */ { 7, s_5_15, 14, 7, 0},
+/* 16 */ { 5, s_5_16, -1, 8, 0},
+/* 17 */ { 5, s_5_17, -1, 7, 0},
+/* 18 */ { 7, s_5_18, 17, 6, 0},
+/* 19 */ { 4, s_5_19, -1, 6, 0},
+/* 20 */ { 4, s_5_20, -1, 7, 0},
+/* 21 */ { 7, s_5_21, -1, 11, 0},
+/* 22 */ { 7, s_5_22, -1, 9, 0},
+/* 23 */ { 7, s_5_23, -1, 10, 0}
+};
+
+static const symbol s_6_0[5] = { 'i', 'c', 'a', 't', 'e' };
+static const symbol s_6_1[5] = { 'a', 't', 'i', 'v', 'e' };
+static const symbol s_6_2[5] = { 'a', 'l', 'i', 'z', 'e' };
+static const symbol s_6_3[5] = { 'i', 'c', 'i', 't', 'i' };
+static const symbol s_6_4[4] = { 'i', 'c', 'a', 'l' };
+static const symbol s_6_5[6] = { 't', 'i', 'o', 'n', 'a', 'l' };
+static const symbol s_6_6[7] = { 'a', 't', 'i', 'o', 'n', 'a', 'l' };
+static const symbol s_6_7[3] = { 'f', 'u', 'l' };
+static const symbol s_6_8[4] = { 'n', 'e', 's', 's' };
+
+static const struct among a_6[9] =
+{
+/* 0 */ { 5, s_6_0, -1, 4, 0},
+/* 1 */ { 5, s_6_1, -1, 6, 0},
+/* 2 */ { 5, s_6_2, -1, 3, 0},
+/* 3 */ { 5, s_6_3, -1, 4, 0},
+/* 4 */ { 4, s_6_4, -1, 4, 0},
+/* 5 */ { 6, s_6_5, -1, 1, 0},
+/* 6 */ { 7, s_6_6, 5, 2, 0},
+/* 7 */ { 3, s_6_7, -1, 5, 0},
+/* 8 */ { 4, s_6_8, -1, 5, 0}
+};
+
+static const symbol s_7_0[2] = { 'i', 'c' };
+static const symbol s_7_1[4] = { 'a', 'n', 'c', 'e' };
+static const symbol s_7_2[4] = { 'e', 'n', 'c', 'e' };
+static const symbol s_7_3[4] = { 'a', 'b', 'l', 'e' };
+static const symbol s_7_4[4] = { 'i', 'b', 'l', 'e' };
+static const symbol s_7_5[3] = { 'a', 't', 'e' };
+static const symbol s_7_6[3] = { 'i', 'v', 'e' };
+static const symbol s_7_7[3] = { 'i', 'z', 'e' };
+static const symbol s_7_8[3] = { 'i', 't', 'i' };
+static const symbol s_7_9[2] = { 'a', 'l' };
+static const symbol s_7_10[3] = { 'i', 's', 'm' };
+static const symbol s_7_11[3] = { 'i', 'o', 'n' };
+static const symbol s_7_12[2] = { 'e', 'r' };
+static const symbol s_7_13[3] = { 'o', 'u', 's' };
+static const symbol s_7_14[3] = { 'a', 'n', 't' };
+static const symbol s_7_15[3] = { 'e', 'n', 't' };
+static const symbol s_7_16[4] = { 'm', 'e', 'n', 't' };
+static const symbol s_7_17[5] = { 'e', 'm', 'e', 'n', 't' };
+
+static const struct among a_7[18] =
+{
+/* 0 */ { 2, s_7_0, -1, 1, 0},
+/* 1 */ { 4, s_7_1, -1, 1, 0},
+/* 2 */ { 4, s_7_2, -1, 1, 0},
+/* 3 */ { 4, s_7_3, -1, 1, 0},
+/* 4 */ { 4, s_7_4, -1, 1, 0},
+/* 5 */ { 3, s_7_5, -1, 1, 0},
+/* 6 */ { 3, s_7_6, -1, 1, 0},
+/* 7 */ { 3, s_7_7, -1, 1, 0},
+/* 8 */ { 3, s_7_8, -1, 1, 0},
+/* 9 */ { 2, s_7_9, -1, 1, 0},
+/* 10 */ { 3, s_7_10, -1, 1, 0},
+/* 11 */ { 3, s_7_11, -1, 2, 0},
+/* 12 */ { 2, s_7_12, -1, 1, 0},
+/* 13 */ { 3, s_7_13, -1, 1, 0},
+/* 14 */ { 3, s_7_14, -1, 1, 0},
+/* 15 */ { 3, s_7_15, -1, 1, 0},
+/* 16 */ { 4, s_7_16, 15, 1, 0},
+/* 17 */ { 5, s_7_17, 16, 1, 0}
+};
+
+static const symbol s_8_0[1] = { 'e' };
+static const symbol s_8_1[1] = { 'l' };
+
+static const struct among a_8[2] =
+{
+/* 0 */ { 1, s_8_0, -1, 1, 0},
+/* 1 */ { 1, s_8_1, -1, 2, 0}
+};
+
+static const symbol s_9_0[7] = { 's', 'u', 'c', 'c', 'e', 'e', 'd' };
+static const symbol s_9_1[7] = { 'p', 'r', 'o', 'c', 'e', 'e', 'd' };
+static const symbol s_9_2[6] = { 'e', 'x', 'c', 'e', 'e', 'd' };
+static const symbol s_9_3[7] = { 'c', 'a', 'n', 'n', 'i', 'n', 'g' };
+static const symbol s_9_4[6] = { 'i', 'n', 'n', 'i', 'n', 'g' };
+static const symbol s_9_5[7] = { 'e', 'a', 'r', 'r', 'i', 'n', 'g' };
+static const symbol s_9_6[7] = { 'h', 'e', 'r', 'r', 'i', 'n', 'g' };
+static const symbol s_9_7[6] = { 'o', 'u', 't', 'i', 'n', 'g' };
+
+static const struct among a_9[8] =
+{
+/* 0 */ { 7, s_9_0, -1, -1, 0},
+/* 1 */ { 7, s_9_1, -1, -1, 0},
+/* 2 */ { 6, s_9_2, -1, -1, 0},
+/* 3 */ { 7, s_9_3, -1, -1, 0},
+/* 4 */ { 6, s_9_4, -1, -1, 0},
+/* 5 */ { 7, s_9_5, -1, -1, 0},
+/* 6 */ { 7, s_9_6, -1, -1, 0},
+/* 7 */ { 6, s_9_7, -1, -1, 0}
+};
+
+static const symbol s_10_0[5] = { 'a', 'n', 'd', 'e', 's' };
+static const symbol s_10_1[5] = { 'a', 't', 'l', 'a', 's' };
+static const symbol s_10_2[4] = { 'b', 'i', 'a', 's' };
+static const symbol s_10_3[6] = { 'c', 'o', 's', 'm', 'o', 's' };
+static const symbol s_10_4[5] = { 'd', 'y', 'i', 'n', 'g' };
+static const symbol s_10_5[5] = { 'e', 'a', 'r', 'l', 'y' };
+static const symbol s_10_6[6] = { 'g', 'e', 'n', 't', 'l', 'y' };
+static const symbol s_10_7[4] = { 'h', 'o', 'w', 'e' };
+static const symbol s_10_8[4] = { 'i', 'd', 'l', 'y' };
+static const symbol s_10_9[5] = { 'l', 'y', 'i', 'n', 'g' };
+static const symbol s_10_10[4] = { 'n', 'e', 'w', 's' };
+static const symbol s_10_11[4] = { 'o', 'n', 'l', 'y' };
+static const symbol s_10_12[6] = { 's', 'i', 'n', 'g', 'l', 'y' };
+static const symbol s_10_13[5] = { 's', 'k', 'i', 'e', 's' };
+static const symbol s_10_14[4] = { 's', 'k', 'i', 's' };
+static const symbol s_10_15[3] = { 's', 'k', 'y' };
+static const symbol s_10_16[5] = { 't', 'y', 'i', 'n', 'g' };
+static const symbol s_10_17[4] = { 'u', 'g', 'l', 'y' };
+
+static const struct among a_10[18] =
+{
+/* 0 */ { 5, s_10_0, -1, -1, 0},
+/* 1 */ { 5, s_10_1, -1, -1, 0},
+/* 2 */ { 4, s_10_2, -1, -1, 0},
+/* 3 */ { 6, s_10_3, -1, -1, 0},
+/* 4 */ { 5, s_10_4, -1, 3, 0},
+/* 5 */ { 5, s_10_5, -1, 9, 0},
+/* 6 */ { 6, s_10_6, -1, 7, 0},
+/* 7 */ { 4, s_10_7, -1, -1, 0},
+/* 8 */ { 4, s_10_8, -1, 6, 0},
+/* 9 */ { 5, s_10_9, -1, 4, 0},
+/* 10 */ { 4, s_10_10, -1, -1, 0},
+/* 11 */ { 4, s_10_11, -1, 10, 0},
+/* 12 */ { 6, s_10_12, -1, 11, 0},
+/* 13 */ { 5, s_10_13, -1, 2, 0},
+/* 14 */ { 4, s_10_14, -1, 1, 0},
+/* 15 */ { 3, s_10_15, -1, -1, 0},
+/* 16 */ { 5, s_10_16, -1, 5, 0},
+/* 17 */ { 4, s_10_17, -1, 8, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1 };
+
+static const unsigned char g_v_WXY[] = { 1, 17, 65, 208, 1 };
+
+static const unsigned char g_valid_LI[] = { 55, 141, 2 };
+
+static const symbol s_0[] = { '\'' };
+static const symbol s_1[] = { 'y' };
+static const symbol s_2[] = { 'Y' };
+static const symbol s_3[] = { 'y' };
+static const symbol s_4[] = { 'Y' };
+static const symbol s_5[] = { 's', 's' };
+static const symbol s_6[] = { 'i' };
+static const symbol s_7[] = { 'i', 'e' };
+static const symbol s_8[] = { 'e', 'e' };
+static const symbol s_9[] = { 'e' };
+static const symbol s_10[] = { 'e' };
+static const symbol s_11[] = { 'y' };
+static const symbol s_12[] = { 'Y' };
+static const symbol s_13[] = { 'i' };
+static const symbol s_14[] = { 't', 'i', 'o', 'n' };
+static const symbol s_15[] = { 'e', 'n', 'c', 'e' };
+static const symbol s_16[] = { 'a', 'n', 'c', 'e' };
+static const symbol s_17[] = { 'a', 'b', 'l', 'e' };
+static const symbol s_18[] = { 'e', 'n', 't' };
+static const symbol s_19[] = { 'i', 'z', 'e' };
+static const symbol s_20[] = { 'a', 't', 'e' };
+static const symbol s_21[] = { 'a', 'l' };
+static const symbol s_22[] = { 'f', 'u', 'l' };
+static const symbol s_23[] = { 'o', 'u', 's' };
+static const symbol s_24[] = { 'i', 'v', 'e' };
+static const symbol s_25[] = { 'b', 'l', 'e' };
+static const symbol s_26[] = { 'l' };
+static const symbol s_27[] = { 'o', 'g' };
+static const symbol s_28[] = { 'f', 'u', 'l' };
+static const symbol s_29[] = { 'l', 'e', 's', 's' };
+static const symbol s_30[] = { 't', 'i', 'o', 'n' };
+static const symbol s_31[] = { 'a', 't', 'e' };
+static const symbol s_32[] = { 'a', 'l' };
+static const symbol s_33[] = { 'i', 'c' };
+static const symbol s_34[] = { 's' };
+static const symbol s_35[] = { 't' };
+static const symbol s_36[] = { 'l' };
+static const symbol s_37[] = { 's', 'k', 'i' };
+static const symbol s_38[] = { 's', 'k', 'y' };
+static const symbol s_39[] = { 'd', 'i', 'e' };
+static const symbol s_40[] = { 'l', 'i', 'e' };
+static const symbol s_41[] = { 't', 'i', 'e' };
+static const symbol s_42[] = { 'i', 'd', 'l' };
+static const symbol s_43[] = { 'g', 'e', 'n', 't', 'l' };
+static const symbol s_44[] = { 'u', 'g', 'l', 'i' };
+static const symbol s_45[] = { 'e', 'a', 'r', 'l', 'i' };
+static const symbol s_46[] = { 'o', 'n', 'l', 'i' };
+static const symbol s_47[] = { 's', 'i', 'n', 'g', 'l' };
+static const symbol s_48[] = { 'Y' };
+static const symbol s_49[] = { 'y' };
+
+static int r_prelude(struct SN_env * z) {
+ z->B[0] = 0; /* unset Y_found, line 26 */
+ { int c1 = z->c; /* do, line 27 */
+ z->bra = z->c; /* [, line 27 */
+ if (!(eq_s(z, 1, s_0))) goto lab0;
+ z->ket = z->c; /* ], line 27 */
+ { int ret = slice_del(z); /* delete, line 27 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 28 */
+ z->bra = z->c; /* [, line 28 */
+ if (!(eq_s(z, 1, s_1))) goto lab1;
+ z->ket = z->c; /* ], line 28 */
+ { int ret = slice_from_s(z, 1, s_2); /* <-, line 28 */
+ if (ret < 0) return ret;
+ }
+ z->B[0] = 1; /* set Y_found, line 28 */
+ lab1:
+ z->c = c2;
+ }
+ { int c3 = z->c; /* do, line 29 */
+ while(1) { /* repeat, line 29 */
+ int c4 = z->c;
+ while(1) { /* goto, line 29 */
+ int c5 = z->c;
+ if (in_grouping_U(z, g_v, 97, 121, 0)) goto lab4;
+ z->bra = z->c; /* [, line 29 */
+ if (!(eq_s(z, 1, s_3))) goto lab4;
+ z->ket = z->c; /* ], line 29 */
+ z->c = c5;
+ break;
+ lab4:
+ z->c = c5;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab3;
+ z->c = ret; /* goto, line 29 */
+ }
+ }
+ { int ret = slice_from_s(z, 1, s_4); /* <-, line 29 */
+ if (ret < 0) return ret;
+ }
+ z->B[0] = 1; /* set Y_found, line 29 */
+ continue;
+ lab3:
+ z->c = c4;
+ break;
+ }
+ z->c = c3;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ { int c1 = z->c; /* do, line 35 */
+ { int c2 = z->c; /* or, line 41 */
+ if (z->c + 4 >= z->l || z->p[z->c + 4] >> 5 != 3 || !((2375680 >> (z->p[z->c + 4] & 0x1f)) & 1)) goto lab2;
+ if (!(find_among(z, a_0, 3))) goto lab2; /* among, line 36 */
+ goto lab1;
+ lab2:
+ z->c = c2;
+ { /* gopast */ /* grouping v, line 41 */
+ int ret = out_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 41 */
+ int ret = in_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ }
+ lab1:
+ z->I[0] = z->c; /* setmark p1, line 42 */
+ { /* gopast */ /* grouping v, line 43 */
+ int ret = out_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 43 */
+ int ret = in_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p2, line 43 */
+ lab0:
+ z->c = c1;
+ }
+ return 1;
+}
+
+static int r_shortv(struct SN_env * z) {
+ { int m1 = z->l - z->c; (void)m1; /* or, line 51 */
+ if (out_grouping_b_U(z, g_v_WXY, 89, 121, 0)) goto lab1;
+ if (in_grouping_b_U(z, g_v, 97, 121, 0)) goto lab1;
+ if (out_grouping_b_U(z, g_v, 97, 121, 0)) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (out_grouping_b_U(z, g_v, 97, 121, 0)) return 0;
+ if (in_grouping_b_U(z, g_v, 97, 121, 0)) return 0;
+ if (z->c > z->lb) return 0; /* atlimit, line 52 */
+ }
+lab0:
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_Step_1a(struct SN_env * z) {
+ int among_var;
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 59 */
+ z->ket = z->c; /* [, line 60 */
+ if (z->c <= z->lb || (z->p[z->c - 1] != 39 && z->p[z->c - 1] != 115)) { z->c = z->l - m_keep; goto lab0; }
+ among_var = find_among_b(z, a_1, 3); /* substring, line 60 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 60 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab0; }
+ case 1:
+ { int ret = slice_del(z); /* delete, line 62 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab0:
+ ;
+ }
+ z->ket = z->c; /* [, line 65 */
+ if (z->c <= z->lb || (z->p[z->c - 1] != 100 && z->p[z->c - 1] != 115)) return 0;
+ among_var = find_among_b(z, a_2, 6); /* substring, line 65 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 65 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 2, s_5); /* <-, line 66 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 68 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, z->l, - 2);
+ if (ret < 0) goto lab2;
+ z->c = ret; /* hop, line 68 */
+ }
+ { int ret = slice_from_s(z, 1, s_6); /* <-, line 68 */
+ if (ret < 0) return ret;
+ }
+ goto lab1;
+ lab2:
+ z->c = z->l - m1;
+ { int ret = slice_from_s(z, 2, s_7); /* <-, line 68 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab1:
+ break;
+ case 3:
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 69 */
+ }
+ { /* gopast */ /* grouping v, line 69 */
+ int ret = out_grouping_b_U(z, g_v, 97, 121, 1);
+ if (ret < 0) return 0;
+ z->c -= ret;
+ }
+ { int ret = slice_del(z); /* delete, line 69 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_1b(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 75 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((33554576 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_4, 6); /* substring, line 75 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 75 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 77 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 2, s_8); /* <-, line 77 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m_test = z->l - z->c; /* test, line 80 */
+ { /* gopast */ /* grouping v, line 80 */
+ int ret = out_grouping_b_U(z, g_v, 97, 121, 1);
+ if (ret < 0) return 0;
+ z->c -= ret;
+ }
+ z->c = z->l - m_test;
+ }
+ { int ret = slice_del(z); /* delete, line 80 */
+ if (ret < 0) return ret;
+ }
+ { int m_test = z->l - z->c; /* test, line 81 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((68514004 >> (z->p[z->c - 1] & 0x1f)) & 1)) among_var = 3; else
+ among_var = find_among_b(z, a_3, 13); /* substring, line 81 */
+ if (!(among_var)) return 0;
+ z->c = z->l - m_test;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 1, s_9); /* <+, line 83 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ z->ket = z->c; /* [, line 86 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 86 */
+ }
+ z->bra = z->c; /* ], line 86 */
+ { int ret = slice_del(z); /* delete, line 86 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ if (z->c != z->I[0]) return 0; /* atmark, line 87 */
+ { int m_test = z->l - z->c; /* test, line 87 */
+ { int ret = r_shortv(z);
+ if (ret == 0) return 0; /* call shortv, line 87 */
+ if (ret < 0) return ret;
+ }
+ z->c = z->l - m_test;
+ }
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 1, s_10); /* <+, line 87 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_1c(struct SN_env * z) {
+ z->ket = z->c; /* [, line 94 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 94 */
+ if (!(eq_s_b(z, 1, s_11))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_12))) return 0;
+ }
+lab0:
+ z->bra = z->c; /* ], line 94 */
+ if (out_grouping_b_U(z, g_v, 97, 121, 0)) return 0;
+ { int m2 = z->l - z->c; (void)m2; /* not, line 95 */
+ if (z->c > z->lb) goto lab2; /* atlimit, line 95 */
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ { int ret = slice_from_s(z, 1, s_13); /* <-, line 96 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_Step_2(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 100 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((815616 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_5, 24); /* substring, line 100 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 100 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 100 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 4, s_14); /* <-, line 101 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 4, s_15); /* <-, line 102 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 4, s_16); /* <-, line 103 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 4, s_17); /* <-, line 104 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 3, s_18); /* <-, line 105 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 3, s_19); /* <-, line 107 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_from_s(z, 3, s_20); /* <-, line 109 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 8:
+ { int ret = slice_from_s(z, 2, s_21); /* <-, line 111 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 9:
+ { int ret = slice_from_s(z, 3, s_22); /* <-, line 112 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 10:
+ { int ret = slice_from_s(z, 3, s_23); /* <-, line 114 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 11:
+ { int ret = slice_from_s(z, 3, s_24); /* <-, line 116 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 12:
+ { int ret = slice_from_s(z, 3, s_25); /* <-, line 118 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 13:
+ if (!(eq_s_b(z, 1, s_26))) return 0;
+ { int ret = slice_from_s(z, 2, s_27); /* <-, line 119 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 14:
+ { int ret = slice_from_s(z, 3, s_28); /* <-, line 120 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 15:
+ { int ret = slice_from_s(z, 4, s_29); /* <-, line 121 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 16:
+ if (in_grouping_b_U(z, g_valid_LI, 99, 116, 0)) return 0;
+ { int ret = slice_del(z); /* delete, line 122 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_3(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 127 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((528928 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_6, 9); /* substring, line 127 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 127 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 127 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 4, s_30); /* <-, line 128 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 3, s_31); /* <-, line 129 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 2, s_32); /* <-, line 130 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 2, s_33); /* <-, line 132 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_del(z); /* delete, line 134 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 136 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 136 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_4(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 141 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1864232 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_7, 18); /* substring, line 141 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 141 */
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 141 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 144 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 145 */
+ if (!(eq_s_b(z, 1, s_34))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_35))) return 0;
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 145 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_5(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 150 */
+ if (z->c <= z->lb || (z->p[z->c - 1] != 101 && z->p[z->c - 1] != 108)) return 0;
+ among_var = find_among_b(z, a_8, 2); /* substring, line 150 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 150 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 151 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab1; /* call R2, line 151 */
+ if (ret < 0) return ret;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 151 */
+ if (ret < 0) return ret;
+ }
+ { int m2 = z->l - z->c; (void)m2; /* not, line 151 */
+ { int ret = r_shortv(z);
+ if (ret == 0) goto lab2; /* call shortv, line 151 */
+ if (ret < 0) return ret;
+ }
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 151 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 152 */
+ if (ret < 0) return ret;
+ }
+ if (!(eq_s_b(z, 1, s_36))) return 0;
+ { int ret = slice_del(z); /* delete, line 152 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_exception2(struct SN_env * z) {
+ z->ket = z->c; /* [, line 158 */
+ if (z->c - 5 <= z->lb || (z->p[z->c - 1] != 100 && z->p[z->c - 1] != 103)) return 0;
+ if (!(find_among_b(z, a_9, 8))) return 0; /* substring, line 158 */
+ z->bra = z->c; /* ], line 158 */
+ if (z->c > z->lb) return 0; /* atlimit, line 158 */
+ return 1;
+}
+
+static int r_exception1(struct SN_env * z) {
+ int among_var;
+ z->bra = z->c; /* [, line 170 */
+ if (z->c + 2 >= z->l || z->p[z->c + 2] >> 5 != 3 || !((42750482 >> (z->p[z->c + 2] & 0x1f)) & 1)) return 0;
+ among_var = find_among(z, a_10, 18); /* substring, line 170 */
+ if (!(among_var)) return 0;
+ z->ket = z->c; /* ], line 170 */
+ if (z->c < z->l) return 0; /* atlimit, line 170 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 3, s_37); /* <-, line 174 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 3, s_38); /* <-, line 175 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 3, s_39); /* <-, line 176 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 3, s_40); /* <-, line 177 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 3, s_41); /* <-, line 178 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 3, s_42); /* <-, line 182 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_from_s(z, 5, s_43); /* <-, line 183 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 8:
+ { int ret = slice_from_s(z, 4, s_44); /* <-, line 184 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 9:
+ { int ret = slice_from_s(z, 5, s_45); /* <-, line 185 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 10:
+ { int ret = slice_from_s(z, 4, s_46); /* <-, line 186 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 11:
+ { int ret = slice_from_s(z, 5, s_47); /* <-, line 187 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ if (!(z->B[0])) return 0; /* Boolean test Y_found, line 203 */
+ while(1) { /* repeat, line 203 */
+ int c1 = z->c;
+ while(1) { /* goto, line 203 */
+ int c2 = z->c;
+ z->bra = z->c; /* [, line 203 */
+ if (!(eq_s(z, 1, s_48))) goto lab1;
+ z->ket = z->c; /* ], line 203 */
+ z->c = c2;
+ break;
+ lab1:
+ z->c = c2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* goto, line 203 */
+ }
+ }
+ { int ret = slice_from_s(z, 1, s_49); /* <-, line 203 */
+ if (ret < 0) return ret;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+extern int english_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* or, line 207 */
+ { int ret = r_exception1(z);
+ if (ret == 0) goto lab1; /* call exception1, line 207 */
+ if (ret < 0) return ret;
+ }
+ goto lab0;
+ lab1:
+ z->c = c1;
+ { int c2 = z->c; /* not, line 208 */
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, + 3);
+ if (ret < 0) goto lab3;
+ z->c = ret; /* hop, line 208 */
+ }
+ goto lab2;
+ lab3:
+ z->c = c2;
+ }
+ goto lab0;
+ lab2:
+ z->c = c1;
+ { int c3 = z->c; /* do, line 209 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab4; /* call prelude, line 209 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ z->c = c3;
+ }
+ { int c4 = z->c; /* do, line 210 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab5; /* call mark_regions, line 210 */
+ if (ret < 0) return ret;
+ }
+ lab5:
+ z->c = c4;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 211 */
+
+ { int m5 = z->l - z->c; (void)m5; /* do, line 213 */
+ { int ret = r_Step_1a(z);
+ if (ret == 0) goto lab6; /* call Step_1a, line 213 */
+ if (ret < 0) return ret;
+ }
+ lab6:
+ z->c = z->l - m5;
+ }
+ { int m6 = z->l - z->c; (void)m6; /* or, line 215 */
+ { int ret = r_exception2(z);
+ if (ret == 0) goto lab8; /* call exception2, line 215 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab8:
+ z->c = z->l - m6;
+ { int m7 = z->l - z->c; (void)m7; /* do, line 217 */
+ { int ret = r_Step_1b(z);
+ if (ret == 0) goto lab9; /* call Step_1b, line 217 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = z->l - m7;
+ }
+ { int m8 = z->l - z->c; (void)m8; /* do, line 218 */
+ { int ret = r_Step_1c(z);
+ if (ret == 0) goto lab10; /* call Step_1c, line 218 */
+ if (ret < 0) return ret;
+ }
+ lab10:
+ z->c = z->l - m8;
+ }
+ { int m9 = z->l - z->c; (void)m9; /* do, line 220 */
+ { int ret = r_Step_2(z);
+ if (ret == 0) goto lab11; /* call Step_2, line 220 */
+ if (ret < 0) return ret;
+ }
+ lab11:
+ z->c = z->l - m9;
+ }
+ { int m10 = z->l - z->c; (void)m10; /* do, line 221 */
+ { int ret = r_Step_3(z);
+ if (ret == 0) goto lab12; /* call Step_3, line 221 */
+ if (ret < 0) return ret;
+ }
+ lab12:
+ z->c = z->l - m10;
+ }
+ { int m11 = z->l - z->c; (void)m11; /* do, line 222 */
+ { int ret = r_Step_4(z);
+ if (ret == 0) goto lab13; /* call Step_4, line 222 */
+ if (ret < 0) return ret;
+ }
+ lab13:
+ z->c = z->l - m11;
+ }
+ { int m12 = z->l - z->c; (void)m12; /* do, line 224 */
+ { int ret = r_Step_5(z);
+ if (ret == 0) goto lab14; /* call Step_5, line 224 */
+ if (ret < 0) return ret;
+ }
+ lab14:
+ z->c = z->l - m12;
+ }
+ }
+ lab7:
+ z->c = z->lb;
+ { int c13 = z->c; /* do, line 227 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab15; /* call postlude, line 227 */
+ if (ret < 0) return ret;
+ }
+ lab15:
+ z->c = c13;
+ }
+ }
+lab0:
+ return 1;
+}
+
+extern struct SN_env * english_UTF_8_create_env(void) { return SN_create_env(0, 2, 1); }
+
+extern void english_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_english.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_english.h
new file mode 100644
index 0000000..619a8bc
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_english.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * english_UTF_8_create_env(void);
+extern void english_UTF_8_close_env(struct SN_env * z);
+
+extern int english_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_finnish.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_finnish.c
new file mode 100644
index 0000000..65a432a
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_finnish.c
@@ -0,0 +1,768 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int finnish_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_tidy(struct SN_env * z);
+static int r_other_endings(struct SN_env * z);
+static int r_t_plural(struct SN_env * z);
+static int r_i_plural(struct SN_env * z);
+static int r_case_ending(struct SN_env * z);
+static int r_VI(struct SN_env * z);
+static int r_LONG(struct SN_env * z);
+static int r_possessive(struct SN_env * z);
+static int r_particle_etc(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * finnish_UTF_8_create_env(void);
+extern void finnish_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[2] = { 'p', 'a' };
+static const symbol s_0_1[3] = { 's', 't', 'i' };
+static const symbol s_0_2[4] = { 'k', 'a', 'a', 'n' };
+static const symbol s_0_3[3] = { 'h', 'a', 'n' };
+static const symbol s_0_4[3] = { 'k', 'i', 'n' };
+static const symbol s_0_5[4] = { 'h', 0xC3, 0xA4, 'n' };
+static const symbol s_0_6[6] = { 'k', 0xC3, 0xA4, 0xC3, 0xA4, 'n' };
+static const symbol s_0_7[2] = { 'k', 'o' };
+static const symbol s_0_8[3] = { 'p', 0xC3, 0xA4 };
+static const symbol s_0_9[3] = { 'k', 0xC3, 0xB6 };
+
+static const struct among a_0[10] =
+{
+/* 0 */ { 2, s_0_0, -1, 1, 0},
+/* 1 */ { 3, s_0_1, -1, 2, 0},
+/* 2 */ { 4, s_0_2, -1, 1, 0},
+/* 3 */ { 3, s_0_3, -1, 1, 0},
+/* 4 */ { 3, s_0_4, -1, 1, 0},
+/* 5 */ { 4, s_0_5, -1, 1, 0},
+/* 6 */ { 6, s_0_6, -1, 1, 0},
+/* 7 */ { 2, s_0_7, -1, 1, 0},
+/* 8 */ { 3, s_0_8, -1, 1, 0},
+/* 9 */ { 3, s_0_9, -1, 1, 0}
+};
+
+static const symbol s_1_0[3] = { 'l', 'l', 'a' };
+static const symbol s_1_1[2] = { 'n', 'a' };
+static const symbol s_1_2[3] = { 's', 's', 'a' };
+static const symbol s_1_3[2] = { 't', 'a' };
+static const symbol s_1_4[3] = { 'l', 't', 'a' };
+static const symbol s_1_5[3] = { 's', 't', 'a' };
+
+static const struct among a_1[6] =
+{
+/* 0 */ { 3, s_1_0, -1, -1, 0},
+/* 1 */ { 2, s_1_1, -1, -1, 0},
+/* 2 */ { 3, s_1_2, -1, -1, 0},
+/* 3 */ { 2, s_1_3, -1, -1, 0},
+/* 4 */ { 3, s_1_4, 3, -1, 0},
+/* 5 */ { 3, s_1_5, 3, -1, 0}
+};
+
+static const symbol s_2_0[4] = { 'l', 'l', 0xC3, 0xA4 };
+static const symbol s_2_1[3] = { 'n', 0xC3, 0xA4 };
+static const symbol s_2_2[4] = { 's', 's', 0xC3, 0xA4 };
+static const symbol s_2_3[3] = { 't', 0xC3, 0xA4 };
+static const symbol s_2_4[4] = { 'l', 't', 0xC3, 0xA4 };
+static const symbol s_2_5[4] = { 's', 't', 0xC3, 0xA4 };
+
+static const struct among a_2[6] =
+{
+/* 0 */ { 4, s_2_0, -1, -1, 0},
+/* 1 */ { 3, s_2_1, -1, -1, 0},
+/* 2 */ { 4, s_2_2, -1, -1, 0},
+/* 3 */ { 3, s_2_3, -1, -1, 0},
+/* 4 */ { 4, s_2_4, 3, -1, 0},
+/* 5 */ { 4, s_2_5, 3, -1, 0}
+};
+
+static const symbol s_3_0[3] = { 'l', 'l', 'e' };
+static const symbol s_3_1[3] = { 'i', 'n', 'e' };
+
+static const struct among a_3[2] =
+{
+/* 0 */ { 3, s_3_0, -1, -1, 0},
+/* 1 */ { 3, s_3_1, -1, -1, 0}
+};
+
+static const symbol s_4_0[3] = { 'n', 's', 'a' };
+static const symbol s_4_1[3] = { 'm', 'm', 'e' };
+static const symbol s_4_2[3] = { 'n', 'n', 'e' };
+static const symbol s_4_3[2] = { 'n', 'i' };
+static const symbol s_4_4[2] = { 's', 'i' };
+static const symbol s_4_5[2] = { 'a', 'n' };
+static const symbol s_4_6[2] = { 'e', 'n' };
+static const symbol s_4_7[3] = { 0xC3, 0xA4, 'n' };
+static const symbol s_4_8[4] = { 'n', 's', 0xC3, 0xA4 };
+
+static const struct among a_4[9] =
+{
+/* 0 */ { 3, s_4_0, -1, 3, 0},
+/* 1 */ { 3, s_4_1, -1, 3, 0},
+/* 2 */ { 3, s_4_2, -1, 3, 0},
+/* 3 */ { 2, s_4_3, -1, 2, 0},
+/* 4 */ { 2, s_4_4, -1, 1, 0},
+/* 5 */ { 2, s_4_5, -1, 4, 0},
+/* 6 */ { 2, s_4_6, -1, 6, 0},
+/* 7 */ { 3, s_4_7, -1, 5, 0},
+/* 8 */ { 4, s_4_8, -1, 3, 0}
+};
+
+static const symbol s_5_0[2] = { 'a', 'a' };
+static const symbol s_5_1[2] = { 'e', 'e' };
+static const symbol s_5_2[2] = { 'i', 'i' };
+static const symbol s_5_3[2] = { 'o', 'o' };
+static const symbol s_5_4[2] = { 'u', 'u' };
+static const symbol s_5_5[4] = { 0xC3, 0xA4, 0xC3, 0xA4 };
+static const symbol s_5_6[4] = { 0xC3, 0xB6, 0xC3, 0xB6 };
+
+static const struct among a_5[7] =
+{
+/* 0 */ { 2, s_5_0, -1, -1, 0},
+/* 1 */ { 2, s_5_1, -1, -1, 0},
+/* 2 */ { 2, s_5_2, -1, -1, 0},
+/* 3 */ { 2, s_5_3, -1, -1, 0},
+/* 4 */ { 2, s_5_4, -1, -1, 0},
+/* 5 */ { 4, s_5_5, -1, -1, 0},
+/* 6 */ { 4, s_5_6, -1, -1, 0}
+};
+
+static const symbol s_6_0[1] = { 'a' };
+static const symbol s_6_1[3] = { 'l', 'l', 'a' };
+static const symbol s_6_2[2] = { 'n', 'a' };
+static const symbol s_6_3[3] = { 's', 's', 'a' };
+static const symbol s_6_4[2] = { 't', 'a' };
+static const symbol s_6_5[3] = { 'l', 't', 'a' };
+static const symbol s_6_6[3] = { 's', 't', 'a' };
+static const symbol s_6_7[3] = { 't', 't', 'a' };
+static const symbol s_6_8[3] = { 'l', 'l', 'e' };
+static const symbol s_6_9[3] = { 'i', 'n', 'e' };
+static const symbol s_6_10[3] = { 'k', 's', 'i' };
+static const symbol s_6_11[1] = { 'n' };
+static const symbol s_6_12[3] = { 'h', 'a', 'n' };
+static const symbol s_6_13[3] = { 'd', 'e', 'n' };
+static const symbol s_6_14[4] = { 's', 'e', 'e', 'n' };
+static const symbol s_6_15[3] = { 'h', 'e', 'n' };
+static const symbol s_6_16[4] = { 't', 't', 'e', 'n' };
+static const symbol s_6_17[3] = { 'h', 'i', 'n' };
+static const symbol s_6_18[4] = { 's', 'i', 'i', 'n' };
+static const symbol s_6_19[3] = { 'h', 'o', 'n' };
+static const symbol s_6_20[4] = { 'h', 0xC3, 0xA4, 'n' };
+static const symbol s_6_21[4] = { 'h', 0xC3, 0xB6, 'n' };
+static const symbol s_6_22[2] = { 0xC3, 0xA4 };
+static const symbol s_6_23[4] = { 'l', 'l', 0xC3, 0xA4 };
+static const symbol s_6_24[3] = { 'n', 0xC3, 0xA4 };
+static const symbol s_6_25[4] = { 's', 's', 0xC3, 0xA4 };
+static const symbol s_6_26[3] = { 't', 0xC3, 0xA4 };
+static const symbol s_6_27[4] = { 'l', 't', 0xC3, 0xA4 };
+static const symbol s_6_28[4] = { 's', 't', 0xC3, 0xA4 };
+static const symbol s_6_29[4] = { 't', 't', 0xC3, 0xA4 };
+
+static const struct among a_6[30] =
+{
+/* 0 */ { 1, s_6_0, -1, 8, 0},
+/* 1 */ { 3, s_6_1, 0, -1, 0},
+/* 2 */ { 2, s_6_2, 0, -1, 0},
+/* 3 */ { 3, s_6_3, 0, -1, 0},
+/* 4 */ { 2, s_6_4, 0, -1, 0},
+/* 5 */ { 3, s_6_5, 4, -1, 0},
+/* 6 */ { 3, s_6_6, 4, -1, 0},
+/* 7 */ { 3, s_6_7, 4, 9, 0},
+/* 8 */ { 3, s_6_8, -1, -1, 0},
+/* 9 */ { 3, s_6_9, -1, -1, 0},
+/* 10 */ { 3, s_6_10, -1, -1, 0},
+/* 11 */ { 1, s_6_11, -1, 7, 0},
+/* 12 */ { 3, s_6_12, 11, 1, 0},
+/* 13 */ { 3, s_6_13, 11, -1, r_VI},
+/* 14 */ { 4, s_6_14, 11, -1, r_LONG},
+/* 15 */ { 3, s_6_15, 11, 2, 0},
+/* 16 */ { 4, s_6_16, 11, -1, r_VI},
+/* 17 */ { 3, s_6_17, 11, 3, 0},
+/* 18 */ { 4, s_6_18, 11, -1, r_VI},
+/* 19 */ { 3, s_6_19, 11, 4, 0},
+/* 20 */ { 4, s_6_20, 11, 5, 0},
+/* 21 */ { 4, s_6_21, 11, 6, 0},
+/* 22 */ { 2, s_6_22, -1, 8, 0},
+/* 23 */ { 4, s_6_23, 22, -1, 0},
+/* 24 */ { 3, s_6_24, 22, -1, 0},
+/* 25 */ { 4, s_6_25, 22, -1, 0},
+/* 26 */ { 3, s_6_26, 22, -1, 0},
+/* 27 */ { 4, s_6_27, 26, -1, 0},
+/* 28 */ { 4, s_6_28, 26, -1, 0},
+/* 29 */ { 4, s_6_29, 26, 9, 0}
+};
+
+static const symbol s_7_0[3] = { 'e', 'j', 'a' };
+static const symbol s_7_1[3] = { 'm', 'm', 'a' };
+static const symbol s_7_2[4] = { 'i', 'm', 'm', 'a' };
+static const symbol s_7_3[3] = { 'm', 'p', 'a' };
+static const symbol s_7_4[4] = { 'i', 'm', 'p', 'a' };
+static const symbol s_7_5[3] = { 'm', 'm', 'i' };
+static const symbol s_7_6[4] = { 'i', 'm', 'm', 'i' };
+static const symbol s_7_7[3] = { 'm', 'p', 'i' };
+static const symbol s_7_8[4] = { 'i', 'm', 'p', 'i' };
+static const symbol s_7_9[4] = { 'e', 'j', 0xC3, 0xA4 };
+static const symbol s_7_10[4] = { 'm', 'm', 0xC3, 0xA4 };
+static const symbol s_7_11[5] = { 'i', 'm', 'm', 0xC3, 0xA4 };
+static const symbol s_7_12[4] = { 'm', 'p', 0xC3, 0xA4 };
+static const symbol s_7_13[5] = { 'i', 'm', 'p', 0xC3, 0xA4 };
+
+static const struct among a_7[14] =
+{
+/* 0 */ { 3, s_7_0, -1, -1, 0},
+/* 1 */ { 3, s_7_1, -1, 1, 0},
+/* 2 */ { 4, s_7_2, 1, -1, 0},
+/* 3 */ { 3, s_7_3, -1, 1, 0},
+/* 4 */ { 4, s_7_4, 3, -1, 0},
+/* 5 */ { 3, s_7_5, -1, 1, 0},
+/* 6 */ { 4, s_7_6, 5, -1, 0},
+/* 7 */ { 3, s_7_7, -1, 1, 0},
+/* 8 */ { 4, s_7_8, 7, -1, 0},
+/* 9 */ { 4, s_7_9, -1, -1, 0},
+/* 10 */ { 4, s_7_10, -1, 1, 0},
+/* 11 */ { 5, s_7_11, 10, -1, 0},
+/* 12 */ { 4, s_7_12, -1, 1, 0},
+/* 13 */ { 5, s_7_13, 12, -1, 0}
+};
+
+static const symbol s_8_0[1] = { 'i' };
+static const symbol s_8_1[1] = { 'j' };
+
+static const struct among a_8[2] =
+{
+/* 0 */ { 1, s_8_0, -1, -1, 0},
+/* 1 */ { 1, s_8_1, -1, -1, 0}
+};
+
+static const symbol s_9_0[3] = { 'm', 'm', 'a' };
+static const symbol s_9_1[4] = { 'i', 'm', 'm', 'a' };
+
+static const struct among a_9[2] =
+{
+/* 0 */ { 3, s_9_0, -1, 1, 0},
+/* 1 */ { 4, s_9_1, 0, -1, 0}
+};
+
+static const unsigned char g_AEI[] = { 17, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8 };
+
+static const unsigned char g_V1[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32 };
+
+static const unsigned char g_V2[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32 };
+
+static const unsigned char g_particle_end[] = { 17, 97, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32 };
+
+static const symbol s_0[] = { 'k' };
+static const symbol s_1[] = { 'k', 's', 'e' };
+static const symbol s_2[] = { 'k', 's', 'i' };
+static const symbol s_3[] = { 'i' };
+static const symbol s_4[] = { 'a' };
+static const symbol s_5[] = { 'e' };
+static const symbol s_6[] = { 'i' };
+static const symbol s_7[] = { 'o' };
+static const symbol s_8[] = { 0xC3, 0xA4 };
+static const symbol s_9[] = { 0xC3, 0xB6 };
+static const symbol s_10[] = { 'i', 'e' };
+static const symbol s_11[] = { 'e' };
+static const symbol s_12[] = { 'p', 'o' };
+static const symbol s_13[] = { 't' };
+static const symbol s_14[] = { 'p', 'o' };
+static const symbol s_15[] = { 'j' };
+static const symbol s_16[] = { 'o' };
+static const symbol s_17[] = { 'u' };
+static const symbol s_18[] = { 'o' };
+static const symbol s_19[] = { 'j' };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ if (out_grouping_U(z, g_V1, 97, 246, 1) < 0) return 0; /* goto */ /* grouping V1, line 46 */
+ { /* gopast */ /* non V1, line 46 */
+ int ret = in_grouping_U(z, g_V1, 97, 246, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 46 */
+ if (out_grouping_U(z, g_V1, 97, 246, 1) < 0) return 0; /* goto */ /* grouping V1, line 47 */
+ { /* gopast */ /* non V1, line 47 */
+ int ret = in_grouping_U(z, g_V1, 97, 246, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p2, line 47 */
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_particle_etc(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 55 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 55 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 55 */
+ among_var = find_among_b(z, a_0, 10); /* substring, line 55 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 55 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ if (in_grouping_b_U(z, g_particle_end, 97, 246, 0)) return 0;
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 64 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ { int ret = slice_del(z); /* delete, line 66 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_possessive(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 69 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 69 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 69 */
+ among_var = find_among_b(z, a_4, 9); /* substring, line 69 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 69 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m2 = z->l - z->c; (void)m2; /* not, line 72 */
+ if (!(eq_s_b(z, 1, s_0))) goto lab0;
+ return 0;
+ lab0:
+ z->c = z->l - m2;
+ }
+ { int ret = slice_del(z); /* delete, line 72 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 74 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 74 */
+ if (!(eq_s_b(z, 3, s_1))) return 0;
+ z->bra = z->c; /* ], line 74 */
+ { int ret = slice_from_s(z, 3, s_2); /* <-, line 74 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 78 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] != 97) return 0;
+ if (!(find_among_b(z, a_1, 6))) return 0; /* among, line 81 */
+ { int ret = slice_del(z); /* delete, line 81 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 164) return 0;
+ if (!(find_among_b(z, a_2, 6))) return 0; /* among, line 83 */
+ { int ret = slice_del(z); /* delete, line 84 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 101) return 0;
+ if (!(find_among_b(z, a_3, 2))) return 0; /* among, line 86 */
+ { int ret = slice_del(z); /* delete, line 86 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_LONG(struct SN_env * z) {
+ if (!(find_among_b(z, a_5, 7))) return 0; /* among, line 91 */
+ return 1;
+}
+
+static int r_VI(struct SN_env * z) {
+ if (!(eq_s_b(z, 1, s_3))) return 0;
+ if (in_grouping_b_U(z, g_V2, 97, 246, 0)) return 0;
+ return 1;
+}
+
+static int r_case_ending(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 96 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 96 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 96 */
+ among_var = find_among_b(z, a_6, 30); /* substring, line 96 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 96 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ if (!(eq_s_b(z, 1, s_4))) return 0;
+ break;
+ case 2:
+ if (!(eq_s_b(z, 1, s_5))) return 0;
+ break;
+ case 3:
+ if (!(eq_s_b(z, 1, s_6))) return 0;
+ break;
+ case 4:
+ if (!(eq_s_b(z, 1, s_7))) return 0;
+ break;
+ case 5:
+ if (!(eq_s_b(z, 2, s_8))) return 0;
+ break;
+ case 6:
+ if (!(eq_s_b(z, 2, s_9))) return 0;
+ break;
+ case 7:
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 111 */
+ { int m2 = z->l - z->c; (void)m2; /* and, line 113 */
+ { int m3 = z->l - z->c; (void)m3; /* or, line 112 */
+ { int ret = r_LONG(z);
+ if (ret == 0) goto lab2; /* call LONG, line 111 */
+ if (ret < 0) return ret;
+ }
+ goto lab1;
+ lab2:
+ z->c = z->l - m3;
+ if (!(eq_s_b(z, 2, s_10))) { z->c = z->l - m_keep; goto lab0; }
+ }
+ lab1:
+ z->c = z->l - m2;
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) { z->c = z->l - m_keep; goto lab0; }
+ z->c = ret; /* next, line 113 */
+ }
+ }
+ z->bra = z->c; /* ], line 113 */
+ lab0:
+ ;
+ }
+ break;
+ case 8:
+ if (in_grouping_b_U(z, g_V1, 97, 246, 0)) return 0;
+ if (out_grouping_b_U(z, g_V1, 97, 246, 0)) return 0;
+ break;
+ case 9:
+ if (!(eq_s_b(z, 1, s_11))) return 0;
+ break;
+ }
+ { int ret = slice_del(z); /* delete, line 138 */
+ if (ret < 0) return ret;
+ }
+ z->B[0] = 1; /* set ending_removed, line 139 */
+ return 1;
+}
+
+static int r_other_endings(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 142 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[1]) return 0;
+ z->c = z->I[1]; /* tomark, line 142 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 142 */
+ among_var = find_among_b(z, a_7, 14); /* substring, line 142 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 142 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m2 = z->l - z->c; (void)m2; /* not, line 146 */
+ if (!(eq_s_b(z, 2, s_12))) goto lab0;
+ return 0;
+ lab0:
+ z->c = z->l - m2;
+ }
+ break;
+ }
+ { int ret = slice_del(z); /* delete, line 151 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_i_plural(struct SN_env * z) {
+ { int mlimit; /* setlimit, line 154 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 154 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 154 */
+ if (z->c <= z->lb || (z->p[z->c - 1] != 105 && z->p[z->c - 1] != 106)) { z->lb = mlimit; return 0; }
+ if (!(find_among_b(z, a_8, 2))) { z->lb = mlimit; return 0; } /* substring, line 154 */
+ z->bra = z->c; /* ], line 154 */
+ z->lb = mlimit;
+ }
+ { int ret = slice_del(z); /* delete, line 158 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_t_plural(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 161 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 161 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 162 */
+ if (!(eq_s_b(z, 1, s_13))) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 162 */
+ { int m_test = z->l - z->c; /* test, line 162 */
+ if (in_grouping_b_U(z, g_V1, 97, 246, 0)) { z->lb = mlimit; return 0; }
+ z->c = z->l - m_test;
+ }
+ { int ret = slice_del(z); /* delete, line 163 */
+ if (ret < 0) return ret;
+ }
+ z->lb = mlimit;
+ }
+ { int mlimit; /* setlimit, line 165 */
+ int m2 = z->l - z->c; (void)m2;
+ if (z->c < z->I[1]) return 0;
+ z->c = z->I[1]; /* tomark, line 165 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m2;
+ z->ket = z->c; /* [, line 165 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 97) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_9, 2); /* substring, line 165 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 165 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m3 = z->l - z->c; (void)m3; /* not, line 167 */
+ if (!(eq_s_b(z, 2, s_14))) goto lab0;
+ return 0;
+ lab0:
+ z->c = z->l - m3;
+ }
+ break;
+ }
+ { int ret = slice_del(z); /* delete, line 170 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_tidy(struct SN_env * z) {
+ { int mlimit; /* setlimit, line 173 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 173 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* do, line 174 */
+ { int m3 = z->l - z->c; (void)m3; /* and, line 174 */
+ { int ret = r_LONG(z);
+ if (ret == 0) goto lab0; /* call LONG, line 174 */
+ if (ret < 0) return ret;
+ }
+ z->c = z->l - m3;
+ z->ket = z->c; /* [, line 174 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 174 */
+ }
+ z->bra = z->c; /* ], line 174 */
+ { int ret = slice_del(z); /* delete, line 174 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab0:
+ z->c = z->l - m2;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 175 */
+ z->ket = z->c; /* [, line 175 */
+ if (in_grouping_b_U(z, g_AEI, 97, 228, 0)) goto lab1;
+ z->bra = z->c; /* ], line 175 */
+ if (out_grouping_b_U(z, g_V1, 97, 246, 0)) goto lab1;
+ { int ret = slice_del(z); /* delete, line 175 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m4;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 176 */
+ z->ket = z->c; /* [, line 176 */
+ if (!(eq_s_b(z, 1, s_15))) goto lab2;
+ z->bra = z->c; /* ], line 176 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 176 */
+ if (!(eq_s_b(z, 1, s_16))) goto lab4;
+ goto lab3;
+ lab4:
+ z->c = z->l - m6;
+ if (!(eq_s_b(z, 1, s_17))) goto lab2;
+ }
+ lab3:
+ { int ret = slice_del(z); /* delete, line 176 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m5;
+ }
+ { int m7 = z->l - z->c; (void)m7; /* do, line 177 */
+ z->ket = z->c; /* [, line 177 */
+ if (!(eq_s_b(z, 1, s_18))) goto lab5;
+ z->bra = z->c; /* ], line 177 */
+ if (!(eq_s_b(z, 1, s_19))) goto lab5;
+ { int ret = slice_del(z); /* delete, line 177 */
+ if (ret < 0) return ret;
+ }
+ lab5:
+ z->c = z->l - m7;
+ }
+ z->lb = mlimit;
+ }
+ if (in_grouping_b_U(z, g_V1, 97, 246, 1) < 0) return 0; /* goto */ /* non V1, line 179 */
+ z->ket = z->c; /* [, line 179 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 179 */
+ }
+ z->bra = z->c; /* ], line 179 */
+ z->S[0] = slice_to(z, z->S[0]); /* -> x, line 179 */
+ if (z->S[0] == 0) return -1; /* -> x, line 179 */
+ if (!(eq_v_b(z, z->S[0]))) return 0; /* name x, line 179 */
+ { int ret = slice_del(z); /* delete, line 179 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+extern int finnish_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 185 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 185 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->B[0] = 0; /* unset ending_removed, line 186 */
+ z->lb = z->c; z->c = z->l; /* backwards, line 187 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 188 */
+ { int ret = r_particle_etc(z);
+ if (ret == 0) goto lab1; /* call particle_etc, line 188 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 189 */
+ { int ret = r_possessive(z);
+ if (ret == 0) goto lab2; /* call possessive, line 189 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 190 */
+ { int ret = r_case_ending(z);
+ if (ret == 0) goto lab3; /* call case_ending, line 190 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = z->l - m4;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 191 */
+ { int ret = r_other_endings(z);
+ if (ret == 0) goto lab4; /* call other_endings, line 191 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ z->c = z->l - m5;
+ }
+ { int m6 = z->l - z->c; (void)m6; /* or, line 192 */
+ if (!(z->B[0])) goto lab6; /* Boolean test ending_removed, line 192 */
+ { int m7 = z->l - z->c; (void)m7; /* do, line 192 */
+ { int ret = r_i_plural(z);
+ if (ret == 0) goto lab7; /* call i_plural, line 192 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = z->l - m7;
+ }
+ goto lab5;
+ lab6:
+ z->c = z->l - m6;
+ { int m8 = z->l - z->c; (void)m8; /* do, line 192 */
+ { int ret = r_t_plural(z);
+ if (ret == 0) goto lab8; /* call t_plural, line 192 */
+ if (ret < 0) return ret;
+ }
+ lab8:
+ z->c = z->l - m8;
+ }
+ }
+lab5:
+ { int m9 = z->l - z->c; (void)m9; /* do, line 193 */
+ { int ret = r_tidy(z);
+ if (ret == 0) goto lab9; /* call tidy, line 193 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = z->l - m9;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern struct SN_env * finnish_UTF_8_create_env(void) { return SN_create_env(1, 2, 1); }
+
+extern void finnish_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 1); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_finnish.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_finnish.h
new file mode 100644
index 0000000..d2f2fd9
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_finnish.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * finnish_UTF_8_create_env(void);
+extern void finnish_UTF_8_close_env(struct SN_env * z);
+
+extern int finnish_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_french.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_french.c
new file mode 100644
index 0000000..55b5fbf
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_french.c
@@ -0,0 +1,1256 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int french_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_un_accent(struct SN_env * z);
+static int r_un_double(struct SN_env * z);
+static int r_residual_suffix(struct SN_env * z);
+static int r_verb_suffix(struct SN_env * z);
+static int r_i_verb_suffix(struct SN_env * z);
+static int r_standard_suffix(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_RV(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * french_UTF_8_create_env(void);
+extern void french_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[3] = { 'c', 'o', 'l' };
+static const symbol s_0_1[3] = { 'p', 'a', 'r' };
+static const symbol s_0_2[3] = { 't', 'a', 'p' };
+
+static const struct among a_0[3] =
+{
+/* 0 */ { 3, s_0_0, -1, -1, 0},
+/* 1 */ { 3, s_0_1, -1, -1, 0},
+/* 2 */ { 3, s_0_2, -1, -1, 0}
+};
+
+static const symbol s_1_1[1] = { 'I' };
+static const symbol s_1_2[1] = { 'U' };
+static const symbol s_1_3[1] = { 'Y' };
+
+static const struct among a_1[4] =
+{
+/* 0 */ { 0, 0, -1, 4, 0},
+/* 1 */ { 1, s_1_1, 0, 1, 0},
+/* 2 */ { 1, s_1_2, 0, 2, 0},
+/* 3 */ { 1, s_1_3, 0, 3, 0}
+};
+
+static const symbol s_2_0[3] = { 'i', 'q', 'U' };
+static const symbol s_2_1[3] = { 'a', 'b', 'l' };
+static const symbol s_2_2[4] = { 'I', 0xC3, 0xA8, 'r' };
+static const symbol s_2_3[4] = { 'i', 0xC3, 0xA8, 'r' };
+static const symbol s_2_4[3] = { 'e', 'u', 's' };
+static const symbol s_2_5[2] = { 'i', 'v' };
+
+static const struct among a_2[6] =
+{
+/* 0 */ { 3, s_2_0, -1, 3, 0},
+/* 1 */ { 3, s_2_1, -1, 3, 0},
+/* 2 */ { 4, s_2_2, -1, 4, 0},
+/* 3 */ { 4, s_2_3, -1, 4, 0},
+/* 4 */ { 3, s_2_4, -1, 2, 0},
+/* 5 */ { 2, s_2_5, -1, 1, 0}
+};
+
+static const symbol s_3_0[2] = { 'i', 'c' };
+static const symbol s_3_1[4] = { 'a', 'b', 'i', 'l' };
+static const symbol s_3_2[2] = { 'i', 'v' };
+
+static const struct among a_3[3] =
+{
+/* 0 */ { 2, s_3_0, -1, 2, 0},
+/* 1 */ { 4, s_3_1, -1, 1, 0},
+/* 2 */ { 2, s_3_2, -1, 3, 0}
+};
+
+static const symbol s_4_0[4] = { 'i', 'q', 'U', 'e' };
+static const symbol s_4_1[6] = { 'a', 't', 'r', 'i', 'c', 'e' };
+static const symbol s_4_2[4] = { 'a', 'n', 'c', 'e' };
+static const symbol s_4_3[4] = { 'e', 'n', 'c', 'e' };
+static const symbol s_4_4[5] = { 'l', 'o', 'g', 'i', 'e' };
+static const symbol s_4_5[4] = { 'a', 'b', 'l', 'e' };
+static const symbol s_4_6[4] = { 'i', 's', 'm', 'e' };
+static const symbol s_4_7[4] = { 'e', 'u', 's', 'e' };
+static const symbol s_4_8[4] = { 'i', 's', 't', 'e' };
+static const symbol s_4_9[3] = { 'i', 'v', 'e' };
+static const symbol s_4_10[2] = { 'i', 'f' };
+static const symbol s_4_11[5] = { 'u', 's', 'i', 'o', 'n' };
+static const symbol s_4_12[5] = { 'a', 't', 'i', 'o', 'n' };
+static const symbol s_4_13[5] = { 'u', 't', 'i', 'o', 'n' };
+static const symbol s_4_14[5] = { 'a', 't', 'e', 'u', 'r' };
+static const symbol s_4_15[5] = { 'i', 'q', 'U', 'e', 's' };
+static const symbol s_4_16[7] = { 'a', 't', 'r', 'i', 'c', 'e', 's' };
+static const symbol s_4_17[5] = { 'a', 'n', 'c', 'e', 's' };
+static const symbol s_4_18[5] = { 'e', 'n', 'c', 'e', 's' };
+static const symbol s_4_19[6] = { 'l', 'o', 'g', 'i', 'e', 's' };
+static const symbol s_4_20[5] = { 'a', 'b', 'l', 'e', 's' };
+static const symbol s_4_21[5] = { 'i', 's', 'm', 'e', 's' };
+static const symbol s_4_22[5] = { 'e', 'u', 's', 'e', 's' };
+static const symbol s_4_23[5] = { 'i', 's', 't', 'e', 's' };
+static const symbol s_4_24[4] = { 'i', 'v', 'e', 's' };
+static const symbol s_4_25[3] = { 'i', 'f', 's' };
+static const symbol s_4_26[6] = { 'u', 's', 'i', 'o', 'n', 's' };
+static const symbol s_4_27[6] = { 'a', 't', 'i', 'o', 'n', 's' };
+static const symbol s_4_28[6] = { 'u', 't', 'i', 'o', 'n', 's' };
+static const symbol s_4_29[6] = { 'a', 't', 'e', 'u', 'r', 's' };
+static const symbol s_4_30[5] = { 'm', 'e', 'n', 't', 's' };
+static const symbol s_4_31[6] = { 'e', 'm', 'e', 'n', 't', 's' };
+static const symbol s_4_32[9] = { 'i', 's', 's', 'e', 'm', 'e', 'n', 't', 's' };
+static const symbol s_4_33[5] = { 'i', 't', 0xC3, 0xA9, 's' };
+static const symbol s_4_34[4] = { 'm', 'e', 'n', 't' };
+static const symbol s_4_35[5] = { 'e', 'm', 'e', 'n', 't' };
+static const symbol s_4_36[8] = { 'i', 's', 's', 'e', 'm', 'e', 'n', 't' };
+static const symbol s_4_37[6] = { 'a', 'm', 'm', 'e', 'n', 't' };
+static const symbol s_4_38[6] = { 'e', 'm', 'm', 'e', 'n', 't' };
+static const symbol s_4_39[3] = { 'a', 'u', 'x' };
+static const symbol s_4_40[4] = { 'e', 'a', 'u', 'x' };
+static const symbol s_4_41[3] = { 'e', 'u', 'x' };
+static const symbol s_4_42[4] = { 'i', 't', 0xC3, 0xA9 };
+
+static const struct among a_4[43] =
+{
+/* 0 */ { 4, s_4_0, -1, 1, 0},
+/* 1 */ { 6, s_4_1, -1, 2, 0},
+/* 2 */ { 4, s_4_2, -1, 1, 0},
+/* 3 */ { 4, s_4_3, -1, 5, 0},
+/* 4 */ { 5, s_4_4, -1, 3, 0},
+/* 5 */ { 4, s_4_5, -1, 1, 0},
+/* 6 */ { 4, s_4_6, -1, 1, 0},
+/* 7 */ { 4, s_4_7, -1, 11, 0},
+/* 8 */ { 4, s_4_8, -1, 1, 0},
+/* 9 */ { 3, s_4_9, -1, 8, 0},
+/* 10 */ { 2, s_4_10, -1, 8, 0},
+/* 11 */ { 5, s_4_11, -1, 4, 0},
+/* 12 */ { 5, s_4_12, -1, 2, 0},
+/* 13 */ { 5, s_4_13, -1, 4, 0},
+/* 14 */ { 5, s_4_14, -1, 2, 0},
+/* 15 */ { 5, s_4_15, -1, 1, 0},
+/* 16 */ { 7, s_4_16, -1, 2, 0},
+/* 17 */ { 5, s_4_17, -1, 1, 0},
+/* 18 */ { 5, s_4_18, -1, 5, 0},
+/* 19 */ { 6, s_4_19, -1, 3, 0},
+/* 20 */ { 5, s_4_20, -1, 1, 0},
+/* 21 */ { 5, s_4_21, -1, 1, 0},
+/* 22 */ { 5, s_4_22, -1, 11, 0},
+/* 23 */ { 5, s_4_23, -1, 1, 0},
+/* 24 */ { 4, s_4_24, -1, 8, 0},
+/* 25 */ { 3, s_4_25, -1, 8, 0},
+/* 26 */ { 6, s_4_26, -1, 4, 0},
+/* 27 */ { 6, s_4_27, -1, 2, 0},
+/* 28 */ { 6, s_4_28, -1, 4, 0},
+/* 29 */ { 6, s_4_29, -1, 2, 0},
+/* 30 */ { 5, s_4_30, -1, 15, 0},
+/* 31 */ { 6, s_4_31, 30, 6, 0},
+/* 32 */ { 9, s_4_32, 31, 12, 0},
+/* 33 */ { 5, s_4_33, -1, 7, 0},
+/* 34 */ { 4, s_4_34, -1, 15, 0},
+/* 35 */ { 5, s_4_35, 34, 6, 0},
+/* 36 */ { 8, s_4_36, 35, 12, 0},
+/* 37 */ { 6, s_4_37, 34, 13, 0},
+/* 38 */ { 6, s_4_38, 34, 14, 0},
+/* 39 */ { 3, s_4_39, -1, 10, 0},
+/* 40 */ { 4, s_4_40, 39, 9, 0},
+/* 41 */ { 3, s_4_41, -1, 1, 0},
+/* 42 */ { 4, s_4_42, -1, 7, 0}
+};
+
+static const symbol s_5_0[3] = { 'i', 'r', 'a' };
+static const symbol s_5_1[2] = { 'i', 'e' };
+static const symbol s_5_2[4] = { 'i', 's', 's', 'e' };
+static const symbol s_5_3[7] = { 'i', 's', 's', 'a', 'n', 't', 'e' };
+static const symbol s_5_4[1] = { 'i' };
+static const symbol s_5_5[4] = { 'i', 'r', 'a', 'i' };
+static const symbol s_5_6[2] = { 'i', 'r' };
+static const symbol s_5_7[4] = { 'i', 'r', 'a', 's' };
+static const symbol s_5_8[3] = { 'i', 'e', 's' };
+static const symbol s_5_9[5] = { 0xC3, 0xAE, 'm', 'e', 's' };
+static const symbol s_5_10[5] = { 'i', 's', 's', 'e', 's' };
+static const symbol s_5_11[8] = { 'i', 's', 's', 'a', 'n', 't', 'e', 's' };
+static const symbol s_5_12[5] = { 0xC3, 0xAE, 't', 'e', 's' };
+static const symbol s_5_13[2] = { 'i', 's' };
+static const symbol s_5_14[5] = { 'i', 'r', 'a', 'i', 's' };
+static const symbol s_5_15[6] = { 'i', 's', 's', 'a', 'i', 's' };
+static const symbol s_5_16[6] = { 'i', 'r', 'i', 'o', 'n', 's' };
+static const symbol s_5_17[7] = { 'i', 's', 's', 'i', 'o', 'n', 's' };
+static const symbol s_5_18[5] = { 'i', 'r', 'o', 'n', 's' };
+static const symbol s_5_19[6] = { 'i', 's', 's', 'o', 'n', 's' };
+static const symbol s_5_20[7] = { 'i', 's', 's', 'a', 'n', 't', 's' };
+static const symbol s_5_21[2] = { 'i', 't' };
+static const symbol s_5_22[5] = { 'i', 'r', 'a', 'i', 't' };
+static const symbol s_5_23[6] = { 'i', 's', 's', 'a', 'i', 't' };
+static const symbol s_5_24[6] = { 'i', 's', 's', 'a', 'n', 't' };
+static const symbol s_5_25[7] = { 'i', 'r', 'a', 'I', 'e', 'n', 't' };
+static const symbol s_5_26[8] = { 'i', 's', 's', 'a', 'I', 'e', 'n', 't' };
+static const symbol s_5_27[5] = { 'i', 'r', 'e', 'n', 't' };
+static const symbol s_5_28[6] = { 'i', 's', 's', 'e', 'n', 't' };
+static const symbol s_5_29[5] = { 'i', 'r', 'o', 'n', 't' };
+static const symbol s_5_30[3] = { 0xC3, 0xAE, 't' };
+static const symbol s_5_31[5] = { 'i', 'r', 'i', 'e', 'z' };
+static const symbol s_5_32[6] = { 'i', 's', 's', 'i', 'e', 'z' };
+static const symbol s_5_33[4] = { 'i', 'r', 'e', 'z' };
+static const symbol s_5_34[5] = { 'i', 's', 's', 'e', 'z' };
+
+static const struct among a_5[35] =
+{
+/* 0 */ { 3, s_5_0, -1, 1, 0},
+/* 1 */ { 2, s_5_1, -1, 1, 0},
+/* 2 */ { 4, s_5_2, -1, 1, 0},
+/* 3 */ { 7, s_5_3, -1, 1, 0},
+/* 4 */ { 1, s_5_4, -1, 1, 0},
+/* 5 */ { 4, s_5_5, 4, 1, 0},
+/* 6 */ { 2, s_5_6, -1, 1, 0},
+/* 7 */ { 4, s_5_7, -1, 1, 0},
+/* 8 */ { 3, s_5_8, -1, 1, 0},
+/* 9 */ { 5, s_5_9, -1, 1, 0},
+/* 10 */ { 5, s_5_10, -1, 1, 0},
+/* 11 */ { 8, s_5_11, -1, 1, 0},
+/* 12 */ { 5, s_5_12, -1, 1, 0},
+/* 13 */ { 2, s_5_13, -1, 1, 0},
+/* 14 */ { 5, s_5_14, 13, 1, 0},
+/* 15 */ { 6, s_5_15, 13, 1, 0},
+/* 16 */ { 6, s_5_16, -1, 1, 0},
+/* 17 */ { 7, s_5_17, -1, 1, 0},
+/* 18 */ { 5, s_5_18, -1, 1, 0},
+/* 19 */ { 6, s_5_19, -1, 1, 0},
+/* 20 */ { 7, s_5_20, -1, 1, 0},
+/* 21 */ { 2, s_5_21, -1, 1, 0},
+/* 22 */ { 5, s_5_22, 21, 1, 0},
+/* 23 */ { 6, s_5_23, 21, 1, 0},
+/* 24 */ { 6, s_5_24, -1, 1, 0},
+/* 25 */ { 7, s_5_25, -1, 1, 0},
+/* 26 */ { 8, s_5_26, -1, 1, 0},
+/* 27 */ { 5, s_5_27, -1, 1, 0},
+/* 28 */ { 6, s_5_28, -1, 1, 0},
+/* 29 */ { 5, s_5_29, -1, 1, 0},
+/* 30 */ { 3, s_5_30, -1, 1, 0},
+/* 31 */ { 5, s_5_31, -1, 1, 0},
+/* 32 */ { 6, s_5_32, -1, 1, 0},
+/* 33 */ { 4, s_5_33, -1, 1, 0},
+/* 34 */ { 5, s_5_34, -1, 1, 0}
+};
+
+static const symbol s_6_0[1] = { 'a' };
+static const symbol s_6_1[3] = { 'e', 'r', 'a' };
+static const symbol s_6_2[4] = { 'a', 's', 's', 'e' };
+static const symbol s_6_3[4] = { 'a', 'n', 't', 'e' };
+static const symbol s_6_4[3] = { 0xC3, 0xA9, 'e' };
+static const symbol s_6_5[2] = { 'a', 'i' };
+static const symbol s_6_6[4] = { 'e', 'r', 'a', 'i' };
+static const symbol s_6_7[2] = { 'e', 'r' };
+static const symbol s_6_8[2] = { 'a', 's' };
+static const symbol s_6_9[4] = { 'e', 'r', 'a', 's' };
+static const symbol s_6_10[5] = { 0xC3, 0xA2, 'm', 'e', 's' };
+static const symbol s_6_11[5] = { 'a', 's', 's', 'e', 's' };
+static const symbol s_6_12[5] = { 'a', 'n', 't', 'e', 's' };
+static const symbol s_6_13[5] = { 0xC3, 0xA2, 't', 'e', 's' };
+static const symbol s_6_14[4] = { 0xC3, 0xA9, 'e', 's' };
+static const symbol s_6_15[3] = { 'a', 'i', 's' };
+static const symbol s_6_16[5] = { 'e', 'r', 'a', 'i', 's' };
+static const symbol s_6_17[4] = { 'i', 'o', 'n', 's' };
+static const symbol s_6_18[6] = { 'e', 'r', 'i', 'o', 'n', 's' };
+static const symbol s_6_19[7] = { 'a', 's', 's', 'i', 'o', 'n', 's' };
+static const symbol s_6_20[5] = { 'e', 'r', 'o', 'n', 's' };
+static const symbol s_6_21[4] = { 'a', 'n', 't', 's' };
+static const symbol s_6_22[3] = { 0xC3, 0xA9, 's' };
+static const symbol s_6_23[3] = { 'a', 'i', 't' };
+static const symbol s_6_24[5] = { 'e', 'r', 'a', 'i', 't' };
+static const symbol s_6_25[3] = { 'a', 'n', 't' };
+static const symbol s_6_26[5] = { 'a', 'I', 'e', 'n', 't' };
+static const symbol s_6_27[7] = { 'e', 'r', 'a', 'I', 'e', 'n', 't' };
+static const symbol s_6_28[6] = { 0xC3, 0xA8, 'r', 'e', 'n', 't' };
+static const symbol s_6_29[6] = { 'a', 's', 's', 'e', 'n', 't' };
+static const symbol s_6_30[5] = { 'e', 'r', 'o', 'n', 't' };
+static const symbol s_6_31[3] = { 0xC3, 0xA2, 't' };
+static const symbol s_6_32[2] = { 'e', 'z' };
+static const symbol s_6_33[3] = { 'i', 'e', 'z' };
+static const symbol s_6_34[5] = { 'e', 'r', 'i', 'e', 'z' };
+static const symbol s_6_35[6] = { 'a', 's', 's', 'i', 'e', 'z' };
+static const symbol s_6_36[4] = { 'e', 'r', 'e', 'z' };
+static const symbol s_6_37[2] = { 0xC3, 0xA9 };
+
+static const struct among a_6[38] =
+{
+/* 0 */ { 1, s_6_0, -1, 3, 0},
+/* 1 */ { 3, s_6_1, 0, 2, 0},
+/* 2 */ { 4, s_6_2, -1, 3, 0},
+/* 3 */ { 4, s_6_3, -1, 3, 0},
+/* 4 */ { 3, s_6_4, -1, 2, 0},
+/* 5 */ { 2, s_6_5, -1, 3, 0},
+/* 6 */ { 4, s_6_6, 5, 2, 0},
+/* 7 */ { 2, s_6_7, -1, 2, 0},
+/* 8 */ { 2, s_6_8, -1, 3, 0},
+/* 9 */ { 4, s_6_9, 8, 2, 0},
+/* 10 */ { 5, s_6_10, -1, 3, 0},
+/* 11 */ { 5, s_6_11, -1, 3, 0},
+/* 12 */ { 5, s_6_12, -1, 3, 0},
+/* 13 */ { 5, s_6_13, -1, 3, 0},
+/* 14 */ { 4, s_6_14, -1, 2, 0},
+/* 15 */ { 3, s_6_15, -1, 3, 0},
+/* 16 */ { 5, s_6_16, 15, 2, 0},
+/* 17 */ { 4, s_6_17, -1, 1, 0},
+/* 18 */ { 6, s_6_18, 17, 2, 0},
+/* 19 */ { 7, s_6_19, 17, 3, 0},
+/* 20 */ { 5, s_6_20, -1, 2, 0},
+/* 21 */ { 4, s_6_21, -1, 3, 0},
+/* 22 */ { 3, s_6_22, -1, 2, 0},
+/* 23 */ { 3, s_6_23, -1, 3, 0},
+/* 24 */ { 5, s_6_24, 23, 2, 0},
+/* 25 */ { 3, s_6_25, -1, 3, 0},
+/* 26 */ { 5, s_6_26, -1, 3, 0},
+/* 27 */ { 7, s_6_27, 26, 2, 0},
+/* 28 */ { 6, s_6_28, -1, 2, 0},
+/* 29 */ { 6, s_6_29, -1, 3, 0},
+/* 30 */ { 5, s_6_30, -1, 2, 0},
+/* 31 */ { 3, s_6_31, -1, 3, 0},
+/* 32 */ { 2, s_6_32, -1, 2, 0},
+/* 33 */ { 3, s_6_33, 32, 2, 0},
+/* 34 */ { 5, s_6_34, 33, 2, 0},
+/* 35 */ { 6, s_6_35, 33, 3, 0},
+/* 36 */ { 4, s_6_36, 32, 2, 0},
+/* 37 */ { 2, s_6_37, -1, 2, 0}
+};
+
+static const symbol s_7_0[1] = { 'e' };
+static const symbol s_7_1[5] = { 'I', 0xC3, 0xA8, 'r', 'e' };
+static const symbol s_7_2[5] = { 'i', 0xC3, 0xA8, 'r', 'e' };
+static const symbol s_7_3[3] = { 'i', 'o', 'n' };
+static const symbol s_7_4[3] = { 'I', 'e', 'r' };
+static const symbol s_7_5[3] = { 'i', 'e', 'r' };
+static const symbol s_7_6[2] = { 0xC3, 0xAB };
+
+static const struct among a_7[7] =
+{
+/* 0 */ { 1, s_7_0, -1, 3, 0},
+/* 1 */ { 5, s_7_1, 0, 2, 0},
+/* 2 */ { 5, s_7_2, 0, 2, 0},
+/* 3 */ { 3, s_7_3, -1, 1, 0},
+/* 4 */ { 3, s_7_4, -1, 2, 0},
+/* 5 */ { 3, s_7_5, -1, 2, 0},
+/* 6 */ { 2, s_7_6, -1, 4, 0}
+};
+
+static const symbol s_8_0[3] = { 'e', 'l', 'l' };
+static const symbol s_8_1[4] = { 'e', 'i', 'l', 'l' };
+static const symbol s_8_2[3] = { 'e', 'n', 'n' };
+static const symbol s_8_3[3] = { 'o', 'n', 'n' };
+static const symbol s_8_4[3] = { 'e', 't', 't' };
+
+static const struct among a_8[5] =
+{
+/* 0 */ { 3, s_8_0, -1, -1, 0},
+/* 1 */ { 4, s_8_1, -1, -1, 0},
+/* 2 */ { 3, s_8_2, -1, -1, 0},
+/* 3 */ { 3, s_8_3, -1, -1, 0},
+/* 4 */ { 3, s_8_4, -1, -1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 130, 103, 8, 5 };
+
+static const unsigned char g_keep_with_s[] = { 1, 65, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128 };
+
+static const symbol s_0[] = { 'u' };
+static const symbol s_1[] = { 'U' };
+static const symbol s_2[] = { 'i' };
+static const symbol s_3[] = { 'I' };
+static const symbol s_4[] = { 'y' };
+static const symbol s_5[] = { 'Y' };
+static const symbol s_6[] = { 'y' };
+static const symbol s_7[] = { 'Y' };
+static const symbol s_8[] = { 'q' };
+static const symbol s_9[] = { 'u' };
+static const symbol s_10[] = { 'U' };
+static const symbol s_11[] = { 'i' };
+static const symbol s_12[] = { 'u' };
+static const symbol s_13[] = { 'y' };
+static const symbol s_14[] = { 'i', 'c' };
+static const symbol s_15[] = { 'i', 'q', 'U' };
+static const symbol s_16[] = { 'l', 'o', 'g' };
+static const symbol s_17[] = { 'u' };
+static const symbol s_18[] = { 'e', 'n', 't' };
+static const symbol s_19[] = { 'a', 't' };
+static const symbol s_20[] = { 'e', 'u', 'x' };
+static const symbol s_21[] = { 'i' };
+static const symbol s_22[] = { 'a', 'b', 'l' };
+static const symbol s_23[] = { 'i', 'q', 'U' };
+static const symbol s_24[] = { 'a', 't' };
+static const symbol s_25[] = { 'i', 'c' };
+static const symbol s_26[] = { 'i', 'q', 'U' };
+static const symbol s_27[] = { 'e', 'a', 'u' };
+static const symbol s_28[] = { 'a', 'l' };
+static const symbol s_29[] = { 'e', 'u', 'x' };
+static const symbol s_30[] = { 'a', 'n', 't' };
+static const symbol s_31[] = { 'e', 'n', 't' };
+static const symbol s_32[] = { 'e' };
+static const symbol s_33[] = { 's' };
+static const symbol s_34[] = { 's' };
+static const symbol s_35[] = { 't' };
+static const symbol s_36[] = { 'i' };
+static const symbol s_37[] = { 'g', 'u' };
+static const symbol s_38[] = { 0xC3, 0xA9 };
+static const symbol s_39[] = { 0xC3, 0xA8 };
+static const symbol s_40[] = { 'e' };
+static const symbol s_41[] = { 'Y' };
+static const symbol s_42[] = { 'i' };
+static const symbol s_43[] = { 0xC3, 0xA7 };
+static const symbol s_44[] = { 'c' };
+
+static int r_prelude(struct SN_env * z) {
+ while(1) { /* repeat, line 38 */
+ int c1 = z->c;
+ while(1) { /* goto, line 38 */
+ int c2 = z->c;
+ { int c3 = z->c; /* or, line 44 */
+ if (in_grouping_U(z, g_v, 97, 251, 0)) goto lab3;
+ z->bra = z->c; /* [, line 40 */
+ { int c4 = z->c; /* or, line 40 */
+ if (!(eq_s(z, 1, s_0))) goto lab5;
+ z->ket = z->c; /* ], line 40 */
+ if (in_grouping_U(z, g_v, 97, 251, 0)) goto lab5;
+ { int ret = slice_from_s(z, 1, s_1); /* <-, line 40 */
+ if (ret < 0) return ret;
+ }
+ goto lab4;
+ lab5:
+ z->c = c4;
+ if (!(eq_s(z, 1, s_2))) goto lab6;
+ z->ket = z->c; /* ], line 41 */
+ if (in_grouping_U(z, g_v, 97, 251, 0)) goto lab6;
+ { int ret = slice_from_s(z, 1, s_3); /* <-, line 41 */
+ if (ret < 0) return ret;
+ }
+ goto lab4;
+ lab6:
+ z->c = c4;
+ if (!(eq_s(z, 1, s_4))) goto lab3;
+ z->ket = z->c; /* ], line 42 */
+ { int ret = slice_from_s(z, 1, s_5); /* <-, line 42 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab4:
+ goto lab2;
+ lab3:
+ z->c = c3;
+ z->bra = z->c; /* [, line 45 */
+ if (!(eq_s(z, 1, s_6))) goto lab7;
+ z->ket = z->c; /* ], line 45 */
+ if (in_grouping_U(z, g_v, 97, 251, 0)) goto lab7;
+ { int ret = slice_from_s(z, 1, s_7); /* <-, line 45 */
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab7:
+ z->c = c3;
+ if (!(eq_s(z, 1, s_8))) goto lab1;
+ z->bra = z->c; /* [, line 47 */
+ if (!(eq_s(z, 1, s_9))) goto lab1;
+ z->ket = z->c; /* ], line 47 */
+ { int ret = slice_from_s(z, 1, s_10); /* <-, line 47 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab2:
+ z->c = c2;
+ break;
+ lab1:
+ z->c = c2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* goto, line 38 */
+ }
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ z->I[2] = z->l;
+ { int c1 = z->c; /* do, line 56 */
+ { int c2 = z->c; /* or, line 58 */
+ if (in_grouping_U(z, g_v, 97, 251, 0)) goto lab2;
+ if (in_grouping_U(z, g_v, 97, 251, 0)) goto lab2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab2;
+ z->c = ret; /* next, line 57 */
+ }
+ goto lab1;
+ lab2:
+ z->c = c2;
+ if (z->c + 2 >= z->l || z->p[z->c + 2] >> 5 != 3 || !((331776 >> (z->p[z->c + 2] & 0x1f)) & 1)) goto lab3;
+ if (!(find_among(z, a_0, 3))) goto lab3; /* among, line 59 */
+ goto lab1;
+ lab3:
+ z->c = c2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 66 */
+ }
+ { /* gopast */ /* grouping v, line 66 */
+ int ret = out_grouping_U(z, g_v, 97, 251, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ }
+ lab1:
+ z->I[0] = z->c; /* setmark pV, line 67 */
+ lab0:
+ z->c = c1;
+ }
+ { int c3 = z->c; /* do, line 69 */
+ { /* gopast */ /* grouping v, line 70 */
+ int ret = out_grouping_U(z, g_v, 97, 251, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 70 */
+ int ret = in_grouping_U(z, g_v, 97, 251, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p1, line 70 */
+ { /* gopast */ /* grouping v, line 71 */
+ int ret = out_grouping_U(z, g_v, 97, 251, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 71 */
+ int ret = in_grouping_U(z, g_v, 97, 251, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ z->I[2] = z->c; /* setmark p2, line 71 */
+ lab4:
+ z->c = c3;
+ }
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 75 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 77 */
+ if (z->c >= z->l || z->p[z->c + 0] >> 5 != 2 || !((35652096 >> (z->p[z->c + 0] & 0x1f)) & 1)) among_var = 4; else
+ among_var = find_among(z, a_1, 4); /* substring, line 77 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 77 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_11); /* <-, line 78 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_12); /* <-, line 79 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_13); /* <-, line 80 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 81 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_RV(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[2] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 92 */
+ among_var = find_among_b(z, a_4, 43); /* substring, line 92 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 92 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 96 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 96 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 99 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 99 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 100 */
+ z->ket = z->c; /* [, line 100 */
+ if (!(eq_s_b(z, 2, s_14))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 100 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 100 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab2; /* call R2, line 100 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 100 */
+ if (ret < 0) return ret;
+ }
+ goto lab1;
+ lab2:
+ z->c = z->l - m1;
+ { int ret = slice_from_s(z, 3, s_15); /* <-, line 100 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab1:
+ lab0:
+ ;
+ }
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 104 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_16); /* <-, line 104 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 107 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 1, s_17); /* <-, line 107 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 110 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_18); /* <-, line 110 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 114 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 114 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 115 */
+ z->ket = z->c; /* [, line 116 */
+ among_var = find_among_b(z, a_2, 6); /* substring, line 116 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab3; }
+ z->bra = z->c; /* ], line 116 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab3; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 117 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 117 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 117 */
+ if (!(eq_s_b(z, 2, s_19))) { z->c = z->l - m_keep; goto lab3; }
+ z->bra = z->c; /* ], line 117 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 117 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 117 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m2 = z->l - z->c; (void)m2; /* or, line 118 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab5; /* call R2, line 118 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 118 */
+ if (ret < 0) return ret;
+ }
+ goto lab4;
+ lab5:
+ z->c = z->l - m2;
+ { int ret = r_R1(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R1, line 118 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_20); /* <-, line 118 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab4:
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 120 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 120 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = r_RV(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call RV, line 122 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 1, s_21); /* <-, line 122 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab3:
+ ;
+ }
+ break;
+ case 7:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 129 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 129 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 130 */
+ z->ket = z->c; /* [, line 131 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4198408 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab6; }
+ among_var = find_among_b(z, a_3, 3); /* substring, line 131 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab6; }
+ z->bra = z->c; /* ], line 131 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab6; }
+ case 1:
+ { int m3 = z->l - z->c; (void)m3; /* or, line 132 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab8; /* call R2, line 132 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 132 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab8:
+ z->c = z->l - m3;
+ { int ret = slice_from_s(z, 3, s_22); /* <-, line 132 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab7:
+ break;
+ case 2:
+ { int m4 = z->l - z->c; (void)m4; /* or, line 133 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab10; /* call R2, line 133 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 133 */
+ if (ret < 0) return ret;
+ }
+ goto lab9;
+ lab10:
+ z->c = z->l - m4;
+ { int ret = slice_from_s(z, 3, s_23); /* <-, line 133 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab9:
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab6; } /* call R2, line 134 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 134 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab6:
+ ;
+ }
+ break;
+ case 8:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 141 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 141 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 142 */
+ z->ket = z->c; /* [, line 142 */
+ if (!(eq_s_b(z, 2, s_24))) { z->c = z->l - m_keep; goto lab11; }
+ z->bra = z->c; /* ], line 142 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab11; } /* call R2, line 142 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 142 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 142 */
+ if (!(eq_s_b(z, 2, s_25))) { z->c = z->l - m_keep; goto lab11; }
+ z->bra = z->c; /* ], line 142 */
+ { int m5 = z->l - z->c; (void)m5; /* or, line 142 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab13; /* call R2, line 142 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 142 */
+ if (ret < 0) return ret;
+ }
+ goto lab12;
+ lab13:
+ z->c = z->l - m5;
+ { int ret = slice_from_s(z, 3, s_26); /* <-, line 142 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab12:
+ lab11:
+ ;
+ }
+ break;
+ case 9:
+ { int ret = slice_from_s(z, 3, s_27); /* <-, line 144 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 10:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 145 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 2, s_28); /* <-, line 145 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 11:
+ { int m6 = z->l - z->c; (void)m6; /* or, line 147 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab15; /* call R2, line 147 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 147 */
+ if (ret < 0) return ret;
+ }
+ goto lab14;
+ lab15:
+ z->c = z->l - m6;
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 147 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_29); /* <-, line 147 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab14:
+ break;
+ case 12:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 150 */
+ if (ret < 0) return ret;
+ }
+ if (out_grouping_b_U(z, g_v, 97, 251, 0)) return 0;
+ { int ret = slice_del(z); /* delete, line 150 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 13:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 155 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_30); /* <-, line 155 */
+ if (ret < 0) return ret;
+ }
+ return 0; /* fail, line 155 */
+ break;
+ case 14:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 156 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_31); /* <-, line 156 */
+ if (ret < 0) return ret;
+ }
+ return 0; /* fail, line 156 */
+ break;
+ case 15:
+ { int m_test = z->l - z->c; /* test, line 158 */
+ if (in_grouping_b_U(z, g_v, 97, 251, 0)) return 0;
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 158 */
+ if (ret < 0) return ret;
+ }
+ z->c = z->l - m_test;
+ }
+ { int ret = slice_del(z); /* delete, line 158 */
+ if (ret < 0) return ret;
+ }
+ return 0; /* fail, line 158 */
+ break;
+ }
+ return 1;
+}
+
+static int r_i_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 163 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 163 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 164 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((68944418 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_5, 35); /* substring, line 164 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 164 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ if (out_grouping_b_U(z, g_v, 97, 251, 0)) { z->lb = mlimit; return 0; }
+ { int ret = slice_del(z); /* delete, line 170 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 174 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 174 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 175 */
+ among_var = find_among_b(z, a_6, 38); /* substring, line 175 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 175 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->lb = mlimit; return 0; } /* call R2, line 177 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 177 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 185 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 190 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 191 */
+ z->ket = z->c; /* [, line 191 */
+ if (!(eq_s_b(z, 1, s_32))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 191 */
+ { int ret = slice_del(z); /* delete, line 191 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ ;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_residual_suffix(struct SN_env * z) {
+ int among_var;
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 199 */
+ z->ket = z->c; /* [, line 199 */
+ if (!(eq_s_b(z, 1, s_33))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 199 */
+ { int m_test = z->l - z->c; /* test, line 199 */
+ if (out_grouping_b_U(z, g_keep_with_s, 97, 232, 0)) { z->c = z->l - m_keep; goto lab0; }
+ z->c = z->l - m_test;
+ }
+ { int ret = slice_del(z); /* delete, line 199 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ ;
+ }
+ { int mlimit; /* setlimit, line 200 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 200 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 201 */
+ among_var = find_among_b(z, a_7, 7); /* substring, line 201 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 201 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->lb = mlimit; return 0; } /* call R2, line 202 */
+ if (ret < 0) return ret;
+ }
+ { int m2 = z->l - z->c; (void)m2; /* or, line 202 */
+ if (!(eq_s_b(z, 1, s_34))) goto lab2;
+ goto lab1;
+ lab2:
+ z->c = z->l - m2;
+ if (!(eq_s_b(z, 1, s_35))) { z->lb = mlimit; return 0; }
+ }
+ lab1:
+ { int ret = slice_del(z); /* delete, line 202 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_36); /* <-, line 204 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 205 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ if (!(eq_s_b(z, 2, s_37))) { z->lb = mlimit; return 0; }
+ { int ret = slice_del(z); /* delete, line 206 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_un_double(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 212 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1069056 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_8, 5))) return 0; /* among, line 212 */
+ z->c = z->l - m_test;
+ }
+ z->ket = z->c; /* [, line 212 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 212 */
+ }
+ z->bra = z->c; /* ], line 212 */
+ { int ret = slice_del(z); /* delete, line 212 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_un_accent(struct SN_env * z) {
+ { int i = 1;
+ while(1) { /* atleast, line 216 */
+ if (out_grouping_b_U(z, g_v, 97, 251, 0)) goto lab0;
+ i--;
+ continue;
+ lab0:
+ break;
+ }
+ if (i > 0) return 0;
+ }
+ z->ket = z->c; /* [, line 217 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 217 */
+ if (!(eq_s_b(z, 2, s_38))) goto lab2;
+ goto lab1;
+ lab2:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_39))) return 0;
+ }
+lab1:
+ z->bra = z->c; /* ], line 217 */
+ { int ret = slice_from_s(z, 1, s_40); /* <-, line 217 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+extern int french_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 223 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab0; /* call prelude, line 223 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 224 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab1; /* call mark_regions, line 224 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = c2;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 225 */
+
+ { int m3 = z->l - z->c; (void)m3; /* do, line 227 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 237 */
+ { int m5 = z->l - z->c; (void)m5; /* and, line 233 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 229 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab6; /* call standard_suffix, line 229 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = z->l - m6;
+ { int ret = r_i_verb_suffix(z);
+ if (ret == 0) goto lab7; /* call i_verb_suffix, line 230 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab7:
+ z->c = z->l - m6;
+ { int ret = r_verb_suffix(z);
+ if (ret == 0) goto lab4; /* call verb_suffix, line 231 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ z->c = z->l - m5;
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 234 */
+ z->ket = z->c; /* [, line 234 */
+ { int m7 = z->l - z->c; (void)m7; /* or, line 234 */
+ if (!(eq_s_b(z, 1, s_41))) goto lab10;
+ z->bra = z->c; /* ], line 234 */
+ { int ret = slice_from_s(z, 1, s_42); /* <-, line 234 */
+ if (ret < 0) return ret;
+ }
+ goto lab9;
+ lab10:
+ z->c = z->l - m7;
+ if (!(eq_s_b(z, 2, s_43))) { z->c = z->l - m_keep; goto lab8; }
+ z->bra = z->c; /* ], line 235 */
+ { int ret = slice_from_s(z, 1, s_44); /* <-, line 235 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab9:
+ lab8:
+ ;
+ }
+ }
+ goto lab3;
+ lab4:
+ z->c = z->l - m4;
+ { int ret = r_residual_suffix(z);
+ if (ret == 0) goto lab2; /* call residual_suffix, line 238 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab3:
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m8 = z->l - z->c; (void)m8; /* do, line 243 */
+ { int ret = r_un_double(z);
+ if (ret == 0) goto lab11; /* call un_double, line 243 */
+ if (ret < 0) return ret;
+ }
+ lab11:
+ z->c = z->l - m8;
+ }
+ { int m9 = z->l - z->c; (void)m9; /* do, line 244 */
+ { int ret = r_un_accent(z);
+ if (ret == 0) goto lab12; /* call un_accent, line 244 */
+ if (ret < 0) return ret;
+ }
+ lab12:
+ z->c = z->l - m9;
+ }
+ z->c = z->lb;
+ { int c10 = z->c; /* do, line 246 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab13; /* call postlude, line 246 */
+ if (ret < 0) return ret;
+ }
+ lab13:
+ z->c = c10;
+ }
+ return 1;
+}
+
+extern struct SN_env * french_UTF_8_create_env(void) { return SN_create_env(0, 3, 0); }
+
+extern void french_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_french.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_french.h
new file mode 100644
index 0000000..08e3418
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_french.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * french_UTF_8_create_env(void);
+extern void french_UTF_8_close_env(struct SN_env * z);
+
+extern int french_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_german.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_german.c
new file mode 100644
index 0000000..2ebe86a
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_german.c
@@ -0,0 +1,527 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int german_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_standard_suffix(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * german_UTF_8_create_env(void);
+extern void german_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_1[1] = { 'U' };
+static const symbol s_0_2[1] = { 'Y' };
+static const symbol s_0_3[2] = { 0xC3, 0xA4 };
+static const symbol s_0_4[2] = { 0xC3, 0xB6 };
+static const symbol s_0_5[2] = { 0xC3, 0xBC };
+
+static const struct among a_0[6] =
+{
+/* 0 */ { 0, 0, -1, 6, 0},
+/* 1 */ { 1, s_0_1, 0, 2, 0},
+/* 2 */ { 1, s_0_2, 0, 1, 0},
+/* 3 */ { 2, s_0_3, 0, 3, 0},
+/* 4 */ { 2, s_0_4, 0, 4, 0},
+/* 5 */ { 2, s_0_5, 0, 5, 0}
+};
+
+static const symbol s_1_0[1] = { 'e' };
+static const symbol s_1_1[2] = { 'e', 'm' };
+static const symbol s_1_2[2] = { 'e', 'n' };
+static const symbol s_1_3[3] = { 'e', 'r', 'n' };
+static const symbol s_1_4[2] = { 'e', 'r' };
+static const symbol s_1_5[1] = { 's' };
+static const symbol s_1_6[2] = { 'e', 's' };
+
+static const struct among a_1[7] =
+{
+/* 0 */ { 1, s_1_0, -1, 2, 0},
+/* 1 */ { 2, s_1_1, -1, 1, 0},
+/* 2 */ { 2, s_1_2, -1, 2, 0},
+/* 3 */ { 3, s_1_3, -1, 1, 0},
+/* 4 */ { 2, s_1_4, -1, 1, 0},
+/* 5 */ { 1, s_1_5, -1, 3, 0},
+/* 6 */ { 2, s_1_6, 5, 2, 0}
+};
+
+static const symbol s_2_0[2] = { 'e', 'n' };
+static const symbol s_2_1[2] = { 'e', 'r' };
+static const symbol s_2_2[2] = { 's', 't' };
+static const symbol s_2_3[3] = { 'e', 's', 't' };
+
+static const struct among a_2[4] =
+{
+/* 0 */ { 2, s_2_0, -1, 1, 0},
+/* 1 */ { 2, s_2_1, -1, 1, 0},
+/* 2 */ { 2, s_2_2, -1, 2, 0},
+/* 3 */ { 3, s_2_3, 2, 1, 0}
+};
+
+static const symbol s_3_0[2] = { 'i', 'g' };
+static const symbol s_3_1[4] = { 'l', 'i', 'c', 'h' };
+
+static const struct among a_3[2] =
+{
+/* 0 */ { 2, s_3_0, -1, 1, 0},
+/* 1 */ { 4, s_3_1, -1, 1, 0}
+};
+
+static const symbol s_4_0[3] = { 'e', 'n', 'd' };
+static const symbol s_4_1[2] = { 'i', 'g' };
+static const symbol s_4_2[3] = { 'u', 'n', 'g' };
+static const symbol s_4_3[4] = { 'l', 'i', 'c', 'h' };
+static const symbol s_4_4[4] = { 'i', 's', 'c', 'h' };
+static const symbol s_4_5[2] = { 'i', 'k' };
+static const symbol s_4_6[4] = { 'h', 'e', 'i', 't' };
+static const symbol s_4_7[4] = { 'k', 'e', 'i', 't' };
+
+static const struct among a_4[8] =
+{
+/* 0 */ { 3, s_4_0, -1, 1, 0},
+/* 1 */ { 2, s_4_1, -1, 2, 0},
+/* 2 */ { 3, s_4_2, -1, 1, 0},
+/* 3 */ { 4, s_4_3, -1, 3, 0},
+/* 4 */ { 4, s_4_4, -1, 2, 0},
+/* 5 */ { 2, s_4_5, -1, 2, 0},
+/* 6 */ { 4, s_4_6, -1, 3, 0},
+/* 7 */ { 4, s_4_7, -1, 4, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 32, 8 };
+
+static const unsigned char g_s_ending[] = { 117, 30, 5 };
+
+static const unsigned char g_st_ending[] = { 117, 30, 4 };
+
+static const symbol s_0[] = { 0xC3, 0x9F };
+static const symbol s_1[] = { 's', 's' };
+static const symbol s_2[] = { 'u' };
+static const symbol s_3[] = { 'U' };
+static const symbol s_4[] = { 'y' };
+static const symbol s_5[] = { 'Y' };
+static const symbol s_6[] = { 'y' };
+static const symbol s_7[] = { 'u' };
+static const symbol s_8[] = { 'a' };
+static const symbol s_9[] = { 'o' };
+static const symbol s_10[] = { 'u' };
+static const symbol s_11[] = { 's' };
+static const symbol s_12[] = { 'n', 'i', 's' };
+static const symbol s_13[] = { 'i', 'g' };
+static const symbol s_14[] = { 'e' };
+static const symbol s_15[] = { 'e' };
+static const symbol s_16[] = { 'e', 'r' };
+static const symbol s_17[] = { 'e', 'n' };
+
+static int r_prelude(struct SN_env * z) {
+ { int c_test = z->c; /* test, line 35 */
+ while(1) { /* repeat, line 35 */
+ int c1 = z->c;
+ { int c2 = z->c; /* or, line 38 */
+ z->bra = z->c; /* [, line 37 */
+ if (!(eq_s(z, 2, s_0))) goto lab2;
+ z->ket = z->c; /* ], line 37 */
+ { int ret = slice_from_s(z, 2, s_1); /* <-, line 37 */
+ if (ret < 0) return ret;
+ }
+ goto lab1;
+ lab2:
+ z->c = c2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 38 */
+ }
+ }
+ lab1:
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ z->c = c_test;
+ }
+ while(1) { /* repeat, line 41 */
+ int c3 = z->c;
+ while(1) { /* goto, line 41 */
+ int c4 = z->c;
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab4;
+ z->bra = z->c; /* [, line 42 */
+ { int c5 = z->c; /* or, line 42 */
+ if (!(eq_s(z, 1, s_2))) goto lab6;
+ z->ket = z->c; /* ], line 42 */
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab6;
+ { int ret = slice_from_s(z, 1, s_3); /* <-, line 42 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = c5;
+ if (!(eq_s(z, 1, s_4))) goto lab4;
+ z->ket = z->c; /* ], line 43 */
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab4;
+ { int ret = slice_from_s(z, 1, s_5); /* <-, line 43 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ z->c = c4;
+ break;
+ lab4:
+ z->c = c4;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab3;
+ z->c = ret; /* goto, line 41 */
+ }
+ }
+ continue;
+ lab3:
+ z->c = c3;
+ break;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ { int c_test = z->c; /* test, line 52 */
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, + 3);
+ if (ret < 0) return 0;
+ z->c = ret; /* hop, line 52 */
+ }
+ z->I[2] = z->c; /* setmark x, line 52 */
+ z->c = c_test;
+ }
+ { /* gopast */ /* grouping v, line 54 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 54 */
+ int ret = in_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 54 */
+ /* try, line 55 */
+ if (!(z->I[0] < z->I[2])) goto lab0;
+ z->I[0] = z->I[2];
+lab0:
+ { /* gopast */ /* grouping v, line 56 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 56 */
+ int ret = in_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p2, line 56 */
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 60 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 62 */
+ among_var = find_among(z, a_0, 6); /* substring, line 62 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 62 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_6); /* <-, line 63 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_7); /* <-, line 64 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_8); /* <-, line 65 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 1, s_9); /* <-, line 66 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 1, s_10); /* <-, line 67 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 68 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ { int m1 = z->l - z->c; (void)m1; /* do, line 79 */
+ z->ket = z->c; /* [, line 80 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((811040 >> (z->p[z->c - 1] & 0x1f)) & 1)) goto lab0;
+ among_var = find_among_b(z, a_1, 7); /* substring, line 80 */
+ if (!(among_var)) goto lab0;
+ z->bra = z->c; /* ], line 80 */
+ { int ret = r_R1(z);
+ if (ret == 0) goto lab0; /* call R1, line 80 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 82 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 85 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 86 */
+ z->ket = z->c; /* [, line 86 */
+ if (!(eq_s_b(z, 1, s_11))) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 86 */
+ if (!(eq_s_b(z, 3, s_12))) { z->c = z->l - m_keep; goto lab1; }
+ { int ret = slice_del(z); /* delete, line 86 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ ;
+ }
+ break;
+ case 3:
+ if (in_grouping_b_U(z, g_s_ending, 98, 116, 0)) goto lab0;
+ { int ret = slice_del(z); /* delete, line 89 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab0:
+ z->c = z->l - m1;
+ }
+ { int m2 = z->l - z->c; (void)m2; /* do, line 93 */
+ z->ket = z->c; /* [, line 94 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1327104 >> (z->p[z->c - 1] & 0x1f)) & 1)) goto lab2;
+ among_var = find_among_b(z, a_2, 4); /* substring, line 94 */
+ if (!(among_var)) goto lab2;
+ z->bra = z->c; /* ], line 94 */
+ { int ret = r_R1(z);
+ if (ret == 0) goto lab2; /* call R1, line 94 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: goto lab2;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 96 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ if (in_grouping_b_U(z, g_st_ending, 98, 116, 0)) goto lab2;
+ { int ret = skip_utf8(z->p, z->c, z->lb, z->l, - 3);
+ if (ret < 0) goto lab2;
+ z->c = ret; /* hop, line 99 */
+ }
+ { int ret = slice_del(z); /* delete, line 99 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab2:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 103 */
+ z->ket = z->c; /* [, line 104 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1051024 >> (z->p[z->c - 1] & 0x1f)) & 1)) goto lab3;
+ among_var = find_among_b(z, a_4, 8); /* substring, line 104 */
+ if (!(among_var)) goto lab3;
+ z->bra = z->c; /* ], line 104 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab3; /* call R2, line 104 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: goto lab3;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 106 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 107 */
+ z->ket = z->c; /* [, line 107 */
+ if (!(eq_s_b(z, 2, s_13))) { z->c = z->l - m_keep; goto lab4; }
+ z->bra = z->c; /* ], line 107 */
+ { int m4 = z->l - z->c; (void)m4; /* not, line 107 */
+ if (!(eq_s_b(z, 1, s_14))) goto lab5;
+ { z->c = z->l - m_keep; goto lab4; }
+ lab5:
+ z->c = z->l - m4;
+ }
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab4; } /* call R2, line 107 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 107 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ ;
+ }
+ break;
+ case 2:
+ { int m5 = z->l - z->c; (void)m5; /* not, line 110 */
+ if (!(eq_s_b(z, 1, s_15))) goto lab6;
+ goto lab3;
+ lab6:
+ z->c = z->l - m5;
+ }
+ { int ret = slice_del(z); /* delete, line 110 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 113 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 114 */
+ z->ket = z->c; /* [, line 115 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 115 */
+ if (!(eq_s_b(z, 2, s_16))) goto lab9;
+ goto lab8;
+ lab9:
+ z->c = z->l - m6;
+ if (!(eq_s_b(z, 2, s_17))) { z->c = z->l - m_keep; goto lab7; }
+ }
+ lab8:
+ z->bra = z->c; /* ], line 115 */
+ { int ret = r_R1(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab7; } /* call R1, line 115 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 115 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ ;
+ }
+ break;
+ case 4:
+ { int ret = slice_del(z); /* delete, line 119 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 120 */
+ z->ket = z->c; /* [, line 121 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 103 && z->p[z->c - 1] != 104)) { z->c = z->l - m_keep; goto lab10; }
+ among_var = find_among_b(z, a_3, 2); /* substring, line 121 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab10; }
+ z->bra = z->c; /* ], line 121 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab10; } /* call R2, line 121 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab10; }
+ case 1:
+ { int ret = slice_del(z); /* delete, line 123 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab10:
+ ;
+ }
+ break;
+ }
+ lab3:
+ z->c = z->l - m3;
+ }
+ return 1;
+}
+
+extern int german_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 134 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab0; /* call prelude, line 134 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 135 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab1; /* call mark_regions, line 135 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = c2;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 136 */
+
+ { int m3 = z->l - z->c; (void)m3; /* do, line 137 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab2; /* call standard_suffix, line 137 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ z->c = z->lb;
+ { int c4 = z->c; /* do, line 138 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab3; /* call postlude, line 138 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = c4;
+ }
+ return 1;
+}
+
+extern struct SN_env * german_UTF_8_create_env(void) { return SN_create_env(0, 3, 0); }
+
+extern void german_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_german.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_german.h
new file mode 100644
index 0000000..5bd84d4
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_german.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * german_UTF_8_create_env(void);
+extern void german_UTF_8_close_env(struct SN_env * z);
+
+extern int german_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_hungarian.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_hungarian.c
new file mode 100644
index 0000000..44a62f7
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_hungarian.c
@@ -0,0 +1,1234 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int hungarian_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_double(struct SN_env * z);
+static int r_undouble(struct SN_env * z);
+static int r_factive(struct SN_env * z);
+static int r_instrum(struct SN_env * z);
+static int r_plur_owner(struct SN_env * z);
+static int r_sing_owner(struct SN_env * z);
+static int r_owned(struct SN_env * z);
+static int r_plural(struct SN_env * z);
+static int r_case_other(struct SN_env * z);
+static int r_case_special(struct SN_env * z);
+static int r_case(struct SN_env * z);
+static int r_v_ending(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * hungarian_UTF_8_create_env(void);
+extern void hungarian_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[2] = { 'c', 's' };
+static const symbol s_0_1[3] = { 'd', 'z', 's' };
+static const symbol s_0_2[2] = { 'g', 'y' };
+static const symbol s_0_3[2] = { 'l', 'y' };
+static const symbol s_0_4[2] = { 'n', 'y' };
+static const symbol s_0_5[2] = { 's', 'z' };
+static const symbol s_0_6[2] = { 't', 'y' };
+static const symbol s_0_7[2] = { 'z', 's' };
+
+static const struct among a_0[8] =
+{
+/* 0 */ { 2, s_0_0, -1, -1, 0},
+/* 1 */ { 3, s_0_1, -1, -1, 0},
+/* 2 */ { 2, s_0_2, -1, -1, 0},
+/* 3 */ { 2, s_0_3, -1, -1, 0},
+/* 4 */ { 2, s_0_4, -1, -1, 0},
+/* 5 */ { 2, s_0_5, -1, -1, 0},
+/* 6 */ { 2, s_0_6, -1, -1, 0},
+/* 7 */ { 2, s_0_7, -1, -1, 0}
+};
+
+static const symbol s_1_0[2] = { 0xC3, 0xA1 };
+static const symbol s_1_1[2] = { 0xC3, 0xA9 };
+
+static const struct among a_1[2] =
+{
+/* 0 */ { 2, s_1_0, -1, 1, 0},
+/* 1 */ { 2, s_1_1, -1, 2, 0}
+};
+
+static const symbol s_2_0[2] = { 'b', 'b' };
+static const symbol s_2_1[2] = { 'c', 'c' };
+static const symbol s_2_2[2] = { 'd', 'd' };
+static const symbol s_2_3[2] = { 'f', 'f' };
+static const symbol s_2_4[2] = { 'g', 'g' };
+static const symbol s_2_5[2] = { 'j', 'j' };
+static const symbol s_2_6[2] = { 'k', 'k' };
+static const symbol s_2_7[2] = { 'l', 'l' };
+static const symbol s_2_8[2] = { 'm', 'm' };
+static const symbol s_2_9[2] = { 'n', 'n' };
+static const symbol s_2_10[2] = { 'p', 'p' };
+static const symbol s_2_11[2] = { 'r', 'r' };
+static const symbol s_2_12[3] = { 'c', 'c', 's' };
+static const symbol s_2_13[2] = { 's', 's' };
+static const symbol s_2_14[3] = { 'z', 'z', 's' };
+static const symbol s_2_15[2] = { 't', 't' };
+static const symbol s_2_16[2] = { 'v', 'v' };
+static const symbol s_2_17[3] = { 'g', 'g', 'y' };
+static const symbol s_2_18[3] = { 'l', 'l', 'y' };
+static const symbol s_2_19[3] = { 'n', 'n', 'y' };
+static const symbol s_2_20[3] = { 't', 't', 'y' };
+static const symbol s_2_21[3] = { 's', 's', 'z' };
+static const symbol s_2_22[2] = { 'z', 'z' };
+
+static const struct among a_2[23] =
+{
+/* 0 */ { 2, s_2_0, -1, -1, 0},
+/* 1 */ { 2, s_2_1, -1, -1, 0},
+/* 2 */ { 2, s_2_2, -1, -1, 0},
+/* 3 */ { 2, s_2_3, -1, -1, 0},
+/* 4 */ { 2, s_2_4, -1, -1, 0},
+/* 5 */ { 2, s_2_5, -1, -1, 0},
+/* 6 */ { 2, s_2_6, -1, -1, 0},
+/* 7 */ { 2, s_2_7, -1, -1, 0},
+/* 8 */ { 2, s_2_8, -1, -1, 0},
+/* 9 */ { 2, s_2_9, -1, -1, 0},
+/* 10 */ { 2, s_2_10, -1, -1, 0},
+/* 11 */ { 2, s_2_11, -1, -1, 0},
+/* 12 */ { 3, s_2_12, -1, -1, 0},
+/* 13 */ { 2, s_2_13, -1, -1, 0},
+/* 14 */ { 3, s_2_14, -1, -1, 0},
+/* 15 */ { 2, s_2_15, -1, -1, 0},
+/* 16 */ { 2, s_2_16, -1, -1, 0},
+/* 17 */ { 3, s_2_17, -1, -1, 0},
+/* 18 */ { 3, s_2_18, -1, -1, 0},
+/* 19 */ { 3, s_2_19, -1, -1, 0},
+/* 20 */ { 3, s_2_20, -1, -1, 0},
+/* 21 */ { 3, s_2_21, -1, -1, 0},
+/* 22 */ { 2, s_2_22, -1, -1, 0}
+};
+
+static const symbol s_3_0[2] = { 'a', 'l' };
+static const symbol s_3_1[2] = { 'e', 'l' };
+
+static const struct among a_3[2] =
+{
+/* 0 */ { 2, s_3_0, -1, 1, 0},
+/* 1 */ { 2, s_3_1, -1, 2, 0}
+};
+
+static const symbol s_4_0[2] = { 'b', 'a' };
+static const symbol s_4_1[2] = { 'r', 'a' };
+static const symbol s_4_2[2] = { 'b', 'e' };
+static const symbol s_4_3[2] = { 'r', 'e' };
+static const symbol s_4_4[2] = { 'i', 'g' };
+static const symbol s_4_5[3] = { 'n', 'a', 'k' };
+static const symbol s_4_6[3] = { 'n', 'e', 'k' };
+static const symbol s_4_7[3] = { 'v', 'a', 'l' };
+static const symbol s_4_8[3] = { 'v', 'e', 'l' };
+static const symbol s_4_9[2] = { 'u', 'l' };
+static const symbol s_4_10[4] = { 'n', 0xC3, 0xA1, 'l' };
+static const symbol s_4_11[4] = { 'n', 0xC3, 0xA9, 'l' };
+static const symbol s_4_12[4] = { 'b', 0xC3, 0xB3, 'l' };
+static const symbol s_4_13[4] = { 'r', 0xC3, 0xB3, 'l' };
+static const symbol s_4_14[4] = { 't', 0xC3, 0xB3, 'l' };
+static const symbol s_4_15[4] = { 'b', 0xC3, 0xB5, 'l' };
+static const symbol s_4_16[4] = { 'r', 0xC3, 0xB5, 'l' };
+static const symbol s_4_17[4] = { 't', 0xC3, 0xB5, 'l' };
+static const symbol s_4_18[3] = { 0xC3, 0xBC, 'l' };
+static const symbol s_4_19[1] = { 'n' };
+static const symbol s_4_20[2] = { 'a', 'n' };
+static const symbol s_4_21[3] = { 'b', 'a', 'n' };
+static const symbol s_4_22[2] = { 'e', 'n' };
+static const symbol s_4_23[3] = { 'b', 'e', 'n' };
+static const symbol s_4_24[7] = { 'k', 0xC3, 0xA9, 'p', 'p', 'e', 'n' };
+static const symbol s_4_25[2] = { 'o', 'n' };
+static const symbol s_4_26[3] = { 0xC3, 0xB6, 'n' };
+static const symbol s_4_27[5] = { 'k', 0xC3, 0xA9, 'p', 'p' };
+static const symbol s_4_28[3] = { 'k', 'o', 'r' };
+static const symbol s_4_29[1] = { 't' };
+static const symbol s_4_30[2] = { 'a', 't' };
+static const symbol s_4_31[2] = { 'e', 't' };
+static const symbol s_4_32[5] = { 'k', 0xC3, 0xA9, 'n', 't' };
+static const symbol s_4_33[7] = { 'a', 'n', 'k', 0xC3, 0xA9, 'n', 't' };
+static const symbol s_4_34[7] = { 'e', 'n', 'k', 0xC3, 0xA9, 'n', 't' };
+static const symbol s_4_35[7] = { 'o', 'n', 'k', 0xC3, 0xA9, 'n', 't' };
+static const symbol s_4_36[2] = { 'o', 't' };
+static const symbol s_4_37[4] = { 0xC3, 0xA9, 'r', 't' };
+static const symbol s_4_38[3] = { 0xC3, 0xB6, 't' };
+static const symbol s_4_39[3] = { 'h', 'e', 'z' };
+static const symbol s_4_40[3] = { 'h', 'o', 'z' };
+static const symbol s_4_41[4] = { 'h', 0xC3, 0xB6, 'z' };
+static const symbol s_4_42[3] = { 'v', 0xC3, 0xA1 };
+static const symbol s_4_43[3] = { 'v', 0xC3, 0xA9 };
+
+static const struct among a_4[44] =
+{
+/* 0 */ { 2, s_4_0, -1, -1, 0},
+/* 1 */ { 2, s_4_1, -1, -1, 0},
+/* 2 */ { 2, s_4_2, -1, -1, 0},
+/* 3 */ { 2, s_4_3, -1, -1, 0},
+/* 4 */ { 2, s_4_4, -1, -1, 0},
+/* 5 */ { 3, s_4_5, -1, -1, 0},
+/* 6 */ { 3, s_4_6, -1, -1, 0},
+/* 7 */ { 3, s_4_7, -1, -1, 0},
+/* 8 */ { 3, s_4_8, -1, -1, 0},
+/* 9 */ { 2, s_4_9, -1, -1, 0},
+/* 10 */ { 4, s_4_10, -1, -1, 0},
+/* 11 */ { 4, s_4_11, -1, -1, 0},
+/* 12 */ { 4, s_4_12, -1, -1, 0},
+/* 13 */ { 4, s_4_13, -1, -1, 0},
+/* 14 */ { 4, s_4_14, -1, -1, 0},
+/* 15 */ { 4, s_4_15, -1, -1, 0},
+/* 16 */ { 4, s_4_16, -1, -1, 0},
+/* 17 */ { 4, s_4_17, -1, -1, 0},
+/* 18 */ { 3, s_4_18, -1, -1, 0},
+/* 19 */ { 1, s_4_19, -1, -1, 0},
+/* 20 */ { 2, s_4_20, 19, -1, 0},
+/* 21 */ { 3, s_4_21, 20, -1, 0},
+/* 22 */ { 2, s_4_22, 19, -1, 0},
+/* 23 */ { 3, s_4_23, 22, -1, 0},
+/* 24 */ { 7, s_4_24, 22, -1, 0},
+/* 25 */ { 2, s_4_25, 19, -1, 0},
+/* 26 */ { 3, s_4_26, 19, -1, 0},
+/* 27 */ { 5, s_4_27, -1, -1, 0},
+/* 28 */ { 3, s_4_28, -1, -1, 0},
+/* 29 */ { 1, s_4_29, -1, -1, 0},
+/* 30 */ { 2, s_4_30, 29, -1, 0},
+/* 31 */ { 2, s_4_31, 29, -1, 0},
+/* 32 */ { 5, s_4_32, 29, -1, 0},
+/* 33 */ { 7, s_4_33, 32, -1, 0},
+/* 34 */ { 7, s_4_34, 32, -1, 0},
+/* 35 */ { 7, s_4_35, 32, -1, 0},
+/* 36 */ { 2, s_4_36, 29, -1, 0},
+/* 37 */ { 4, s_4_37, 29, -1, 0},
+/* 38 */ { 3, s_4_38, 29, -1, 0},
+/* 39 */ { 3, s_4_39, -1, -1, 0},
+/* 40 */ { 3, s_4_40, -1, -1, 0},
+/* 41 */ { 4, s_4_41, -1, -1, 0},
+/* 42 */ { 3, s_4_42, -1, -1, 0},
+/* 43 */ { 3, s_4_43, -1, -1, 0}
+};
+
+static const symbol s_5_0[3] = { 0xC3, 0xA1, 'n' };
+static const symbol s_5_1[3] = { 0xC3, 0xA9, 'n' };
+static const symbol s_5_2[8] = { 0xC3, 0xA1, 'n', 'k', 0xC3, 0xA9, 'n', 't' };
+
+static const struct among a_5[3] =
+{
+/* 0 */ { 3, s_5_0, -1, 2, 0},
+/* 1 */ { 3, s_5_1, -1, 1, 0},
+/* 2 */ { 8, s_5_2, -1, 3, 0}
+};
+
+static const symbol s_6_0[4] = { 's', 't', 'u', 'l' };
+static const symbol s_6_1[5] = { 'a', 's', 't', 'u', 'l' };
+static const symbol s_6_2[6] = { 0xC3, 0xA1, 's', 't', 'u', 'l' };
+static const symbol s_6_3[5] = { 's', 't', 0xC3, 0xBC, 'l' };
+static const symbol s_6_4[6] = { 'e', 's', 't', 0xC3, 0xBC, 'l' };
+static const symbol s_6_5[7] = { 0xC3, 0xA9, 's', 't', 0xC3, 0xBC, 'l' };
+
+static const struct among a_6[6] =
+{
+/* 0 */ { 4, s_6_0, -1, 2, 0},
+/* 1 */ { 5, s_6_1, 0, 1, 0},
+/* 2 */ { 6, s_6_2, 0, 3, 0},
+/* 3 */ { 5, s_6_3, -1, 2, 0},
+/* 4 */ { 6, s_6_4, 3, 1, 0},
+/* 5 */ { 7, s_6_5, 3, 4, 0}
+};
+
+static const symbol s_7_0[2] = { 0xC3, 0xA1 };
+static const symbol s_7_1[2] = { 0xC3, 0xA9 };
+
+static const struct among a_7[2] =
+{
+/* 0 */ { 2, s_7_0, -1, 1, 0},
+/* 1 */ { 2, s_7_1, -1, 2, 0}
+};
+
+static const symbol s_8_0[1] = { 'k' };
+static const symbol s_8_1[2] = { 'a', 'k' };
+static const symbol s_8_2[2] = { 'e', 'k' };
+static const symbol s_8_3[2] = { 'o', 'k' };
+static const symbol s_8_4[3] = { 0xC3, 0xA1, 'k' };
+static const symbol s_8_5[3] = { 0xC3, 0xA9, 'k' };
+static const symbol s_8_6[3] = { 0xC3, 0xB6, 'k' };
+
+static const struct among a_8[7] =
+{
+/* 0 */ { 1, s_8_0, -1, 7, 0},
+/* 1 */ { 2, s_8_1, 0, 4, 0},
+/* 2 */ { 2, s_8_2, 0, 6, 0},
+/* 3 */ { 2, s_8_3, 0, 5, 0},
+/* 4 */ { 3, s_8_4, 0, 1, 0},
+/* 5 */ { 3, s_8_5, 0, 2, 0},
+/* 6 */ { 3, s_8_6, 0, 3, 0}
+};
+
+static const symbol s_9_0[3] = { 0xC3, 0xA9, 'i' };
+static const symbol s_9_1[5] = { 0xC3, 0xA1, 0xC3, 0xA9, 'i' };
+static const symbol s_9_2[5] = { 0xC3, 0xA9, 0xC3, 0xA9, 'i' };
+static const symbol s_9_3[2] = { 0xC3, 0xA9 };
+static const symbol s_9_4[3] = { 'k', 0xC3, 0xA9 };
+static const symbol s_9_5[4] = { 'a', 'k', 0xC3, 0xA9 };
+static const symbol s_9_6[4] = { 'e', 'k', 0xC3, 0xA9 };
+static const symbol s_9_7[4] = { 'o', 'k', 0xC3, 0xA9 };
+static const symbol s_9_8[5] = { 0xC3, 0xA1, 'k', 0xC3, 0xA9 };
+static const symbol s_9_9[5] = { 0xC3, 0xA9, 'k', 0xC3, 0xA9 };
+static const symbol s_9_10[5] = { 0xC3, 0xB6, 'k', 0xC3, 0xA9 };
+static const symbol s_9_11[4] = { 0xC3, 0xA9, 0xC3, 0xA9 };
+
+static const struct among a_9[12] =
+{
+/* 0 */ { 3, s_9_0, -1, 7, 0},
+/* 1 */ { 5, s_9_1, 0, 6, 0},
+/* 2 */ { 5, s_9_2, 0, 5, 0},
+/* 3 */ { 2, s_9_3, -1, 9, 0},
+/* 4 */ { 3, s_9_4, 3, 4, 0},
+/* 5 */ { 4, s_9_5, 4, 1, 0},
+/* 6 */ { 4, s_9_6, 4, 1, 0},
+/* 7 */ { 4, s_9_7, 4, 1, 0},
+/* 8 */ { 5, s_9_8, 4, 3, 0},
+/* 9 */ { 5, s_9_9, 4, 2, 0},
+/* 10 */ { 5, s_9_10, 4, 1, 0},
+/* 11 */ { 4, s_9_11, 3, 8, 0}
+};
+
+static const symbol s_10_0[1] = { 'a' };
+static const symbol s_10_1[2] = { 'j', 'a' };
+static const symbol s_10_2[1] = { 'd' };
+static const symbol s_10_3[2] = { 'a', 'd' };
+static const symbol s_10_4[2] = { 'e', 'd' };
+static const symbol s_10_5[2] = { 'o', 'd' };
+static const symbol s_10_6[3] = { 0xC3, 0xA1, 'd' };
+static const symbol s_10_7[3] = { 0xC3, 0xA9, 'd' };
+static const symbol s_10_8[3] = { 0xC3, 0xB6, 'd' };
+static const symbol s_10_9[1] = { 'e' };
+static const symbol s_10_10[2] = { 'j', 'e' };
+static const symbol s_10_11[2] = { 'n', 'k' };
+static const symbol s_10_12[3] = { 'u', 'n', 'k' };
+static const symbol s_10_13[4] = { 0xC3, 0xA1, 'n', 'k' };
+static const symbol s_10_14[4] = { 0xC3, 0xA9, 'n', 'k' };
+static const symbol s_10_15[4] = { 0xC3, 0xBC, 'n', 'k' };
+static const symbol s_10_16[2] = { 'u', 'k' };
+static const symbol s_10_17[3] = { 'j', 'u', 'k' };
+static const symbol s_10_18[5] = { 0xC3, 0xA1, 'j', 'u', 'k' };
+static const symbol s_10_19[3] = { 0xC3, 0xBC, 'k' };
+static const symbol s_10_20[4] = { 'j', 0xC3, 0xBC, 'k' };
+static const symbol s_10_21[6] = { 0xC3, 0xA9, 'j', 0xC3, 0xBC, 'k' };
+static const symbol s_10_22[1] = { 'm' };
+static const symbol s_10_23[2] = { 'a', 'm' };
+static const symbol s_10_24[2] = { 'e', 'm' };
+static const symbol s_10_25[2] = { 'o', 'm' };
+static const symbol s_10_26[3] = { 0xC3, 0xA1, 'm' };
+static const symbol s_10_27[3] = { 0xC3, 0xA9, 'm' };
+static const symbol s_10_28[1] = { 'o' };
+static const symbol s_10_29[2] = { 0xC3, 0xA1 };
+static const symbol s_10_30[2] = { 0xC3, 0xA9 };
+
+static const struct among a_10[31] =
+{
+/* 0 */ { 1, s_10_0, -1, 18, 0},
+/* 1 */ { 2, s_10_1, 0, 17, 0},
+/* 2 */ { 1, s_10_2, -1, 16, 0},
+/* 3 */ { 2, s_10_3, 2, 13, 0},
+/* 4 */ { 2, s_10_4, 2, 13, 0},
+/* 5 */ { 2, s_10_5, 2, 13, 0},
+/* 6 */ { 3, s_10_6, 2, 14, 0},
+/* 7 */ { 3, s_10_7, 2, 15, 0},
+/* 8 */ { 3, s_10_8, 2, 13, 0},
+/* 9 */ { 1, s_10_9, -1, 18, 0},
+/* 10 */ { 2, s_10_10, 9, 17, 0},
+/* 11 */ { 2, s_10_11, -1, 4, 0},
+/* 12 */ { 3, s_10_12, 11, 1, 0},
+/* 13 */ { 4, s_10_13, 11, 2, 0},
+/* 14 */ { 4, s_10_14, 11, 3, 0},
+/* 15 */ { 4, s_10_15, 11, 1, 0},
+/* 16 */ { 2, s_10_16, -1, 8, 0},
+/* 17 */ { 3, s_10_17, 16, 7, 0},
+/* 18 */ { 5, s_10_18, 17, 5, 0},
+/* 19 */ { 3, s_10_19, -1, 8, 0},
+/* 20 */ { 4, s_10_20, 19, 7, 0},
+/* 21 */ { 6, s_10_21, 20, 6, 0},
+/* 22 */ { 1, s_10_22, -1, 12, 0},
+/* 23 */ { 2, s_10_23, 22, 9, 0},
+/* 24 */ { 2, s_10_24, 22, 9, 0},
+/* 25 */ { 2, s_10_25, 22, 9, 0},
+/* 26 */ { 3, s_10_26, 22, 10, 0},
+/* 27 */ { 3, s_10_27, 22, 11, 0},
+/* 28 */ { 1, s_10_28, -1, 18, 0},
+/* 29 */ { 2, s_10_29, -1, 19, 0},
+/* 30 */ { 2, s_10_30, -1, 20, 0}
+};
+
+static const symbol s_11_0[2] = { 'i', 'd' };
+static const symbol s_11_1[3] = { 'a', 'i', 'd' };
+static const symbol s_11_2[4] = { 'j', 'a', 'i', 'd' };
+static const symbol s_11_3[3] = { 'e', 'i', 'd' };
+static const symbol s_11_4[4] = { 'j', 'e', 'i', 'd' };
+static const symbol s_11_5[4] = { 0xC3, 0xA1, 'i', 'd' };
+static const symbol s_11_6[4] = { 0xC3, 0xA9, 'i', 'd' };
+static const symbol s_11_7[1] = { 'i' };
+static const symbol s_11_8[2] = { 'a', 'i' };
+static const symbol s_11_9[3] = { 'j', 'a', 'i' };
+static const symbol s_11_10[2] = { 'e', 'i' };
+static const symbol s_11_11[3] = { 'j', 'e', 'i' };
+static const symbol s_11_12[3] = { 0xC3, 0xA1, 'i' };
+static const symbol s_11_13[3] = { 0xC3, 0xA9, 'i' };
+static const symbol s_11_14[4] = { 'i', 't', 'e', 'k' };
+static const symbol s_11_15[5] = { 'e', 'i', 't', 'e', 'k' };
+static const symbol s_11_16[6] = { 'j', 'e', 'i', 't', 'e', 'k' };
+static const symbol s_11_17[6] = { 0xC3, 0xA9, 'i', 't', 'e', 'k' };
+static const symbol s_11_18[2] = { 'i', 'k' };
+static const symbol s_11_19[3] = { 'a', 'i', 'k' };
+static const symbol s_11_20[4] = { 'j', 'a', 'i', 'k' };
+static const symbol s_11_21[3] = { 'e', 'i', 'k' };
+static const symbol s_11_22[4] = { 'j', 'e', 'i', 'k' };
+static const symbol s_11_23[4] = { 0xC3, 0xA1, 'i', 'k' };
+static const symbol s_11_24[4] = { 0xC3, 0xA9, 'i', 'k' };
+static const symbol s_11_25[3] = { 'i', 'n', 'k' };
+static const symbol s_11_26[4] = { 'a', 'i', 'n', 'k' };
+static const symbol s_11_27[5] = { 'j', 'a', 'i', 'n', 'k' };
+static const symbol s_11_28[4] = { 'e', 'i', 'n', 'k' };
+static const symbol s_11_29[5] = { 'j', 'e', 'i', 'n', 'k' };
+static const symbol s_11_30[5] = { 0xC3, 0xA1, 'i', 'n', 'k' };
+static const symbol s_11_31[5] = { 0xC3, 0xA9, 'i', 'n', 'k' };
+static const symbol s_11_32[5] = { 'a', 'i', 't', 'o', 'k' };
+static const symbol s_11_33[6] = { 'j', 'a', 'i', 't', 'o', 'k' };
+static const symbol s_11_34[6] = { 0xC3, 0xA1, 'i', 't', 'o', 'k' };
+static const symbol s_11_35[2] = { 'i', 'm' };
+static const symbol s_11_36[3] = { 'a', 'i', 'm' };
+static const symbol s_11_37[4] = { 'j', 'a', 'i', 'm' };
+static const symbol s_11_38[3] = { 'e', 'i', 'm' };
+static const symbol s_11_39[4] = { 'j', 'e', 'i', 'm' };
+static const symbol s_11_40[4] = { 0xC3, 0xA1, 'i', 'm' };
+static const symbol s_11_41[4] = { 0xC3, 0xA9, 'i', 'm' };
+
+static const struct among a_11[42] =
+{
+/* 0 */ { 2, s_11_0, -1, 10, 0},
+/* 1 */ { 3, s_11_1, 0, 9, 0},
+/* 2 */ { 4, s_11_2, 1, 6, 0},
+/* 3 */ { 3, s_11_3, 0, 9, 0},
+/* 4 */ { 4, s_11_4, 3, 6, 0},
+/* 5 */ { 4, s_11_5, 0, 7, 0},
+/* 6 */ { 4, s_11_6, 0, 8, 0},
+/* 7 */ { 1, s_11_7, -1, 15, 0},
+/* 8 */ { 2, s_11_8, 7, 14, 0},
+/* 9 */ { 3, s_11_9, 8, 11, 0},
+/* 10 */ { 2, s_11_10, 7, 14, 0},
+/* 11 */ { 3, s_11_11, 10, 11, 0},
+/* 12 */ { 3, s_11_12, 7, 12, 0},
+/* 13 */ { 3, s_11_13, 7, 13, 0},
+/* 14 */ { 4, s_11_14, -1, 24, 0},
+/* 15 */ { 5, s_11_15, 14, 21, 0},
+/* 16 */ { 6, s_11_16, 15, 20, 0},
+/* 17 */ { 6, s_11_17, 14, 23, 0},
+/* 18 */ { 2, s_11_18, -1, 29, 0},
+/* 19 */ { 3, s_11_19, 18, 26, 0},
+/* 20 */ { 4, s_11_20, 19, 25, 0},
+/* 21 */ { 3, s_11_21, 18, 26, 0},
+/* 22 */ { 4, s_11_22, 21, 25, 0},
+/* 23 */ { 4, s_11_23, 18, 27, 0},
+/* 24 */ { 4, s_11_24, 18, 28, 0},
+/* 25 */ { 3, s_11_25, -1, 20, 0},
+/* 26 */ { 4, s_11_26, 25, 17, 0},
+/* 27 */ { 5, s_11_27, 26, 16, 0},
+/* 28 */ { 4, s_11_28, 25, 17, 0},
+/* 29 */ { 5, s_11_29, 28, 16, 0},
+/* 30 */ { 5, s_11_30, 25, 18, 0},
+/* 31 */ { 5, s_11_31, 25, 19, 0},
+/* 32 */ { 5, s_11_32, -1, 21, 0},
+/* 33 */ { 6, s_11_33, 32, 20, 0},
+/* 34 */ { 6, s_11_34, -1, 22, 0},
+/* 35 */ { 2, s_11_35, -1, 5, 0},
+/* 36 */ { 3, s_11_36, 35, 4, 0},
+/* 37 */ { 4, s_11_37, 36, 1, 0},
+/* 38 */ { 3, s_11_38, 35, 4, 0},
+/* 39 */ { 4, s_11_39, 38, 1, 0},
+/* 40 */ { 4, s_11_40, 35, 2, 0},
+/* 41 */ { 4, s_11_41, 35, 3, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 52, 14 };
+
+static const symbol s_0[] = { 'a' };
+static const symbol s_1[] = { 'e' };
+static const symbol s_2[] = { 'e' };
+static const symbol s_3[] = { 'a' };
+static const symbol s_4[] = { 'a' };
+static const symbol s_5[] = { 'a' };
+static const symbol s_6[] = { 'e' };
+static const symbol s_7[] = { 'a' };
+static const symbol s_8[] = { 'e' };
+static const symbol s_9[] = { 'e' };
+static const symbol s_10[] = { 'a' };
+static const symbol s_11[] = { 'e' };
+static const symbol s_12[] = { 'a' };
+static const symbol s_13[] = { 'e' };
+static const symbol s_14[] = { 'a' };
+static const symbol s_15[] = { 'e' };
+static const symbol s_16[] = { 'a' };
+static const symbol s_17[] = { 'e' };
+static const symbol s_18[] = { 'a' };
+static const symbol s_19[] = { 'e' };
+static const symbol s_20[] = { 'a' };
+static const symbol s_21[] = { 'e' };
+static const symbol s_22[] = { 'a' };
+static const symbol s_23[] = { 'e' };
+static const symbol s_24[] = { 'a' };
+static const symbol s_25[] = { 'e' };
+static const symbol s_26[] = { 'a' };
+static const symbol s_27[] = { 'e' };
+static const symbol s_28[] = { 'a' };
+static const symbol s_29[] = { 'e' };
+static const symbol s_30[] = { 'a' };
+static const symbol s_31[] = { 'e' };
+static const symbol s_32[] = { 'a' };
+static const symbol s_33[] = { 'e' };
+static const symbol s_34[] = { 'a' };
+static const symbol s_35[] = { 'e' };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ { int c1 = z->c; /* or, line 51 */
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab1;
+ if (in_grouping_U(z, g_v, 97, 252, 1) < 0) goto lab1; /* goto */ /* non v, line 48 */
+ { int c2 = z->c; /* or, line 49 */
+ if (z->c + 1 >= z->l || z->p[z->c + 1] >> 5 != 3 || !((101187584 >> (z->p[z->c + 1] & 0x1f)) & 1)) goto lab3;
+ if (!(find_among(z, a_0, 8))) goto lab3; /* among, line 49 */
+ goto lab2;
+ lab3:
+ z->c = c2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* next, line 49 */
+ }
+ }
+ lab2:
+ z->I[0] = z->c; /* setmark p1, line 50 */
+ goto lab0;
+ lab1:
+ z->c = c1;
+ if (out_grouping_U(z, g_v, 97, 252, 0)) return 0;
+ { /* gopast */ /* grouping v, line 53 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 53 */
+ }
+lab0:
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_v_ending(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 61 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 161 && z->p[z->c - 1] != 169)) return 0;
+ among_var = find_among_b(z, a_1, 2); /* substring, line 61 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 61 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 61 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_0); /* <-, line 62 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_1); /* <-, line 63 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_double(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 68 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((106790108 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_2, 23))) return 0; /* among, line 68 */
+ z->c = z->l - m_test;
+ }
+ return 1;
+}
+
+static int r_undouble(struct SN_env * z) {
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 73 */
+ }
+ z->ket = z->c; /* [, line 73 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, z->l, - 1);
+ if (ret < 0) return 0;
+ z->c = ret; /* hop, line 73 */
+ }
+ z->bra = z->c; /* ], line 73 */
+ { int ret = slice_del(z); /* delete, line 73 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_instrum(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 77 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] != 108) return 0;
+ among_var = find_among_b(z, a_3, 2); /* substring, line 77 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 77 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 77 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_double(z);
+ if (ret == 0) return 0; /* call double, line 78 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_double(z);
+ if (ret == 0) return 0; /* call double, line 79 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ { int ret = slice_del(z); /* delete, line 81 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_undouble(z);
+ if (ret == 0) return 0; /* call undouble, line 82 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_case(struct SN_env * z) {
+ z->ket = z->c; /* [, line 87 */
+ if (!(find_among_b(z, a_4, 44))) return 0; /* substring, line 87 */
+ z->bra = z->c; /* ], line 87 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 87 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 111 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_v_ending(z);
+ if (ret == 0) return 0; /* call v_ending, line 112 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_case_special(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 116 */
+ if (z->c - 2 <= z->lb || (z->p[z->c - 1] != 110 && z->p[z->c - 1] != 116)) return 0;
+ among_var = find_among_b(z, a_5, 3); /* substring, line 116 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 116 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 116 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_2); /* <-, line 117 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_3); /* <-, line 118 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_4); /* <-, line 119 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_case_other(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 124 */
+ if (z->c - 3 <= z->lb || z->p[z->c - 1] != 108) return 0;
+ among_var = find_among_b(z, a_6, 6); /* substring, line 124 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 124 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 124 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 125 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 126 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_5); /* <-, line 127 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 1, s_6); /* <-, line 128 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_factive(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 133 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 161 && z->p[z->c - 1] != 169)) return 0;
+ among_var = find_among_b(z, a_7, 2); /* substring, line 133 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 133 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 133 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_double(z);
+ if (ret == 0) return 0; /* call double, line 134 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_double(z);
+ if (ret == 0) return 0; /* call double, line 135 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ { int ret = slice_del(z); /* delete, line 137 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_undouble(z);
+ if (ret == 0) return 0; /* call undouble, line 138 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_plural(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 142 */
+ if (z->c <= z->lb || z->p[z->c - 1] != 107) return 0;
+ among_var = find_among_b(z, a_8, 7); /* substring, line 142 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 142 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 142 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_7); /* <-, line 143 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_8); /* <-, line 144 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 145 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_del(z); /* delete, line 146 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_del(z); /* delete, line 147 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_del(z); /* delete, line 148 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_del(z); /* delete, line 149 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_owned(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 154 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 105 && z->p[z->c - 1] != 169)) return 0;
+ among_var = find_among_b(z, a_9, 12); /* substring, line 154 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 154 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 154 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 155 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_9); /* <-, line 156 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_10); /* <-, line 157 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_del(z); /* delete, line 158 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 1, s_11); /* <-, line 159 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 1, s_12); /* <-, line 160 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_del(z); /* delete, line 161 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 8:
+ { int ret = slice_from_s(z, 1, s_13); /* <-, line 162 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 9:
+ { int ret = slice_del(z); /* delete, line 163 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_sing_owner(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 168 */
+ among_var = find_among_b(z, a_10, 31); /* substring, line 168 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 168 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 168 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 169 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_14); /* <-, line 170 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_15); /* <-, line 171 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_del(z); /* delete, line 172 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 1, s_16); /* <-, line 173 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 1, s_17); /* <-, line 174 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_del(z); /* delete, line 175 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 8:
+ { int ret = slice_del(z); /* delete, line 176 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 9:
+ { int ret = slice_del(z); /* delete, line 177 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 10:
+ { int ret = slice_from_s(z, 1, s_18); /* <-, line 178 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 11:
+ { int ret = slice_from_s(z, 1, s_19); /* <-, line 179 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 12:
+ { int ret = slice_del(z); /* delete, line 180 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 13:
+ { int ret = slice_del(z); /* delete, line 181 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 14:
+ { int ret = slice_from_s(z, 1, s_20); /* <-, line 182 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 15:
+ { int ret = slice_from_s(z, 1, s_21); /* <-, line 183 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 16:
+ { int ret = slice_del(z); /* delete, line 184 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 17:
+ { int ret = slice_del(z); /* delete, line 185 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 18:
+ { int ret = slice_del(z); /* delete, line 186 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 19:
+ { int ret = slice_from_s(z, 1, s_22); /* <-, line 187 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 20:
+ { int ret = slice_from_s(z, 1, s_23); /* <-, line 188 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_plur_owner(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 193 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((10768 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_11, 42); /* substring, line 193 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 193 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 193 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 194 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_24); /* <-, line 195 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_25); /* <-, line 196 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_del(z); /* delete, line 197 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_del(z); /* delete, line 198 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_del(z); /* delete, line 199 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_from_s(z, 1, s_26); /* <-, line 200 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 8:
+ { int ret = slice_from_s(z, 1, s_27); /* <-, line 201 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 9:
+ { int ret = slice_del(z); /* delete, line 202 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 10:
+ { int ret = slice_del(z); /* delete, line 203 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 11:
+ { int ret = slice_del(z); /* delete, line 204 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 12:
+ { int ret = slice_from_s(z, 1, s_28); /* <-, line 205 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 13:
+ { int ret = slice_from_s(z, 1, s_29); /* <-, line 206 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 14:
+ { int ret = slice_del(z); /* delete, line 207 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 15:
+ { int ret = slice_del(z); /* delete, line 208 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 16:
+ { int ret = slice_del(z); /* delete, line 209 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 17:
+ { int ret = slice_del(z); /* delete, line 210 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 18:
+ { int ret = slice_from_s(z, 1, s_30); /* <-, line 211 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 19:
+ { int ret = slice_from_s(z, 1, s_31); /* <-, line 212 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 20:
+ { int ret = slice_del(z); /* delete, line 214 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 21:
+ { int ret = slice_del(z); /* delete, line 215 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 22:
+ { int ret = slice_from_s(z, 1, s_32); /* <-, line 216 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 23:
+ { int ret = slice_from_s(z, 1, s_33); /* <-, line 217 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 24:
+ { int ret = slice_del(z); /* delete, line 218 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 25:
+ { int ret = slice_del(z); /* delete, line 219 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 26:
+ { int ret = slice_del(z); /* delete, line 220 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 27:
+ { int ret = slice_from_s(z, 1, s_34); /* <-, line 221 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 28:
+ { int ret = slice_from_s(z, 1, s_35); /* <-, line 222 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 29:
+ { int ret = slice_del(z); /* delete, line 223 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+extern int hungarian_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 229 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 229 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 230 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 231 */
+ { int ret = r_instrum(z);
+ if (ret == 0) goto lab1; /* call instrum, line 231 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 232 */
+ { int ret = r_case(z);
+ if (ret == 0) goto lab2; /* call case, line 232 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 233 */
+ { int ret = r_case_special(z);
+ if (ret == 0) goto lab3; /* call case_special, line 233 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = z->l - m4;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 234 */
+ { int ret = r_case_other(z);
+ if (ret == 0) goto lab4; /* call case_other, line 234 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ z->c = z->l - m5;
+ }
+ { int m6 = z->l - z->c; (void)m6; /* do, line 235 */
+ { int ret = r_factive(z);
+ if (ret == 0) goto lab5; /* call factive, line 235 */
+ if (ret < 0) return ret;
+ }
+ lab5:
+ z->c = z->l - m6;
+ }
+ { int m7 = z->l - z->c; (void)m7; /* do, line 236 */
+ { int ret = r_owned(z);
+ if (ret == 0) goto lab6; /* call owned, line 236 */
+ if (ret < 0) return ret;
+ }
+ lab6:
+ z->c = z->l - m7;
+ }
+ { int m8 = z->l - z->c; (void)m8; /* do, line 237 */
+ { int ret = r_sing_owner(z);
+ if (ret == 0) goto lab7; /* call sing_owner, line 237 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = z->l - m8;
+ }
+ { int m9 = z->l - z->c; (void)m9; /* do, line 238 */
+ { int ret = r_plur_owner(z);
+ if (ret == 0) goto lab8; /* call plur_owner, line 238 */
+ if (ret < 0) return ret;
+ }
+ lab8:
+ z->c = z->l - m9;
+ }
+ { int m10 = z->l - z->c; (void)m10; /* do, line 239 */
+ { int ret = r_plural(z);
+ if (ret == 0) goto lab9; /* call plural, line 239 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = z->l - m10;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern struct SN_env * hungarian_UTF_8_create_env(void) { return SN_create_env(0, 1, 0); }
+
+extern void hungarian_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_hungarian.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_hungarian.h
new file mode 100644
index 0000000..d81bd23
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_hungarian.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * hungarian_UTF_8_create_env(void);
+extern void hungarian_UTF_8_close_env(struct SN_env * z);
+
+extern int hungarian_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_italian.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_italian.c
new file mode 100644
index 0000000..7bb0511
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_italian.c
@@ -0,0 +1,1073 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int italian_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_vowel_suffix(struct SN_env * z);
+static int r_verb_suffix(struct SN_env * z);
+static int r_standard_suffix(struct SN_env * z);
+static int r_attached_pronoun(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_RV(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * italian_UTF_8_create_env(void);
+extern void italian_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_1[2] = { 'q', 'u' };
+static const symbol s_0_2[2] = { 0xC3, 0xA1 };
+static const symbol s_0_3[2] = { 0xC3, 0xA9 };
+static const symbol s_0_4[2] = { 0xC3, 0xAD };
+static const symbol s_0_5[2] = { 0xC3, 0xB3 };
+static const symbol s_0_6[2] = { 0xC3, 0xBA };
+
+static const struct among a_0[7] =
+{
+/* 0 */ { 0, 0, -1, 7, 0},
+/* 1 */ { 2, s_0_1, 0, 6, 0},
+/* 2 */ { 2, s_0_2, 0, 1, 0},
+/* 3 */ { 2, s_0_3, 0, 2, 0},
+/* 4 */ { 2, s_0_4, 0, 3, 0},
+/* 5 */ { 2, s_0_5, 0, 4, 0},
+/* 6 */ { 2, s_0_6, 0, 5, 0}
+};
+
+static const symbol s_1_1[1] = { 'I' };
+static const symbol s_1_2[1] = { 'U' };
+
+static const struct among a_1[3] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 1, s_1_1, 0, 1, 0},
+/* 2 */ { 1, s_1_2, 0, 2, 0}
+};
+
+static const symbol s_2_0[2] = { 'l', 'a' };
+static const symbol s_2_1[4] = { 'c', 'e', 'l', 'a' };
+static const symbol s_2_2[6] = { 'g', 'l', 'i', 'e', 'l', 'a' };
+static const symbol s_2_3[4] = { 'm', 'e', 'l', 'a' };
+static const symbol s_2_4[4] = { 't', 'e', 'l', 'a' };
+static const symbol s_2_5[4] = { 'v', 'e', 'l', 'a' };
+static const symbol s_2_6[2] = { 'l', 'e' };
+static const symbol s_2_7[4] = { 'c', 'e', 'l', 'e' };
+static const symbol s_2_8[6] = { 'g', 'l', 'i', 'e', 'l', 'e' };
+static const symbol s_2_9[4] = { 'm', 'e', 'l', 'e' };
+static const symbol s_2_10[4] = { 't', 'e', 'l', 'e' };
+static const symbol s_2_11[4] = { 'v', 'e', 'l', 'e' };
+static const symbol s_2_12[2] = { 'n', 'e' };
+static const symbol s_2_13[4] = { 'c', 'e', 'n', 'e' };
+static const symbol s_2_14[6] = { 'g', 'l', 'i', 'e', 'n', 'e' };
+static const symbol s_2_15[4] = { 'm', 'e', 'n', 'e' };
+static const symbol s_2_16[4] = { 's', 'e', 'n', 'e' };
+static const symbol s_2_17[4] = { 't', 'e', 'n', 'e' };
+static const symbol s_2_18[4] = { 'v', 'e', 'n', 'e' };
+static const symbol s_2_19[2] = { 'c', 'i' };
+static const symbol s_2_20[2] = { 'l', 'i' };
+static const symbol s_2_21[4] = { 'c', 'e', 'l', 'i' };
+static const symbol s_2_22[6] = { 'g', 'l', 'i', 'e', 'l', 'i' };
+static const symbol s_2_23[4] = { 'm', 'e', 'l', 'i' };
+static const symbol s_2_24[4] = { 't', 'e', 'l', 'i' };
+static const symbol s_2_25[4] = { 'v', 'e', 'l', 'i' };
+static const symbol s_2_26[3] = { 'g', 'l', 'i' };
+static const symbol s_2_27[2] = { 'm', 'i' };
+static const symbol s_2_28[2] = { 's', 'i' };
+static const symbol s_2_29[2] = { 't', 'i' };
+static const symbol s_2_30[2] = { 'v', 'i' };
+static const symbol s_2_31[2] = { 'l', 'o' };
+static const symbol s_2_32[4] = { 'c', 'e', 'l', 'o' };
+static const symbol s_2_33[6] = { 'g', 'l', 'i', 'e', 'l', 'o' };
+static const symbol s_2_34[4] = { 'm', 'e', 'l', 'o' };
+static const symbol s_2_35[4] = { 't', 'e', 'l', 'o' };
+static const symbol s_2_36[4] = { 'v', 'e', 'l', 'o' };
+
+static const struct among a_2[37] =
+{
+/* 0 */ { 2, s_2_0, -1, -1, 0},
+/* 1 */ { 4, s_2_1, 0, -1, 0},
+/* 2 */ { 6, s_2_2, 0, -1, 0},
+/* 3 */ { 4, s_2_3, 0, -1, 0},
+/* 4 */ { 4, s_2_4, 0, -1, 0},
+/* 5 */ { 4, s_2_5, 0, -1, 0},
+/* 6 */ { 2, s_2_6, -1, -1, 0},
+/* 7 */ { 4, s_2_7, 6, -1, 0},
+/* 8 */ { 6, s_2_8, 6, -1, 0},
+/* 9 */ { 4, s_2_9, 6, -1, 0},
+/* 10 */ { 4, s_2_10, 6, -1, 0},
+/* 11 */ { 4, s_2_11, 6, -1, 0},
+/* 12 */ { 2, s_2_12, -1, -1, 0},
+/* 13 */ { 4, s_2_13, 12, -1, 0},
+/* 14 */ { 6, s_2_14, 12, -1, 0},
+/* 15 */ { 4, s_2_15, 12, -1, 0},
+/* 16 */ { 4, s_2_16, 12, -1, 0},
+/* 17 */ { 4, s_2_17, 12, -1, 0},
+/* 18 */ { 4, s_2_18, 12, -1, 0},
+/* 19 */ { 2, s_2_19, -1, -1, 0},
+/* 20 */ { 2, s_2_20, -1, -1, 0},
+/* 21 */ { 4, s_2_21, 20, -1, 0},
+/* 22 */ { 6, s_2_22, 20, -1, 0},
+/* 23 */ { 4, s_2_23, 20, -1, 0},
+/* 24 */ { 4, s_2_24, 20, -1, 0},
+/* 25 */ { 4, s_2_25, 20, -1, 0},
+/* 26 */ { 3, s_2_26, 20, -1, 0},
+/* 27 */ { 2, s_2_27, -1, -1, 0},
+/* 28 */ { 2, s_2_28, -1, -1, 0},
+/* 29 */ { 2, s_2_29, -1, -1, 0},
+/* 30 */ { 2, s_2_30, -1, -1, 0},
+/* 31 */ { 2, s_2_31, -1, -1, 0},
+/* 32 */ { 4, s_2_32, 31, -1, 0},
+/* 33 */ { 6, s_2_33, 31, -1, 0},
+/* 34 */ { 4, s_2_34, 31, -1, 0},
+/* 35 */ { 4, s_2_35, 31, -1, 0},
+/* 36 */ { 4, s_2_36, 31, -1, 0}
+};
+
+static const symbol s_3_0[4] = { 'a', 'n', 'd', 'o' };
+static const symbol s_3_1[4] = { 'e', 'n', 'd', 'o' };
+static const symbol s_3_2[2] = { 'a', 'r' };
+static const symbol s_3_3[2] = { 'e', 'r' };
+static const symbol s_3_4[2] = { 'i', 'r' };
+
+static const struct among a_3[5] =
+{
+/* 0 */ { 4, s_3_0, -1, 1, 0},
+/* 1 */ { 4, s_3_1, -1, 1, 0},
+/* 2 */ { 2, s_3_2, -1, 2, 0},
+/* 3 */ { 2, s_3_3, -1, 2, 0},
+/* 4 */ { 2, s_3_4, -1, 2, 0}
+};
+
+static const symbol s_4_0[2] = { 'i', 'c' };
+static const symbol s_4_1[4] = { 'a', 'b', 'i', 'l' };
+static const symbol s_4_2[2] = { 'o', 's' };
+static const symbol s_4_3[2] = { 'i', 'v' };
+
+static const struct among a_4[4] =
+{
+/* 0 */ { 2, s_4_0, -1, -1, 0},
+/* 1 */ { 4, s_4_1, -1, -1, 0},
+/* 2 */ { 2, s_4_2, -1, -1, 0},
+/* 3 */ { 2, s_4_3, -1, 1, 0}
+};
+
+static const symbol s_5_0[2] = { 'i', 'c' };
+static const symbol s_5_1[4] = { 'a', 'b', 'i', 'l' };
+static const symbol s_5_2[2] = { 'i', 'v' };
+
+static const struct among a_5[3] =
+{
+/* 0 */ { 2, s_5_0, -1, 1, 0},
+/* 1 */ { 4, s_5_1, -1, 1, 0},
+/* 2 */ { 2, s_5_2, -1, 1, 0}
+};
+
+static const symbol s_6_0[3] = { 'i', 'c', 'a' };
+static const symbol s_6_1[5] = { 'l', 'o', 'g', 'i', 'a' };
+static const symbol s_6_2[3] = { 'o', 's', 'a' };
+static const symbol s_6_3[4] = { 'i', 's', 't', 'a' };
+static const symbol s_6_4[3] = { 'i', 'v', 'a' };
+static const symbol s_6_5[4] = { 'a', 'n', 'z', 'a' };
+static const symbol s_6_6[4] = { 'e', 'n', 'z', 'a' };
+static const symbol s_6_7[3] = { 'i', 'c', 'e' };
+static const symbol s_6_8[6] = { 'a', 't', 'r', 'i', 'c', 'e' };
+static const symbol s_6_9[4] = { 'i', 'c', 'h', 'e' };
+static const symbol s_6_10[5] = { 'l', 'o', 'g', 'i', 'e' };
+static const symbol s_6_11[5] = { 'a', 'b', 'i', 'l', 'e' };
+static const symbol s_6_12[5] = { 'i', 'b', 'i', 'l', 'e' };
+static const symbol s_6_13[6] = { 'u', 's', 'i', 'o', 'n', 'e' };
+static const symbol s_6_14[6] = { 'a', 'z', 'i', 'o', 'n', 'e' };
+static const symbol s_6_15[6] = { 'u', 'z', 'i', 'o', 'n', 'e' };
+static const symbol s_6_16[5] = { 'a', 't', 'o', 'r', 'e' };
+static const symbol s_6_17[3] = { 'o', 's', 'e' };
+static const symbol s_6_18[4] = { 'a', 'n', 't', 'e' };
+static const symbol s_6_19[5] = { 'm', 'e', 'n', 't', 'e' };
+static const symbol s_6_20[6] = { 'a', 'm', 'e', 'n', 't', 'e' };
+static const symbol s_6_21[4] = { 'i', 's', 't', 'e' };
+static const symbol s_6_22[3] = { 'i', 'v', 'e' };
+static const symbol s_6_23[4] = { 'a', 'n', 'z', 'e' };
+static const symbol s_6_24[4] = { 'e', 'n', 'z', 'e' };
+static const symbol s_6_25[3] = { 'i', 'c', 'i' };
+static const symbol s_6_26[6] = { 'a', 't', 'r', 'i', 'c', 'i' };
+static const symbol s_6_27[4] = { 'i', 'c', 'h', 'i' };
+static const symbol s_6_28[5] = { 'a', 'b', 'i', 'l', 'i' };
+static const symbol s_6_29[5] = { 'i', 'b', 'i', 'l', 'i' };
+static const symbol s_6_30[4] = { 'i', 's', 'm', 'i' };
+static const symbol s_6_31[6] = { 'u', 's', 'i', 'o', 'n', 'i' };
+static const symbol s_6_32[6] = { 'a', 'z', 'i', 'o', 'n', 'i' };
+static const symbol s_6_33[6] = { 'u', 'z', 'i', 'o', 'n', 'i' };
+static const symbol s_6_34[5] = { 'a', 't', 'o', 'r', 'i' };
+static const symbol s_6_35[3] = { 'o', 's', 'i' };
+static const symbol s_6_36[4] = { 'a', 'n', 't', 'i' };
+static const symbol s_6_37[6] = { 'a', 'm', 'e', 'n', 't', 'i' };
+static const symbol s_6_38[6] = { 'i', 'm', 'e', 'n', 't', 'i' };
+static const symbol s_6_39[4] = { 'i', 's', 't', 'i' };
+static const symbol s_6_40[3] = { 'i', 'v', 'i' };
+static const symbol s_6_41[3] = { 'i', 'c', 'o' };
+static const symbol s_6_42[4] = { 'i', 's', 'm', 'o' };
+static const symbol s_6_43[3] = { 'o', 's', 'o' };
+static const symbol s_6_44[6] = { 'a', 'm', 'e', 'n', 't', 'o' };
+static const symbol s_6_45[6] = { 'i', 'm', 'e', 'n', 't', 'o' };
+static const symbol s_6_46[3] = { 'i', 'v', 'o' };
+static const symbol s_6_47[4] = { 'i', 't', 0xC3, 0xA0 };
+static const symbol s_6_48[5] = { 'i', 's', 't', 0xC3, 0xA0 };
+static const symbol s_6_49[5] = { 'i', 's', 't', 0xC3, 0xA8 };
+static const symbol s_6_50[5] = { 'i', 's', 't', 0xC3, 0xAC };
+
+static const struct among a_6[51] =
+{
+/* 0 */ { 3, s_6_0, -1, 1, 0},
+/* 1 */ { 5, s_6_1, -1, 3, 0},
+/* 2 */ { 3, s_6_2, -1, 1, 0},
+/* 3 */ { 4, s_6_3, -1, 1, 0},
+/* 4 */ { 3, s_6_4, -1, 9, 0},
+/* 5 */ { 4, s_6_5, -1, 1, 0},
+/* 6 */ { 4, s_6_6, -1, 5, 0},
+/* 7 */ { 3, s_6_7, -1, 1, 0},
+/* 8 */ { 6, s_6_8, 7, 1, 0},
+/* 9 */ { 4, s_6_9, -1, 1, 0},
+/* 10 */ { 5, s_6_10, -1, 3, 0},
+/* 11 */ { 5, s_6_11, -1, 1, 0},
+/* 12 */ { 5, s_6_12, -1, 1, 0},
+/* 13 */ { 6, s_6_13, -1, 4, 0},
+/* 14 */ { 6, s_6_14, -1, 2, 0},
+/* 15 */ { 6, s_6_15, -1, 4, 0},
+/* 16 */ { 5, s_6_16, -1, 2, 0},
+/* 17 */ { 3, s_6_17, -1, 1, 0},
+/* 18 */ { 4, s_6_18, -1, 1, 0},
+/* 19 */ { 5, s_6_19, -1, 1, 0},
+/* 20 */ { 6, s_6_20, 19, 7, 0},
+/* 21 */ { 4, s_6_21, -1, 1, 0},
+/* 22 */ { 3, s_6_22, -1, 9, 0},
+/* 23 */ { 4, s_6_23, -1, 1, 0},
+/* 24 */ { 4, s_6_24, -1, 5, 0},
+/* 25 */ { 3, s_6_25, -1, 1, 0},
+/* 26 */ { 6, s_6_26, 25, 1, 0},
+/* 27 */ { 4, s_6_27, -1, 1, 0},
+/* 28 */ { 5, s_6_28, -1, 1, 0},
+/* 29 */ { 5, s_6_29, -1, 1, 0},
+/* 30 */ { 4, s_6_30, -1, 1, 0},
+/* 31 */ { 6, s_6_31, -1, 4, 0},
+/* 32 */ { 6, s_6_32, -1, 2, 0},
+/* 33 */ { 6, s_6_33, -1, 4, 0},
+/* 34 */ { 5, s_6_34, -1, 2, 0},
+/* 35 */ { 3, s_6_35, -1, 1, 0},
+/* 36 */ { 4, s_6_36, -1, 1, 0},
+/* 37 */ { 6, s_6_37, -1, 6, 0},
+/* 38 */ { 6, s_6_38, -1, 6, 0},
+/* 39 */ { 4, s_6_39, -1, 1, 0},
+/* 40 */ { 3, s_6_40, -1, 9, 0},
+/* 41 */ { 3, s_6_41, -1, 1, 0},
+/* 42 */ { 4, s_6_42, -1, 1, 0},
+/* 43 */ { 3, s_6_43, -1, 1, 0},
+/* 44 */ { 6, s_6_44, -1, 6, 0},
+/* 45 */ { 6, s_6_45, -1, 6, 0},
+/* 46 */ { 3, s_6_46, -1, 9, 0},
+/* 47 */ { 4, s_6_47, -1, 8, 0},
+/* 48 */ { 5, s_6_48, -1, 1, 0},
+/* 49 */ { 5, s_6_49, -1, 1, 0},
+/* 50 */ { 5, s_6_50, -1, 1, 0}
+};
+
+static const symbol s_7_0[4] = { 'i', 's', 'c', 'a' };
+static const symbol s_7_1[4] = { 'e', 'n', 'd', 'a' };
+static const symbol s_7_2[3] = { 'a', 't', 'a' };
+static const symbol s_7_3[3] = { 'i', 't', 'a' };
+static const symbol s_7_4[3] = { 'u', 't', 'a' };
+static const symbol s_7_5[3] = { 'a', 'v', 'a' };
+static const symbol s_7_6[3] = { 'e', 'v', 'a' };
+static const symbol s_7_7[3] = { 'i', 'v', 'a' };
+static const symbol s_7_8[6] = { 'e', 'r', 'e', 'b', 'b', 'e' };
+static const symbol s_7_9[6] = { 'i', 'r', 'e', 'b', 'b', 'e' };
+static const symbol s_7_10[4] = { 'i', 's', 'c', 'e' };
+static const symbol s_7_11[4] = { 'e', 'n', 'd', 'e' };
+static const symbol s_7_12[3] = { 'a', 'r', 'e' };
+static const symbol s_7_13[3] = { 'e', 'r', 'e' };
+static const symbol s_7_14[3] = { 'i', 'r', 'e' };
+static const symbol s_7_15[4] = { 'a', 's', 's', 'e' };
+static const symbol s_7_16[3] = { 'a', 't', 'e' };
+static const symbol s_7_17[5] = { 'a', 'v', 'a', 't', 'e' };
+static const symbol s_7_18[5] = { 'e', 'v', 'a', 't', 'e' };
+static const symbol s_7_19[5] = { 'i', 'v', 'a', 't', 'e' };
+static const symbol s_7_20[3] = { 'e', 't', 'e' };
+static const symbol s_7_21[5] = { 'e', 'r', 'e', 't', 'e' };
+static const symbol s_7_22[5] = { 'i', 'r', 'e', 't', 'e' };
+static const symbol s_7_23[3] = { 'i', 't', 'e' };
+static const symbol s_7_24[6] = { 'e', 'r', 'e', 's', 't', 'e' };
+static const symbol s_7_25[6] = { 'i', 'r', 'e', 's', 't', 'e' };
+static const symbol s_7_26[3] = { 'u', 't', 'e' };
+static const symbol s_7_27[4] = { 'e', 'r', 'a', 'i' };
+static const symbol s_7_28[4] = { 'i', 'r', 'a', 'i' };
+static const symbol s_7_29[4] = { 'i', 's', 'c', 'i' };
+static const symbol s_7_30[4] = { 'e', 'n', 'd', 'i' };
+static const symbol s_7_31[4] = { 'e', 'r', 'e', 'i' };
+static const symbol s_7_32[4] = { 'i', 'r', 'e', 'i' };
+static const symbol s_7_33[4] = { 'a', 's', 's', 'i' };
+static const symbol s_7_34[3] = { 'a', 't', 'i' };
+static const symbol s_7_35[3] = { 'i', 't', 'i' };
+static const symbol s_7_36[6] = { 'e', 'r', 'e', 's', 't', 'i' };
+static const symbol s_7_37[6] = { 'i', 'r', 'e', 's', 't', 'i' };
+static const symbol s_7_38[3] = { 'u', 't', 'i' };
+static const symbol s_7_39[3] = { 'a', 'v', 'i' };
+static const symbol s_7_40[3] = { 'e', 'v', 'i' };
+static const symbol s_7_41[3] = { 'i', 'v', 'i' };
+static const symbol s_7_42[4] = { 'i', 's', 'c', 'o' };
+static const symbol s_7_43[4] = { 'a', 'n', 'd', 'o' };
+static const symbol s_7_44[4] = { 'e', 'n', 'd', 'o' };
+static const symbol s_7_45[4] = { 'Y', 'a', 'm', 'o' };
+static const symbol s_7_46[4] = { 'i', 'a', 'm', 'o' };
+static const symbol s_7_47[5] = { 'a', 'v', 'a', 'm', 'o' };
+static const symbol s_7_48[5] = { 'e', 'v', 'a', 'm', 'o' };
+static const symbol s_7_49[5] = { 'i', 'v', 'a', 'm', 'o' };
+static const symbol s_7_50[5] = { 'e', 'r', 'e', 'm', 'o' };
+static const symbol s_7_51[5] = { 'i', 'r', 'e', 'm', 'o' };
+static const symbol s_7_52[6] = { 'a', 's', 's', 'i', 'm', 'o' };
+static const symbol s_7_53[4] = { 'a', 'm', 'm', 'o' };
+static const symbol s_7_54[4] = { 'e', 'm', 'm', 'o' };
+static const symbol s_7_55[6] = { 'e', 'r', 'e', 'm', 'm', 'o' };
+static const symbol s_7_56[6] = { 'i', 'r', 'e', 'm', 'm', 'o' };
+static const symbol s_7_57[4] = { 'i', 'm', 'm', 'o' };
+static const symbol s_7_58[3] = { 'a', 'n', 'o' };
+static const symbol s_7_59[6] = { 'i', 's', 'c', 'a', 'n', 'o' };
+static const symbol s_7_60[5] = { 'a', 'v', 'a', 'n', 'o' };
+static const symbol s_7_61[5] = { 'e', 'v', 'a', 'n', 'o' };
+static const symbol s_7_62[5] = { 'i', 'v', 'a', 'n', 'o' };
+static const symbol s_7_63[6] = { 'e', 'r', 'a', 'n', 'n', 'o' };
+static const symbol s_7_64[6] = { 'i', 'r', 'a', 'n', 'n', 'o' };
+static const symbol s_7_65[3] = { 'o', 'n', 'o' };
+static const symbol s_7_66[6] = { 'i', 's', 'c', 'o', 'n', 'o' };
+static const symbol s_7_67[5] = { 'a', 'r', 'o', 'n', 'o' };
+static const symbol s_7_68[5] = { 'e', 'r', 'o', 'n', 'o' };
+static const symbol s_7_69[5] = { 'i', 'r', 'o', 'n', 'o' };
+static const symbol s_7_70[8] = { 'e', 'r', 'e', 'b', 'b', 'e', 'r', 'o' };
+static const symbol s_7_71[8] = { 'i', 'r', 'e', 'b', 'b', 'e', 'r', 'o' };
+static const symbol s_7_72[6] = { 'a', 's', 's', 'e', 'r', 'o' };
+static const symbol s_7_73[6] = { 'e', 's', 's', 'e', 'r', 'o' };
+static const symbol s_7_74[6] = { 'i', 's', 's', 'e', 'r', 'o' };
+static const symbol s_7_75[3] = { 'a', 't', 'o' };
+static const symbol s_7_76[3] = { 'i', 't', 'o' };
+static const symbol s_7_77[3] = { 'u', 't', 'o' };
+static const symbol s_7_78[3] = { 'a', 'v', 'o' };
+static const symbol s_7_79[3] = { 'e', 'v', 'o' };
+static const symbol s_7_80[3] = { 'i', 'v', 'o' };
+static const symbol s_7_81[2] = { 'a', 'r' };
+static const symbol s_7_82[2] = { 'i', 'r' };
+static const symbol s_7_83[4] = { 'e', 'r', 0xC3, 0xA0 };
+static const symbol s_7_84[4] = { 'i', 'r', 0xC3, 0xA0 };
+static const symbol s_7_85[4] = { 'e', 'r', 0xC3, 0xB2 };
+static const symbol s_7_86[4] = { 'i', 'r', 0xC3, 0xB2 };
+
+static const struct among a_7[87] =
+{
+/* 0 */ { 4, s_7_0, -1, 1, 0},
+/* 1 */ { 4, s_7_1, -1, 1, 0},
+/* 2 */ { 3, s_7_2, -1, 1, 0},
+/* 3 */ { 3, s_7_3, -1, 1, 0},
+/* 4 */ { 3, s_7_4, -1, 1, 0},
+/* 5 */ { 3, s_7_5, -1, 1, 0},
+/* 6 */ { 3, s_7_6, -1, 1, 0},
+/* 7 */ { 3, s_7_7, -1, 1, 0},
+/* 8 */ { 6, s_7_8, -1, 1, 0},
+/* 9 */ { 6, s_7_9, -1, 1, 0},
+/* 10 */ { 4, s_7_10, -1, 1, 0},
+/* 11 */ { 4, s_7_11, -1, 1, 0},
+/* 12 */ { 3, s_7_12, -1, 1, 0},
+/* 13 */ { 3, s_7_13, -1, 1, 0},
+/* 14 */ { 3, s_7_14, -1, 1, 0},
+/* 15 */ { 4, s_7_15, -1, 1, 0},
+/* 16 */ { 3, s_7_16, -1, 1, 0},
+/* 17 */ { 5, s_7_17, 16, 1, 0},
+/* 18 */ { 5, s_7_18, 16, 1, 0},
+/* 19 */ { 5, s_7_19, 16, 1, 0},
+/* 20 */ { 3, s_7_20, -1, 1, 0},
+/* 21 */ { 5, s_7_21, 20, 1, 0},
+/* 22 */ { 5, s_7_22, 20, 1, 0},
+/* 23 */ { 3, s_7_23, -1, 1, 0},
+/* 24 */ { 6, s_7_24, -1, 1, 0},
+/* 25 */ { 6, s_7_25, -1, 1, 0},
+/* 26 */ { 3, s_7_26, -1, 1, 0},
+/* 27 */ { 4, s_7_27, -1, 1, 0},
+/* 28 */ { 4, s_7_28, -1, 1, 0},
+/* 29 */ { 4, s_7_29, -1, 1, 0},
+/* 30 */ { 4, s_7_30, -1, 1, 0},
+/* 31 */ { 4, s_7_31, -1, 1, 0},
+/* 32 */ { 4, s_7_32, -1, 1, 0},
+/* 33 */ { 4, s_7_33, -1, 1, 0},
+/* 34 */ { 3, s_7_34, -1, 1, 0},
+/* 35 */ { 3, s_7_35, -1, 1, 0},
+/* 36 */ { 6, s_7_36, -1, 1, 0},
+/* 37 */ { 6, s_7_37, -1, 1, 0},
+/* 38 */ { 3, s_7_38, -1, 1, 0},
+/* 39 */ { 3, s_7_39, -1, 1, 0},
+/* 40 */ { 3, s_7_40, -1, 1, 0},
+/* 41 */ { 3, s_7_41, -1, 1, 0},
+/* 42 */ { 4, s_7_42, -1, 1, 0},
+/* 43 */ { 4, s_7_43, -1, 1, 0},
+/* 44 */ { 4, s_7_44, -1, 1, 0},
+/* 45 */ { 4, s_7_45, -1, 1, 0},
+/* 46 */ { 4, s_7_46, -1, 1, 0},
+/* 47 */ { 5, s_7_47, -1, 1, 0},
+/* 48 */ { 5, s_7_48, -1, 1, 0},
+/* 49 */ { 5, s_7_49, -1, 1, 0},
+/* 50 */ { 5, s_7_50, -1, 1, 0},
+/* 51 */ { 5, s_7_51, -1, 1, 0},
+/* 52 */ { 6, s_7_52, -1, 1, 0},
+/* 53 */ { 4, s_7_53, -1, 1, 0},
+/* 54 */ { 4, s_7_54, -1, 1, 0},
+/* 55 */ { 6, s_7_55, 54, 1, 0},
+/* 56 */ { 6, s_7_56, 54, 1, 0},
+/* 57 */ { 4, s_7_57, -1, 1, 0},
+/* 58 */ { 3, s_7_58, -1, 1, 0},
+/* 59 */ { 6, s_7_59, 58, 1, 0},
+/* 60 */ { 5, s_7_60, 58, 1, 0},
+/* 61 */ { 5, s_7_61, 58, 1, 0},
+/* 62 */ { 5, s_7_62, 58, 1, 0},
+/* 63 */ { 6, s_7_63, -1, 1, 0},
+/* 64 */ { 6, s_7_64, -1, 1, 0},
+/* 65 */ { 3, s_7_65, -1, 1, 0},
+/* 66 */ { 6, s_7_66, 65, 1, 0},
+/* 67 */ { 5, s_7_67, 65, 1, 0},
+/* 68 */ { 5, s_7_68, 65, 1, 0},
+/* 69 */ { 5, s_7_69, 65, 1, 0},
+/* 70 */ { 8, s_7_70, -1, 1, 0},
+/* 71 */ { 8, s_7_71, -1, 1, 0},
+/* 72 */ { 6, s_7_72, -1, 1, 0},
+/* 73 */ { 6, s_7_73, -1, 1, 0},
+/* 74 */ { 6, s_7_74, -1, 1, 0},
+/* 75 */ { 3, s_7_75, -1, 1, 0},
+/* 76 */ { 3, s_7_76, -1, 1, 0},
+/* 77 */ { 3, s_7_77, -1, 1, 0},
+/* 78 */ { 3, s_7_78, -1, 1, 0},
+/* 79 */ { 3, s_7_79, -1, 1, 0},
+/* 80 */ { 3, s_7_80, -1, 1, 0},
+/* 81 */ { 2, s_7_81, -1, 1, 0},
+/* 82 */ { 2, s_7_82, -1, 1, 0},
+/* 83 */ { 4, s_7_83, -1, 1, 0},
+/* 84 */ { 4, s_7_84, -1, 1, 0},
+/* 85 */ { 4, s_7_85, -1, 1, 0},
+/* 86 */ { 4, s_7_86, -1, 1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 8, 2, 1 };
+
+static const unsigned char g_AEIO[] = { 17, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 128, 8, 2 };
+
+static const unsigned char g_CG[] = { 17 };
+
+static const symbol s_0[] = { 0xC3, 0xA0 };
+static const symbol s_1[] = { 0xC3, 0xA8 };
+static const symbol s_2[] = { 0xC3, 0xAC };
+static const symbol s_3[] = { 0xC3, 0xB2 };
+static const symbol s_4[] = { 0xC3, 0xB9 };
+static const symbol s_5[] = { 'q', 'U' };
+static const symbol s_6[] = { 'u' };
+static const symbol s_7[] = { 'U' };
+static const symbol s_8[] = { 'i' };
+static const symbol s_9[] = { 'I' };
+static const symbol s_10[] = { 'i' };
+static const symbol s_11[] = { 'u' };
+static const symbol s_12[] = { 'e' };
+static const symbol s_13[] = { 'i', 'c' };
+static const symbol s_14[] = { 'l', 'o', 'g' };
+static const symbol s_15[] = { 'u' };
+static const symbol s_16[] = { 'e', 'n', 't', 'e' };
+static const symbol s_17[] = { 'a', 't' };
+static const symbol s_18[] = { 'a', 't' };
+static const symbol s_19[] = { 'i', 'c' };
+static const symbol s_20[] = { 'i' };
+static const symbol s_21[] = { 'h' };
+
+static int r_prelude(struct SN_env * z) {
+ int among_var;
+ { int c_test = z->c; /* test, line 35 */
+ while(1) { /* repeat, line 35 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 36 */
+ among_var = find_among(z, a_0, 7); /* substring, line 36 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 36 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 2, s_0); /* <-, line 37 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 2, s_1); /* <-, line 38 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 2, s_2); /* <-, line 39 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 2, s_3); /* <-, line 40 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 2, s_4); /* <-, line 41 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 2, s_5); /* <-, line 42 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 43 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ z->c = c_test;
+ }
+ while(1) { /* repeat, line 46 */
+ int c2 = z->c;
+ while(1) { /* goto, line 46 */
+ int c3 = z->c;
+ if (in_grouping_U(z, g_v, 97, 249, 0)) goto lab2;
+ z->bra = z->c; /* [, line 47 */
+ { int c4 = z->c; /* or, line 47 */
+ if (!(eq_s(z, 1, s_6))) goto lab4;
+ z->ket = z->c; /* ], line 47 */
+ if (in_grouping_U(z, g_v, 97, 249, 0)) goto lab4;
+ { int ret = slice_from_s(z, 1, s_7); /* <-, line 47 */
+ if (ret < 0) return ret;
+ }
+ goto lab3;
+ lab4:
+ z->c = c4;
+ if (!(eq_s(z, 1, s_8))) goto lab2;
+ z->ket = z->c; /* ], line 48 */
+ if (in_grouping_U(z, g_v, 97, 249, 0)) goto lab2;
+ { int ret = slice_from_s(z, 1, s_9); /* <-, line 48 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab3:
+ z->c = c3;
+ break;
+ lab2:
+ z->c = c3;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* goto, line 46 */
+ }
+ }
+ continue;
+ lab1:
+ z->c = c2;
+ break;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ z->I[2] = z->l;
+ { int c1 = z->c; /* do, line 58 */
+ { int c2 = z->c; /* or, line 60 */
+ if (in_grouping_U(z, g_v, 97, 249, 0)) goto lab2;
+ { int c3 = z->c; /* or, line 59 */
+ if (out_grouping_U(z, g_v, 97, 249, 0)) goto lab4;
+ { /* gopast */ /* grouping v, line 59 */
+ int ret = out_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ goto lab3;
+ lab4:
+ z->c = c3;
+ if (in_grouping_U(z, g_v, 97, 249, 0)) goto lab2;
+ { /* gopast */ /* non v, line 59 */
+ int ret = in_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab2;
+ z->c += ret;
+ }
+ }
+ lab3:
+ goto lab1;
+ lab2:
+ z->c = c2;
+ if (out_grouping_U(z, g_v, 97, 249, 0)) goto lab0;
+ { int c4 = z->c; /* or, line 61 */
+ if (out_grouping_U(z, g_v, 97, 249, 0)) goto lab6;
+ { /* gopast */ /* grouping v, line 61 */
+ int ret = out_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab6;
+ z->c += ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = c4;
+ if (in_grouping_U(z, g_v, 97, 249, 0)) goto lab0;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 61 */
+ }
+ }
+ lab5:
+ ;
+ }
+ lab1:
+ z->I[0] = z->c; /* setmark pV, line 62 */
+ lab0:
+ z->c = c1;
+ }
+ { int c5 = z->c; /* do, line 64 */
+ { /* gopast */ /* grouping v, line 65 */
+ int ret = out_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 65 */
+ int ret = in_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p1, line 65 */
+ { /* gopast */ /* grouping v, line 66 */
+ int ret = out_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 66 */
+ int ret = in_grouping_U(z, g_v, 97, 249, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[2] = z->c; /* setmark p2, line 66 */
+ lab7:
+ z->c = c5;
+ }
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 70 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 72 */
+ if (z->c >= z->l || (z->p[z->c + 0] != 73 && z->p[z->c + 0] != 85)) among_var = 3; else
+ among_var = find_among(z, a_1, 3); /* substring, line 72 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 72 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_10); /* <-, line 73 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_11); /* <-, line 74 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 75 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_RV(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[2] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_attached_pronoun(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 87 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((33314 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_2, 37))) return 0; /* substring, line 87 */
+ z->bra = z->c; /* ], line 87 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 111 && z->p[z->c - 1] != 114)) return 0;
+ among_var = find_among_b(z, a_3, 5); /* among, line 97 */
+ if (!(among_var)) return 0;
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 97 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 98 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_12); /* <-, line 99 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 104 */
+ among_var = find_among_b(z, a_6, 51); /* substring, line 104 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 104 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 111 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 111 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 113 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 113 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 114 */
+ z->ket = z->c; /* [, line 114 */
+ if (!(eq_s_b(z, 2, s_13))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 114 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call R2, line 114 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 114 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ ;
+ }
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 117 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_14); /* <-, line 117 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 119 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 1, s_15); /* <-, line 119 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 121 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 4, s_16); /* <-, line 121 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 123 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 123 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 125 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 125 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 126 */
+ z->ket = z->c; /* [, line 127 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4722696 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab1; }
+ among_var = find_among_b(z, a_4, 4); /* substring, line 127 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 127 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab1; } /* call R2, line 127 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 127 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab1; }
+ case 1:
+ z->ket = z->c; /* [, line 128 */
+ if (!(eq_s_b(z, 2, s_17))) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 128 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab1; } /* call R2, line 128 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 128 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab1:
+ ;
+ }
+ break;
+ case 8:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 134 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 134 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 135 */
+ z->ket = z->c; /* [, line 136 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4198408 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab2; }
+ among_var = find_among_b(z, a_5, 3); /* substring, line 136 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab2; }
+ z->bra = z->c; /* ], line 136 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab2; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab2; } /* call R2, line 137 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 137 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab2:
+ ;
+ }
+ break;
+ case 9:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 142 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 142 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 143 */
+ z->ket = z->c; /* [, line 143 */
+ if (!(eq_s_b(z, 2, s_18))) { z->c = z->l - m_keep; goto lab3; }
+ z->bra = z->c; /* ], line 143 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 143 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 143 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 143 */
+ if (!(eq_s_b(z, 2, s_19))) { z->c = z->l - m_keep; goto lab3; }
+ z->bra = z->c; /* ], line 143 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 143 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 143 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ ;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 148 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 148 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 149 */
+ among_var = find_among_b(z, a_7, 87); /* substring, line 149 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 149 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ { int ret = slice_del(z); /* delete, line 163 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_vowel_suffix(struct SN_env * z) {
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 171 */
+ z->ket = z->c; /* [, line 172 */
+ if (in_grouping_b_U(z, g_AEIO, 97, 242, 0)) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 172 */
+ { int ret = r_RV(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call RV, line 172 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 172 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 173 */
+ if (!(eq_s_b(z, 1, s_20))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 173 */
+ { int ret = r_RV(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call RV, line 173 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 173 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ ;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 175 */
+ z->ket = z->c; /* [, line 176 */
+ if (!(eq_s_b(z, 1, s_21))) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 176 */
+ if (in_grouping_b_U(z, g_CG, 99, 103, 0)) { z->c = z->l - m_keep; goto lab1; }
+ { int ret = r_RV(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab1; } /* call RV, line 176 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 176 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ ;
+ }
+ return 1;
+}
+
+extern int italian_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 182 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab0; /* call prelude, line 182 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 183 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab1; /* call mark_regions, line 183 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = c2;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 184 */
+
+ { int m3 = z->l - z->c; (void)m3; /* do, line 185 */
+ { int ret = r_attached_pronoun(z);
+ if (ret == 0) goto lab2; /* call attached_pronoun, line 185 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 186 */
+ { int m5 = z->l - z->c; (void)m5; /* or, line 186 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab5; /* call standard_suffix, line 186 */
+ if (ret < 0) return ret;
+ }
+ goto lab4;
+ lab5:
+ z->c = z->l - m5;
+ { int ret = r_verb_suffix(z);
+ if (ret == 0) goto lab3; /* call verb_suffix, line 186 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab4:
+ lab3:
+ z->c = z->l - m4;
+ }
+ { int m6 = z->l - z->c; (void)m6; /* do, line 187 */
+ { int ret = r_vowel_suffix(z);
+ if (ret == 0) goto lab6; /* call vowel_suffix, line 187 */
+ if (ret < 0) return ret;
+ }
+ lab6:
+ z->c = z->l - m6;
+ }
+ z->c = z->lb;
+ { int c7 = z->c; /* do, line 189 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab7; /* call postlude, line 189 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = c7;
+ }
+ return 1;
+}
+
+extern struct SN_env * italian_UTF_8_create_env(void) { return SN_create_env(0, 3, 0); }
+
+extern void italian_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_italian.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_italian.h
new file mode 100644
index 0000000..3bee080
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_italian.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * italian_UTF_8_create_env(void);
+extern void italian_UTF_8_close_env(struct SN_env * z);
+
+extern int italian_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_norwegian.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_norwegian.c
new file mode 100644
index 0000000..6a7b5f4
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_norwegian.c
@@ -0,0 +1,299 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int norwegian_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_other_suffix(struct SN_env * z);
+static int r_consonant_pair(struct SN_env * z);
+static int r_main_suffix(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * norwegian_UTF_8_create_env(void);
+extern void norwegian_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[1] = { 'a' };
+static const symbol s_0_1[1] = { 'e' };
+static const symbol s_0_2[3] = { 'e', 'd', 'e' };
+static const symbol s_0_3[4] = { 'a', 'n', 'd', 'e' };
+static const symbol s_0_4[4] = { 'e', 'n', 'd', 'e' };
+static const symbol s_0_5[3] = { 'a', 'n', 'e' };
+static const symbol s_0_6[3] = { 'e', 'n', 'e' };
+static const symbol s_0_7[6] = { 'h', 'e', 't', 'e', 'n', 'e' };
+static const symbol s_0_8[4] = { 'e', 'r', 't', 'e' };
+static const symbol s_0_9[2] = { 'e', 'n' };
+static const symbol s_0_10[5] = { 'h', 'e', 't', 'e', 'n' };
+static const symbol s_0_11[2] = { 'a', 'r' };
+static const symbol s_0_12[2] = { 'e', 'r' };
+static const symbol s_0_13[5] = { 'h', 'e', 't', 'e', 'r' };
+static const symbol s_0_14[1] = { 's' };
+static const symbol s_0_15[2] = { 'a', 's' };
+static const symbol s_0_16[2] = { 'e', 's' };
+static const symbol s_0_17[4] = { 'e', 'd', 'e', 's' };
+static const symbol s_0_18[5] = { 'e', 'n', 'd', 'e', 's' };
+static const symbol s_0_19[4] = { 'e', 'n', 'e', 's' };
+static const symbol s_0_20[7] = { 'h', 'e', 't', 'e', 'n', 'e', 's' };
+static const symbol s_0_21[3] = { 'e', 'n', 's' };
+static const symbol s_0_22[6] = { 'h', 'e', 't', 'e', 'n', 's' };
+static const symbol s_0_23[3] = { 'e', 'r', 's' };
+static const symbol s_0_24[3] = { 'e', 't', 's' };
+static const symbol s_0_25[2] = { 'e', 't' };
+static const symbol s_0_26[3] = { 'h', 'e', 't' };
+static const symbol s_0_27[3] = { 'e', 'r', 't' };
+static const symbol s_0_28[3] = { 'a', 's', 't' };
+
+static const struct among a_0[29] =
+{
+/* 0 */ { 1, s_0_0, -1, 1, 0},
+/* 1 */ { 1, s_0_1, -1, 1, 0},
+/* 2 */ { 3, s_0_2, 1, 1, 0},
+/* 3 */ { 4, s_0_3, 1, 1, 0},
+/* 4 */ { 4, s_0_4, 1, 1, 0},
+/* 5 */ { 3, s_0_5, 1, 1, 0},
+/* 6 */ { 3, s_0_6, 1, 1, 0},
+/* 7 */ { 6, s_0_7, 6, 1, 0},
+/* 8 */ { 4, s_0_8, 1, 3, 0},
+/* 9 */ { 2, s_0_9, -1, 1, 0},
+/* 10 */ { 5, s_0_10, 9, 1, 0},
+/* 11 */ { 2, s_0_11, -1, 1, 0},
+/* 12 */ { 2, s_0_12, -1, 1, 0},
+/* 13 */ { 5, s_0_13, 12, 1, 0},
+/* 14 */ { 1, s_0_14, -1, 2, 0},
+/* 15 */ { 2, s_0_15, 14, 1, 0},
+/* 16 */ { 2, s_0_16, 14, 1, 0},
+/* 17 */ { 4, s_0_17, 16, 1, 0},
+/* 18 */ { 5, s_0_18, 16, 1, 0},
+/* 19 */ { 4, s_0_19, 16, 1, 0},
+/* 20 */ { 7, s_0_20, 19, 1, 0},
+/* 21 */ { 3, s_0_21, 14, 1, 0},
+/* 22 */ { 6, s_0_22, 21, 1, 0},
+/* 23 */ { 3, s_0_23, 14, 1, 0},
+/* 24 */ { 3, s_0_24, 14, 1, 0},
+/* 25 */ { 2, s_0_25, -1, 1, 0},
+/* 26 */ { 3, s_0_26, 25, 1, 0},
+/* 27 */ { 3, s_0_27, -1, 3, 0},
+/* 28 */ { 3, s_0_28, -1, 1, 0}
+};
+
+static const symbol s_1_0[2] = { 'd', 't' };
+static const symbol s_1_1[2] = { 'v', 't' };
+
+static const struct among a_1[2] =
+{
+/* 0 */ { 2, s_1_0, -1, -1, 0},
+/* 1 */ { 2, s_1_1, -1, -1, 0}
+};
+
+static const symbol s_2_0[3] = { 'l', 'e', 'g' };
+static const symbol s_2_1[4] = { 'e', 'l', 'e', 'g' };
+static const symbol s_2_2[2] = { 'i', 'g' };
+static const symbol s_2_3[3] = { 'e', 'i', 'g' };
+static const symbol s_2_4[3] = { 'l', 'i', 'g' };
+static const symbol s_2_5[4] = { 'e', 'l', 'i', 'g' };
+static const symbol s_2_6[3] = { 'e', 'l', 's' };
+static const symbol s_2_7[3] = { 'l', 'o', 'v' };
+static const symbol s_2_8[4] = { 'e', 'l', 'o', 'v' };
+static const symbol s_2_9[4] = { 's', 'l', 'o', 'v' };
+static const symbol s_2_10[7] = { 'h', 'e', 't', 's', 'l', 'o', 'v' };
+
+static const struct among a_2[11] =
+{
+/* 0 */ { 3, s_2_0, -1, 1, 0},
+/* 1 */ { 4, s_2_1, 0, 1, 0},
+/* 2 */ { 2, s_2_2, -1, 1, 0},
+/* 3 */ { 3, s_2_3, 2, 1, 0},
+/* 4 */ { 3, s_2_4, 2, 1, 0},
+/* 5 */ { 4, s_2_5, 4, 1, 0},
+/* 6 */ { 3, s_2_6, -1, 1, 0},
+/* 7 */ { 3, s_2_7, -1, 1, 0},
+/* 8 */ { 4, s_2_8, 7, 1, 0},
+/* 9 */ { 4, s_2_9, 7, 1, 0},
+/* 10 */ { 7, s_2_10, 9, 1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 128 };
+
+static const unsigned char g_s_ending[] = { 119, 125, 149, 1 };
+
+static const symbol s_0[] = { 'k' };
+static const symbol s_1[] = { 'e', 'r' };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ { int c_test = z->c; /* test, line 30 */
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, + 3);
+ if (ret < 0) return 0;
+ z->c = ret; /* hop, line 30 */
+ }
+ z->I[1] = z->c; /* setmark x, line 30 */
+ z->c = c_test;
+ }
+ if (out_grouping_U(z, g_v, 97, 248, 1) < 0) return 0; /* goto */ /* grouping v, line 31 */
+ { /* gopast */ /* non v, line 31 */
+ int ret = in_grouping_U(z, g_v, 97, 248, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 31 */
+ /* try, line 32 */
+ if (!(z->I[0] < z->I[1])) goto lab0;
+ z->I[0] = z->I[1];
+lab0:
+ return 1;
+}
+
+static int r_main_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 38 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 38 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 38 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1851426 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_0, 29); /* substring, line 38 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 38 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 44 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m2 = z->l - z->c; (void)m2; /* or, line 46 */
+ if (in_grouping_b_U(z, g_s_ending, 98, 122, 0)) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m2;
+ if (!(eq_s_b(z, 1, s_0))) return 0;
+ if (out_grouping_b_U(z, g_v, 97, 248, 0)) return 0;
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 46 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 2, s_1); /* <-, line 48 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_consonant_pair(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 53 */
+ { int mlimit; /* setlimit, line 54 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 54 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 54 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] != 116) { z->lb = mlimit; return 0; }
+ if (!(find_among_b(z, a_1, 2))) { z->lb = mlimit; return 0; } /* substring, line 54 */
+ z->bra = z->c; /* ], line 54 */
+ z->lb = mlimit;
+ }
+ z->c = z->l - m_test;
+ }
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 59 */
+ }
+ z->bra = z->c; /* ], line 59 */
+ { int ret = slice_del(z); /* delete, line 59 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_other_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 63 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 63 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 63 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4718720 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_2, 11); /* substring, line 63 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 63 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 67 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+extern int norwegian_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 74 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 74 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 75 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 76 */
+ { int ret = r_main_suffix(z);
+ if (ret == 0) goto lab1; /* call main_suffix, line 76 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 77 */
+ { int ret = r_consonant_pair(z);
+ if (ret == 0) goto lab2; /* call consonant_pair, line 77 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 78 */
+ { int ret = r_other_suffix(z);
+ if (ret == 0) goto lab3; /* call other_suffix, line 78 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = z->l - m4;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern struct SN_env * norwegian_UTF_8_create_env(void) { return SN_create_env(0, 2, 0); }
+
+extern void norwegian_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_norwegian.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_norwegian.h
new file mode 100644
index 0000000..c75444b
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_norwegian.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * norwegian_UTF_8_create_env(void);
+extern void norwegian_UTF_8_close_env(struct SN_env * z);
+
+extern int norwegian_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_porter.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_porter.c
new file mode 100644
index 0000000..0c4813f
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_porter.c
@@ -0,0 +1,755 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int porter_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_Step_5b(struct SN_env * z);
+static int r_Step_5a(struct SN_env * z);
+static int r_Step_4(struct SN_env * z);
+static int r_Step_3(struct SN_env * z);
+static int r_Step_2(struct SN_env * z);
+static int r_Step_1c(struct SN_env * z);
+static int r_Step_1b(struct SN_env * z);
+static int r_Step_1a(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_shortv(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * porter_UTF_8_create_env(void);
+extern void porter_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[1] = { 's' };
+static const symbol s_0_1[3] = { 'i', 'e', 's' };
+static const symbol s_0_2[4] = { 's', 's', 'e', 's' };
+static const symbol s_0_3[2] = { 's', 's' };
+
+static const struct among a_0[4] =
+{
+/* 0 */ { 1, s_0_0, -1, 3, 0},
+/* 1 */ { 3, s_0_1, 0, 2, 0},
+/* 2 */ { 4, s_0_2, 0, 1, 0},
+/* 3 */ { 2, s_0_3, 0, -1, 0}
+};
+
+static const symbol s_1_1[2] = { 'b', 'b' };
+static const symbol s_1_2[2] = { 'd', 'd' };
+static const symbol s_1_3[2] = { 'f', 'f' };
+static const symbol s_1_4[2] = { 'g', 'g' };
+static const symbol s_1_5[2] = { 'b', 'l' };
+static const symbol s_1_6[2] = { 'm', 'm' };
+static const symbol s_1_7[2] = { 'n', 'n' };
+static const symbol s_1_8[2] = { 'p', 'p' };
+static const symbol s_1_9[2] = { 'r', 'r' };
+static const symbol s_1_10[2] = { 'a', 't' };
+static const symbol s_1_11[2] = { 't', 't' };
+static const symbol s_1_12[2] = { 'i', 'z' };
+
+static const struct among a_1[13] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 2, s_1_1, 0, 2, 0},
+/* 2 */ { 2, s_1_2, 0, 2, 0},
+/* 3 */ { 2, s_1_3, 0, 2, 0},
+/* 4 */ { 2, s_1_4, 0, 2, 0},
+/* 5 */ { 2, s_1_5, 0, 1, 0},
+/* 6 */ { 2, s_1_6, 0, 2, 0},
+/* 7 */ { 2, s_1_7, 0, 2, 0},
+/* 8 */ { 2, s_1_8, 0, 2, 0},
+/* 9 */ { 2, s_1_9, 0, 2, 0},
+/* 10 */ { 2, s_1_10, 0, 1, 0},
+/* 11 */ { 2, s_1_11, 0, 2, 0},
+/* 12 */ { 2, s_1_12, 0, 1, 0}
+};
+
+static const symbol s_2_0[2] = { 'e', 'd' };
+static const symbol s_2_1[3] = { 'e', 'e', 'd' };
+static const symbol s_2_2[3] = { 'i', 'n', 'g' };
+
+static const struct among a_2[3] =
+{
+/* 0 */ { 2, s_2_0, -1, 2, 0},
+/* 1 */ { 3, s_2_1, 0, 1, 0},
+/* 2 */ { 3, s_2_2, -1, 2, 0}
+};
+
+static const symbol s_3_0[4] = { 'a', 'n', 'c', 'i' };
+static const symbol s_3_1[4] = { 'e', 'n', 'c', 'i' };
+static const symbol s_3_2[4] = { 'a', 'b', 'l', 'i' };
+static const symbol s_3_3[3] = { 'e', 'l', 'i' };
+static const symbol s_3_4[4] = { 'a', 'l', 'l', 'i' };
+static const symbol s_3_5[5] = { 'o', 'u', 's', 'l', 'i' };
+static const symbol s_3_6[5] = { 'e', 'n', 't', 'l', 'i' };
+static const symbol s_3_7[5] = { 'a', 'l', 'i', 't', 'i' };
+static const symbol s_3_8[6] = { 'b', 'i', 'l', 'i', 't', 'i' };
+static const symbol s_3_9[5] = { 'i', 'v', 'i', 't', 'i' };
+static const symbol s_3_10[6] = { 't', 'i', 'o', 'n', 'a', 'l' };
+static const symbol s_3_11[7] = { 'a', 't', 'i', 'o', 'n', 'a', 'l' };
+static const symbol s_3_12[5] = { 'a', 'l', 'i', 's', 'm' };
+static const symbol s_3_13[5] = { 'a', 't', 'i', 'o', 'n' };
+static const symbol s_3_14[7] = { 'i', 'z', 'a', 't', 'i', 'o', 'n' };
+static const symbol s_3_15[4] = { 'i', 'z', 'e', 'r' };
+static const symbol s_3_16[4] = { 'a', 't', 'o', 'r' };
+static const symbol s_3_17[7] = { 'i', 'v', 'e', 'n', 'e', 's', 's' };
+static const symbol s_3_18[7] = { 'f', 'u', 'l', 'n', 'e', 's', 's' };
+static const symbol s_3_19[7] = { 'o', 'u', 's', 'n', 'e', 's', 's' };
+
+static const struct among a_3[20] =
+{
+/* 0 */ { 4, s_3_0, -1, 3, 0},
+/* 1 */ { 4, s_3_1, -1, 2, 0},
+/* 2 */ { 4, s_3_2, -1, 4, 0},
+/* 3 */ { 3, s_3_3, -1, 6, 0},
+/* 4 */ { 4, s_3_4, -1, 9, 0},
+/* 5 */ { 5, s_3_5, -1, 12, 0},
+/* 6 */ { 5, s_3_6, -1, 5, 0},
+/* 7 */ { 5, s_3_7, -1, 10, 0},
+/* 8 */ { 6, s_3_8, -1, 14, 0},
+/* 9 */ { 5, s_3_9, -1, 13, 0},
+/* 10 */ { 6, s_3_10, -1, 1, 0},
+/* 11 */ { 7, s_3_11, 10, 8, 0},
+/* 12 */ { 5, s_3_12, -1, 10, 0},
+/* 13 */ { 5, s_3_13, -1, 8, 0},
+/* 14 */ { 7, s_3_14, 13, 7, 0},
+/* 15 */ { 4, s_3_15, -1, 7, 0},
+/* 16 */ { 4, s_3_16, -1, 8, 0},
+/* 17 */ { 7, s_3_17, -1, 13, 0},
+/* 18 */ { 7, s_3_18, -1, 11, 0},
+/* 19 */ { 7, s_3_19, -1, 12, 0}
+};
+
+static const symbol s_4_0[5] = { 'i', 'c', 'a', 't', 'e' };
+static const symbol s_4_1[5] = { 'a', 't', 'i', 'v', 'e' };
+static const symbol s_4_2[5] = { 'a', 'l', 'i', 'z', 'e' };
+static const symbol s_4_3[5] = { 'i', 'c', 'i', 't', 'i' };
+static const symbol s_4_4[4] = { 'i', 'c', 'a', 'l' };
+static const symbol s_4_5[3] = { 'f', 'u', 'l' };
+static const symbol s_4_6[4] = { 'n', 'e', 's', 's' };
+
+static const struct among a_4[7] =
+{
+/* 0 */ { 5, s_4_0, -1, 2, 0},
+/* 1 */ { 5, s_4_1, -1, 3, 0},
+/* 2 */ { 5, s_4_2, -1, 1, 0},
+/* 3 */ { 5, s_4_3, -1, 2, 0},
+/* 4 */ { 4, s_4_4, -1, 2, 0},
+/* 5 */ { 3, s_4_5, -1, 3, 0},
+/* 6 */ { 4, s_4_6, -1, 3, 0}
+};
+
+static const symbol s_5_0[2] = { 'i', 'c' };
+static const symbol s_5_1[4] = { 'a', 'n', 'c', 'e' };
+static const symbol s_5_2[4] = { 'e', 'n', 'c', 'e' };
+static const symbol s_5_3[4] = { 'a', 'b', 'l', 'e' };
+static const symbol s_5_4[4] = { 'i', 'b', 'l', 'e' };
+static const symbol s_5_5[3] = { 'a', 't', 'e' };
+static const symbol s_5_6[3] = { 'i', 'v', 'e' };
+static const symbol s_5_7[3] = { 'i', 'z', 'e' };
+static const symbol s_5_8[3] = { 'i', 't', 'i' };
+static const symbol s_5_9[2] = { 'a', 'l' };
+static const symbol s_5_10[3] = { 'i', 's', 'm' };
+static const symbol s_5_11[3] = { 'i', 'o', 'n' };
+static const symbol s_5_12[2] = { 'e', 'r' };
+static const symbol s_5_13[3] = { 'o', 'u', 's' };
+static const symbol s_5_14[3] = { 'a', 'n', 't' };
+static const symbol s_5_15[3] = { 'e', 'n', 't' };
+static const symbol s_5_16[4] = { 'm', 'e', 'n', 't' };
+static const symbol s_5_17[5] = { 'e', 'm', 'e', 'n', 't' };
+static const symbol s_5_18[2] = { 'o', 'u' };
+
+static const struct among a_5[19] =
+{
+/* 0 */ { 2, s_5_0, -1, 1, 0},
+/* 1 */ { 4, s_5_1, -1, 1, 0},
+/* 2 */ { 4, s_5_2, -1, 1, 0},
+/* 3 */ { 4, s_5_3, -1, 1, 0},
+/* 4 */ { 4, s_5_4, -1, 1, 0},
+/* 5 */ { 3, s_5_5, -1, 1, 0},
+/* 6 */ { 3, s_5_6, -1, 1, 0},
+/* 7 */ { 3, s_5_7, -1, 1, 0},
+/* 8 */ { 3, s_5_8, -1, 1, 0},
+/* 9 */ { 2, s_5_9, -1, 1, 0},
+/* 10 */ { 3, s_5_10, -1, 1, 0},
+/* 11 */ { 3, s_5_11, -1, 2, 0},
+/* 12 */ { 2, s_5_12, -1, 1, 0},
+/* 13 */ { 3, s_5_13, -1, 1, 0},
+/* 14 */ { 3, s_5_14, -1, 1, 0},
+/* 15 */ { 3, s_5_15, -1, 1, 0},
+/* 16 */ { 4, s_5_16, 15, 1, 0},
+/* 17 */ { 5, s_5_17, 16, 1, 0},
+/* 18 */ { 2, s_5_18, -1, 1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1 };
+
+static const unsigned char g_v_WXY[] = { 1, 17, 65, 208, 1 };
+
+static const symbol s_0[] = { 's', 's' };
+static const symbol s_1[] = { 'i' };
+static const symbol s_2[] = { 'e', 'e' };
+static const symbol s_3[] = { 'e' };
+static const symbol s_4[] = { 'e' };
+static const symbol s_5[] = { 'y' };
+static const symbol s_6[] = { 'Y' };
+static const symbol s_7[] = { 'i' };
+static const symbol s_8[] = { 't', 'i', 'o', 'n' };
+static const symbol s_9[] = { 'e', 'n', 'c', 'e' };
+static const symbol s_10[] = { 'a', 'n', 'c', 'e' };
+static const symbol s_11[] = { 'a', 'b', 'l', 'e' };
+static const symbol s_12[] = { 'e', 'n', 't' };
+static const symbol s_13[] = { 'e' };
+static const symbol s_14[] = { 'i', 'z', 'e' };
+static const symbol s_15[] = { 'a', 't', 'e' };
+static const symbol s_16[] = { 'a', 'l' };
+static const symbol s_17[] = { 'a', 'l' };
+static const symbol s_18[] = { 'f', 'u', 'l' };
+static const symbol s_19[] = { 'o', 'u', 's' };
+static const symbol s_20[] = { 'i', 'v', 'e' };
+static const symbol s_21[] = { 'b', 'l', 'e' };
+static const symbol s_22[] = { 'a', 'l' };
+static const symbol s_23[] = { 'i', 'c' };
+static const symbol s_24[] = { 's' };
+static const symbol s_25[] = { 't' };
+static const symbol s_26[] = { 'e' };
+static const symbol s_27[] = { 'l' };
+static const symbol s_28[] = { 'l' };
+static const symbol s_29[] = { 'y' };
+static const symbol s_30[] = { 'Y' };
+static const symbol s_31[] = { 'y' };
+static const symbol s_32[] = { 'Y' };
+static const symbol s_33[] = { 'Y' };
+static const symbol s_34[] = { 'y' };
+
+static int r_shortv(struct SN_env * z) {
+ if (out_grouping_b_U(z, g_v_WXY, 89, 121, 0)) return 0;
+ if (in_grouping_b_U(z, g_v, 97, 121, 0)) return 0;
+ if (out_grouping_b_U(z, g_v, 97, 121, 0)) return 0;
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_Step_1a(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 25 */
+ if (z->c <= z->lb || z->p[z->c - 1] != 115) return 0;
+ among_var = find_among_b(z, a_0, 4); /* substring, line 25 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 25 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 2, s_0); /* <-, line 26 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_1); /* <-, line 27 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 29 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_1b(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 34 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 100 && z->p[z->c - 1] != 103)) return 0;
+ among_var = find_among_b(z, a_2, 3); /* substring, line 34 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 34 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 35 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 2, s_2); /* <-, line 35 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m_test = z->l - z->c; /* test, line 38 */
+ { /* gopast */ /* grouping v, line 38 */
+ int ret = out_grouping_b_U(z, g_v, 97, 121, 1);
+ if (ret < 0) return 0;
+ z->c -= ret;
+ }
+ z->c = z->l - m_test;
+ }
+ { int ret = slice_del(z); /* delete, line 38 */
+ if (ret < 0) return ret;
+ }
+ { int m_test = z->l - z->c; /* test, line 39 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((68514004 >> (z->p[z->c - 1] & 0x1f)) & 1)) among_var = 3; else
+ among_var = find_among_b(z, a_1, 13); /* substring, line 39 */
+ if (!(among_var)) return 0;
+ z->c = z->l - m_test;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 1, s_3); /* <+, line 41 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ z->ket = z->c; /* [, line 44 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 44 */
+ }
+ z->bra = z->c; /* ], line 44 */
+ { int ret = slice_del(z); /* delete, line 44 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ if (z->c != z->I[0]) return 0; /* atmark, line 45 */
+ { int m_test = z->l - z->c; /* test, line 45 */
+ { int ret = r_shortv(z);
+ if (ret == 0) return 0; /* call shortv, line 45 */
+ if (ret < 0) return ret;
+ }
+ z->c = z->l - m_test;
+ }
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 1, s_4); /* <+, line 45 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_1c(struct SN_env * z) {
+ z->ket = z->c; /* [, line 52 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 52 */
+ if (!(eq_s_b(z, 1, s_5))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_6))) return 0;
+ }
+lab0:
+ z->bra = z->c; /* ], line 52 */
+ { /* gopast */ /* grouping v, line 53 */
+ int ret = out_grouping_b_U(z, g_v, 97, 121, 1);
+ if (ret < 0) return 0;
+ z->c -= ret;
+ }
+ { int ret = slice_from_s(z, 1, s_7); /* <-, line 54 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_Step_2(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 58 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((815616 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_3, 20); /* substring, line 58 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 58 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 58 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 4, s_8); /* <-, line 59 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 4, s_9); /* <-, line 60 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 4, s_10); /* <-, line 61 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 4, s_11); /* <-, line 62 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 3, s_12); /* <-, line 63 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 1, s_13); /* <-, line 64 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_from_s(z, 3, s_14); /* <-, line 66 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 8:
+ { int ret = slice_from_s(z, 3, s_15); /* <-, line 68 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 9:
+ { int ret = slice_from_s(z, 2, s_16); /* <-, line 69 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 10:
+ { int ret = slice_from_s(z, 2, s_17); /* <-, line 71 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 11:
+ { int ret = slice_from_s(z, 3, s_18); /* <-, line 72 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 12:
+ { int ret = slice_from_s(z, 3, s_19); /* <-, line 74 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 13:
+ { int ret = slice_from_s(z, 3, s_20); /* <-, line 76 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 14:
+ { int ret = slice_from_s(z, 3, s_21); /* <-, line 77 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_3(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 82 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((528928 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_4, 7); /* substring, line 82 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 82 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 82 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 2, s_22); /* <-, line 83 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 2, s_23); /* <-, line 85 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 87 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_4(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 92 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((3961384 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_5, 19); /* substring, line 92 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 92 */
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 92 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 95 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 96 */
+ if (!(eq_s_b(z, 1, s_24))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_25))) return 0;
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 96 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_Step_5a(struct SN_env * z) {
+ z->ket = z->c; /* [, line 101 */
+ if (!(eq_s_b(z, 1, s_26))) return 0;
+ z->bra = z->c; /* ], line 101 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 102 */
+ { int ret = r_R2(z);
+ if (ret == 0) goto lab1; /* call R2, line 102 */
+ if (ret < 0) return ret;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 102 */
+ if (ret < 0) return ret;
+ }
+ { int m2 = z->l - z->c; (void)m2; /* not, line 102 */
+ { int ret = r_shortv(z);
+ if (ret == 0) goto lab2; /* call shortv, line 102 */
+ if (ret < 0) return ret;
+ }
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ }
+lab0:
+ { int ret = slice_del(z); /* delete, line 103 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_Step_5b(struct SN_env * z) {
+ z->ket = z->c; /* [, line 107 */
+ if (!(eq_s_b(z, 1, s_27))) return 0;
+ z->bra = z->c; /* ], line 107 */
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 108 */
+ if (ret < 0) return ret;
+ }
+ if (!(eq_s_b(z, 1, s_28))) return 0;
+ { int ret = slice_del(z); /* delete, line 109 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+extern int porter_UTF_8_stem(struct SN_env * z) {
+ z->B[0] = 0; /* unset Y_found, line 115 */
+ { int c1 = z->c; /* do, line 116 */
+ z->bra = z->c; /* [, line 116 */
+ if (!(eq_s(z, 1, s_29))) goto lab0;
+ z->ket = z->c; /* ], line 116 */
+ { int ret = slice_from_s(z, 1, s_30); /* <-, line 116 */
+ if (ret < 0) return ret;
+ }
+ z->B[0] = 1; /* set Y_found, line 116 */
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 117 */
+ while(1) { /* repeat, line 117 */
+ int c3 = z->c;
+ while(1) { /* goto, line 117 */
+ int c4 = z->c;
+ if (in_grouping_U(z, g_v, 97, 121, 0)) goto lab3;
+ z->bra = z->c; /* [, line 117 */
+ if (!(eq_s(z, 1, s_31))) goto lab3;
+ z->ket = z->c; /* ], line 117 */
+ z->c = c4;
+ break;
+ lab3:
+ z->c = c4;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab2;
+ z->c = ret; /* goto, line 117 */
+ }
+ }
+ { int ret = slice_from_s(z, 1, s_32); /* <-, line 117 */
+ if (ret < 0) return ret;
+ }
+ z->B[0] = 1; /* set Y_found, line 117 */
+ continue;
+ lab2:
+ z->c = c3;
+ break;
+ }
+ z->c = c2;
+ }
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ { int c5 = z->c; /* do, line 121 */
+ { /* gopast */ /* grouping v, line 122 */
+ int ret = out_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 122 */
+ int ret = in_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 122 */
+ { /* gopast */ /* grouping v, line 123 */
+ int ret = out_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 123 */
+ int ret = in_grouping_U(z, g_v, 97, 121, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p2, line 123 */
+ lab4:
+ z->c = c5;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 126 */
+
+ { int m6 = z->l - z->c; (void)m6; /* do, line 127 */
+ { int ret = r_Step_1a(z);
+ if (ret == 0) goto lab5; /* call Step_1a, line 127 */
+ if (ret < 0) return ret;
+ }
+ lab5:
+ z->c = z->l - m6;
+ }
+ { int m7 = z->l - z->c; (void)m7; /* do, line 128 */
+ { int ret = r_Step_1b(z);
+ if (ret == 0) goto lab6; /* call Step_1b, line 128 */
+ if (ret < 0) return ret;
+ }
+ lab6:
+ z->c = z->l - m7;
+ }
+ { int m8 = z->l - z->c; (void)m8; /* do, line 129 */
+ { int ret = r_Step_1c(z);
+ if (ret == 0) goto lab7; /* call Step_1c, line 129 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = z->l - m8;
+ }
+ { int m9 = z->l - z->c; (void)m9; /* do, line 130 */
+ { int ret = r_Step_2(z);
+ if (ret == 0) goto lab8; /* call Step_2, line 130 */
+ if (ret < 0) return ret;
+ }
+ lab8:
+ z->c = z->l - m9;
+ }
+ { int m10 = z->l - z->c; (void)m10; /* do, line 131 */
+ { int ret = r_Step_3(z);
+ if (ret == 0) goto lab9; /* call Step_3, line 131 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = z->l - m10;
+ }
+ { int m11 = z->l - z->c; (void)m11; /* do, line 132 */
+ { int ret = r_Step_4(z);
+ if (ret == 0) goto lab10; /* call Step_4, line 132 */
+ if (ret < 0) return ret;
+ }
+ lab10:
+ z->c = z->l - m11;
+ }
+ { int m12 = z->l - z->c; (void)m12; /* do, line 133 */
+ { int ret = r_Step_5a(z);
+ if (ret == 0) goto lab11; /* call Step_5a, line 133 */
+ if (ret < 0) return ret;
+ }
+ lab11:
+ z->c = z->l - m12;
+ }
+ { int m13 = z->l - z->c; (void)m13; /* do, line 134 */
+ { int ret = r_Step_5b(z);
+ if (ret == 0) goto lab12; /* call Step_5b, line 134 */
+ if (ret < 0) return ret;
+ }
+ lab12:
+ z->c = z->l - m13;
+ }
+ z->c = z->lb;
+ { int c14 = z->c; /* do, line 137 */
+ if (!(z->B[0])) goto lab13; /* Boolean test Y_found, line 137 */
+ while(1) { /* repeat, line 137 */
+ int c15 = z->c;
+ while(1) { /* goto, line 137 */
+ int c16 = z->c;
+ z->bra = z->c; /* [, line 137 */
+ if (!(eq_s(z, 1, s_33))) goto lab15;
+ z->ket = z->c; /* ], line 137 */
+ z->c = c16;
+ break;
+ lab15:
+ z->c = c16;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab14;
+ z->c = ret; /* goto, line 137 */
+ }
+ }
+ { int ret = slice_from_s(z, 1, s_34); /* <-, line 137 */
+ if (ret < 0) return ret;
+ }
+ continue;
+ lab14:
+ z->c = c15;
+ break;
+ }
+ lab13:
+ z->c = c14;
+ }
+ return 1;
+}
+
+extern struct SN_env * porter_UTF_8_create_env(void) { return SN_create_env(0, 2, 1); }
+
+extern void porter_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_porter.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_porter.h
new file mode 100644
index 0000000..82d469a
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_porter.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * porter_UTF_8_create_env(void);
+extern void porter_UTF_8_close_env(struct SN_env * z);
+
+extern int porter_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_portuguese.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_portuguese.c
new file mode 100644
index 0000000..282dbb7
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_portuguese.c
@@ -0,0 +1,1023 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int portuguese_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_residual_form(struct SN_env * z);
+static int r_residual_suffix(struct SN_env * z);
+static int r_verb_suffix(struct SN_env * z);
+static int r_standard_suffix(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_RV(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * portuguese_UTF_8_create_env(void);
+extern void portuguese_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_1[2] = { 0xC3, 0xA3 };
+static const symbol s_0_2[2] = { 0xC3, 0xB5 };
+
+static const struct among a_0[3] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 2, s_0_1, 0, 1, 0},
+/* 2 */ { 2, s_0_2, 0, 2, 0}
+};
+
+static const symbol s_1_1[2] = { 'a', '~' };
+static const symbol s_1_2[2] = { 'o', '~' };
+
+static const struct among a_1[3] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 2, s_1_1, 0, 1, 0},
+/* 2 */ { 2, s_1_2, 0, 2, 0}
+};
+
+static const symbol s_2_0[2] = { 'i', 'c' };
+static const symbol s_2_1[2] = { 'a', 'd' };
+static const symbol s_2_2[2] = { 'o', 's' };
+static const symbol s_2_3[2] = { 'i', 'v' };
+
+static const struct among a_2[4] =
+{
+/* 0 */ { 2, s_2_0, -1, -1, 0},
+/* 1 */ { 2, s_2_1, -1, -1, 0},
+/* 2 */ { 2, s_2_2, -1, -1, 0},
+/* 3 */ { 2, s_2_3, -1, 1, 0}
+};
+
+static const symbol s_3_0[4] = { 'a', 'n', 't', 'e' };
+static const symbol s_3_1[4] = { 'a', 'v', 'e', 'l' };
+static const symbol s_3_2[5] = { 0xC3, 0xAD, 'v', 'e', 'l' };
+
+static const struct among a_3[3] =
+{
+/* 0 */ { 4, s_3_0, -1, 1, 0},
+/* 1 */ { 4, s_3_1, -1, 1, 0},
+/* 2 */ { 5, s_3_2, -1, 1, 0}
+};
+
+static const symbol s_4_0[2] = { 'i', 'c' };
+static const symbol s_4_1[4] = { 'a', 'b', 'i', 'l' };
+static const symbol s_4_2[2] = { 'i', 'v' };
+
+static const struct among a_4[3] =
+{
+/* 0 */ { 2, s_4_0, -1, 1, 0},
+/* 1 */ { 4, s_4_1, -1, 1, 0},
+/* 2 */ { 2, s_4_2, -1, 1, 0}
+};
+
+static const symbol s_5_0[3] = { 'i', 'c', 'a' };
+static const symbol s_5_1[6] = { 0xC3, 0xA2, 'n', 'c', 'i', 'a' };
+static const symbol s_5_2[6] = { 0xC3, 0xAA, 'n', 'c', 'i', 'a' };
+static const symbol s_5_3[3] = { 'i', 'r', 'a' };
+static const symbol s_5_4[5] = { 'a', 'd', 'o', 'r', 'a' };
+static const symbol s_5_5[3] = { 'o', 's', 'a' };
+static const symbol s_5_6[4] = { 'i', 's', 't', 'a' };
+static const symbol s_5_7[3] = { 'i', 'v', 'a' };
+static const symbol s_5_8[3] = { 'e', 'z', 'a' };
+static const symbol s_5_9[6] = { 'l', 'o', 'g', 0xC3, 0xAD, 'a' };
+static const symbol s_5_10[5] = { 'i', 'd', 'a', 'd', 'e' };
+static const symbol s_5_11[4] = { 'a', 'n', 't', 'e' };
+static const symbol s_5_12[5] = { 'm', 'e', 'n', 't', 'e' };
+static const symbol s_5_13[6] = { 'a', 'm', 'e', 'n', 't', 'e' };
+static const symbol s_5_14[5] = { 0xC3, 0xA1, 'v', 'e', 'l' };
+static const symbol s_5_15[5] = { 0xC3, 0xAD, 'v', 'e', 'l' };
+static const symbol s_5_16[6] = { 'u', 'c', 'i', 0xC3, 0xB3, 'n' };
+static const symbol s_5_17[3] = { 'i', 'c', 'o' };
+static const symbol s_5_18[4] = { 'i', 's', 'm', 'o' };
+static const symbol s_5_19[3] = { 'o', 's', 'o' };
+static const symbol s_5_20[6] = { 'a', 'm', 'e', 'n', 't', 'o' };
+static const symbol s_5_21[6] = { 'i', 'm', 'e', 'n', 't', 'o' };
+static const symbol s_5_22[3] = { 'i', 'v', 'o' };
+static const symbol s_5_23[6] = { 'a', 0xC3, 0xA7, 'a', '~', 'o' };
+static const symbol s_5_24[4] = { 'a', 'd', 'o', 'r' };
+static const symbol s_5_25[4] = { 'i', 'c', 'a', 's' };
+static const symbol s_5_26[7] = { 0xC3, 0xAA, 'n', 'c', 'i', 'a', 's' };
+static const symbol s_5_27[4] = { 'i', 'r', 'a', 's' };
+static const symbol s_5_28[6] = { 'a', 'd', 'o', 'r', 'a', 's' };
+static const symbol s_5_29[4] = { 'o', 's', 'a', 's' };
+static const symbol s_5_30[5] = { 'i', 's', 't', 'a', 's' };
+static const symbol s_5_31[4] = { 'i', 'v', 'a', 's' };
+static const symbol s_5_32[4] = { 'e', 'z', 'a', 's' };
+static const symbol s_5_33[7] = { 'l', 'o', 'g', 0xC3, 0xAD, 'a', 's' };
+static const symbol s_5_34[6] = { 'i', 'd', 'a', 'd', 'e', 's' };
+static const symbol s_5_35[7] = { 'u', 'c', 'i', 'o', 'n', 'e', 's' };
+static const symbol s_5_36[6] = { 'a', 'd', 'o', 'r', 'e', 's' };
+static const symbol s_5_37[5] = { 'a', 'n', 't', 'e', 's' };
+static const symbol s_5_38[7] = { 'a', 0xC3, 0xA7, 'o', '~', 'e', 's' };
+static const symbol s_5_39[4] = { 'i', 'c', 'o', 's' };
+static const symbol s_5_40[5] = { 'i', 's', 'm', 'o', 's' };
+static const symbol s_5_41[4] = { 'o', 's', 'o', 's' };
+static const symbol s_5_42[7] = { 'a', 'm', 'e', 'n', 't', 'o', 's' };
+static const symbol s_5_43[7] = { 'i', 'm', 'e', 'n', 't', 'o', 's' };
+static const symbol s_5_44[4] = { 'i', 'v', 'o', 's' };
+
+static const struct among a_5[45] =
+{
+/* 0 */ { 3, s_5_0, -1, 1, 0},
+/* 1 */ { 6, s_5_1, -1, 1, 0},
+/* 2 */ { 6, s_5_2, -1, 4, 0},
+/* 3 */ { 3, s_5_3, -1, 9, 0},
+/* 4 */ { 5, s_5_4, -1, 1, 0},
+/* 5 */ { 3, s_5_5, -1, 1, 0},
+/* 6 */ { 4, s_5_6, -1, 1, 0},
+/* 7 */ { 3, s_5_7, -1, 8, 0},
+/* 8 */ { 3, s_5_8, -1, 1, 0},
+/* 9 */ { 6, s_5_9, -1, 2, 0},
+/* 10 */ { 5, s_5_10, -1, 7, 0},
+/* 11 */ { 4, s_5_11, -1, 1, 0},
+/* 12 */ { 5, s_5_12, -1, 6, 0},
+/* 13 */ { 6, s_5_13, 12, 5, 0},
+/* 14 */ { 5, s_5_14, -1, 1, 0},
+/* 15 */ { 5, s_5_15, -1, 1, 0},
+/* 16 */ { 6, s_5_16, -1, 3, 0},
+/* 17 */ { 3, s_5_17, -1, 1, 0},
+/* 18 */ { 4, s_5_18, -1, 1, 0},
+/* 19 */ { 3, s_5_19, -1, 1, 0},
+/* 20 */ { 6, s_5_20, -1, 1, 0},
+/* 21 */ { 6, s_5_21, -1, 1, 0},
+/* 22 */ { 3, s_5_22, -1, 8, 0},
+/* 23 */ { 6, s_5_23, -1, 1, 0},
+/* 24 */ { 4, s_5_24, -1, 1, 0},
+/* 25 */ { 4, s_5_25, -1, 1, 0},
+/* 26 */ { 7, s_5_26, -1, 4, 0},
+/* 27 */ { 4, s_5_27, -1, 9, 0},
+/* 28 */ { 6, s_5_28, -1, 1, 0},
+/* 29 */ { 4, s_5_29, -1, 1, 0},
+/* 30 */ { 5, s_5_30, -1, 1, 0},
+/* 31 */ { 4, s_5_31, -1, 8, 0},
+/* 32 */ { 4, s_5_32, -1, 1, 0},
+/* 33 */ { 7, s_5_33, -1, 2, 0},
+/* 34 */ { 6, s_5_34, -1, 7, 0},
+/* 35 */ { 7, s_5_35, -1, 3, 0},
+/* 36 */ { 6, s_5_36, -1, 1, 0},
+/* 37 */ { 5, s_5_37, -1, 1, 0},
+/* 38 */ { 7, s_5_38, -1, 1, 0},
+/* 39 */ { 4, s_5_39, -1, 1, 0},
+/* 40 */ { 5, s_5_40, -1, 1, 0},
+/* 41 */ { 4, s_5_41, -1, 1, 0},
+/* 42 */ { 7, s_5_42, -1, 1, 0},
+/* 43 */ { 7, s_5_43, -1, 1, 0},
+/* 44 */ { 4, s_5_44, -1, 8, 0}
+};
+
+static const symbol s_6_0[3] = { 'a', 'd', 'a' };
+static const symbol s_6_1[3] = { 'i', 'd', 'a' };
+static const symbol s_6_2[2] = { 'i', 'a' };
+static const symbol s_6_3[4] = { 'a', 'r', 'i', 'a' };
+static const symbol s_6_4[4] = { 'e', 'r', 'i', 'a' };
+static const symbol s_6_5[4] = { 'i', 'r', 'i', 'a' };
+static const symbol s_6_6[3] = { 'a', 'r', 'a' };
+static const symbol s_6_7[3] = { 'e', 'r', 'a' };
+static const symbol s_6_8[3] = { 'i', 'r', 'a' };
+static const symbol s_6_9[3] = { 'a', 'v', 'a' };
+static const symbol s_6_10[4] = { 'a', 's', 's', 'e' };
+static const symbol s_6_11[4] = { 'e', 's', 's', 'e' };
+static const symbol s_6_12[4] = { 'i', 's', 's', 'e' };
+static const symbol s_6_13[4] = { 'a', 's', 't', 'e' };
+static const symbol s_6_14[4] = { 'e', 's', 't', 'e' };
+static const symbol s_6_15[4] = { 'i', 's', 't', 'e' };
+static const symbol s_6_16[2] = { 'e', 'i' };
+static const symbol s_6_17[4] = { 'a', 'r', 'e', 'i' };
+static const symbol s_6_18[4] = { 'e', 'r', 'e', 'i' };
+static const symbol s_6_19[4] = { 'i', 'r', 'e', 'i' };
+static const symbol s_6_20[2] = { 'a', 'm' };
+static const symbol s_6_21[3] = { 'i', 'a', 'm' };
+static const symbol s_6_22[5] = { 'a', 'r', 'i', 'a', 'm' };
+static const symbol s_6_23[5] = { 'e', 'r', 'i', 'a', 'm' };
+static const symbol s_6_24[5] = { 'i', 'r', 'i', 'a', 'm' };
+static const symbol s_6_25[4] = { 'a', 'r', 'a', 'm' };
+static const symbol s_6_26[4] = { 'e', 'r', 'a', 'm' };
+static const symbol s_6_27[4] = { 'i', 'r', 'a', 'm' };
+static const symbol s_6_28[4] = { 'a', 'v', 'a', 'm' };
+static const symbol s_6_29[2] = { 'e', 'm' };
+static const symbol s_6_30[4] = { 'a', 'r', 'e', 'm' };
+static const symbol s_6_31[4] = { 'e', 'r', 'e', 'm' };
+static const symbol s_6_32[4] = { 'i', 'r', 'e', 'm' };
+static const symbol s_6_33[5] = { 'a', 's', 's', 'e', 'm' };
+static const symbol s_6_34[5] = { 'e', 's', 's', 'e', 'm' };
+static const symbol s_6_35[5] = { 'i', 's', 's', 'e', 'm' };
+static const symbol s_6_36[3] = { 'a', 'd', 'o' };
+static const symbol s_6_37[3] = { 'i', 'd', 'o' };
+static const symbol s_6_38[4] = { 'a', 'n', 'd', 'o' };
+static const symbol s_6_39[4] = { 'e', 'n', 'd', 'o' };
+static const symbol s_6_40[4] = { 'i', 'n', 'd', 'o' };
+static const symbol s_6_41[5] = { 'a', 'r', 'a', '~', 'o' };
+static const symbol s_6_42[5] = { 'e', 'r', 'a', '~', 'o' };
+static const symbol s_6_43[5] = { 'i', 'r', 'a', '~', 'o' };
+static const symbol s_6_44[2] = { 'a', 'r' };
+static const symbol s_6_45[2] = { 'e', 'r' };
+static const symbol s_6_46[2] = { 'i', 'r' };
+static const symbol s_6_47[2] = { 'a', 's' };
+static const symbol s_6_48[4] = { 'a', 'd', 'a', 's' };
+static const symbol s_6_49[4] = { 'i', 'd', 'a', 's' };
+static const symbol s_6_50[3] = { 'i', 'a', 's' };
+static const symbol s_6_51[5] = { 'a', 'r', 'i', 'a', 's' };
+static const symbol s_6_52[5] = { 'e', 'r', 'i', 'a', 's' };
+static const symbol s_6_53[5] = { 'i', 'r', 'i', 'a', 's' };
+static const symbol s_6_54[4] = { 'a', 'r', 'a', 's' };
+static const symbol s_6_55[4] = { 'e', 'r', 'a', 's' };
+static const symbol s_6_56[4] = { 'i', 'r', 'a', 's' };
+static const symbol s_6_57[4] = { 'a', 'v', 'a', 's' };
+static const symbol s_6_58[2] = { 'e', 's' };
+static const symbol s_6_59[5] = { 'a', 'r', 'd', 'e', 's' };
+static const symbol s_6_60[5] = { 'e', 'r', 'd', 'e', 's' };
+static const symbol s_6_61[5] = { 'i', 'r', 'd', 'e', 's' };
+static const symbol s_6_62[4] = { 'a', 'r', 'e', 's' };
+static const symbol s_6_63[4] = { 'e', 'r', 'e', 's' };
+static const symbol s_6_64[4] = { 'i', 'r', 'e', 's' };
+static const symbol s_6_65[5] = { 'a', 's', 's', 'e', 's' };
+static const symbol s_6_66[5] = { 'e', 's', 's', 'e', 's' };
+static const symbol s_6_67[5] = { 'i', 's', 's', 'e', 's' };
+static const symbol s_6_68[5] = { 'a', 's', 't', 'e', 's' };
+static const symbol s_6_69[5] = { 'e', 's', 't', 'e', 's' };
+static const symbol s_6_70[5] = { 'i', 's', 't', 'e', 's' };
+static const symbol s_6_71[2] = { 'i', 's' };
+static const symbol s_6_72[3] = { 'a', 'i', 's' };
+static const symbol s_6_73[3] = { 'e', 'i', 's' };
+static const symbol s_6_74[5] = { 'a', 'r', 'e', 'i', 's' };
+static const symbol s_6_75[5] = { 'e', 'r', 'e', 'i', 's' };
+static const symbol s_6_76[5] = { 'i', 'r', 'e', 'i', 's' };
+static const symbol s_6_77[6] = { 0xC3, 0xA1, 'r', 'e', 'i', 's' };
+static const symbol s_6_78[6] = { 0xC3, 0xA9, 'r', 'e', 'i', 's' };
+static const symbol s_6_79[6] = { 0xC3, 0xAD, 'r', 'e', 'i', 's' };
+static const symbol s_6_80[7] = { 0xC3, 0xA1, 's', 's', 'e', 'i', 's' };
+static const symbol s_6_81[7] = { 0xC3, 0xA9, 's', 's', 'e', 'i', 's' };
+static const symbol s_6_82[7] = { 0xC3, 0xAD, 's', 's', 'e', 'i', 's' };
+static const symbol s_6_83[6] = { 0xC3, 0xA1, 'v', 'e', 'i', 's' };
+static const symbol s_6_84[5] = { 0xC3, 0xAD, 'e', 'i', 's' };
+static const symbol s_6_85[7] = { 'a', 'r', 0xC3, 0xAD, 'e', 'i', 's' };
+static const symbol s_6_86[7] = { 'e', 'r', 0xC3, 0xAD, 'e', 'i', 's' };
+static const symbol s_6_87[7] = { 'i', 'r', 0xC3, 0xAD, 'e', 'i', 's' };
+static const symbol s_6_88[4] = { 'a', 'd', 'o', 's' };
+static const symbol s_6_89[4] = { 'i', 'd', 'o', 's' };
+static const symbol s_6_90[4] = { 'a', 'm', 'o', 's' };
+static const symbol s_6_91[7] = { 0xC3, 0xA1, 'r', 'a', 'm', 'o', 's' };
+static const symbol s_6_92[7] = { 0xC3, 0xA9, 'r', 'a', 'm', 'o', 's' };
+static const symbol s_6_93[7] = { 0xC3, 0xAD, 'r', 'a', 'm', 'o', 's' };
+static const symbol s_6_94[7] = { 0xC3, 0xA1, 'v', 'a', 'm', 'o', 's' };
+static const symbol s_6_95[6] = { 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_6_96[8] = { 'a', 'r', 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_6_97[8] = { 'e', 'r', 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_6_98[8] = { 'i', 'r', 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_6_99[4] = { 'e', 'm', 'o', 's' };
+static const symbol s_6_100[6] = { 'a', 'r', 'e', 'm', 'o', 's' };
+static const symbol s_6_101[6] = { 'e', 'r', 'e', 'm', 'o', 's' };
+static const symbol s_6_102[6] = { 'i', 'r', 'e', 'm', 'o', 's' };
+static const symbol s_6_103[8] = { 0xC3, 0xA1, 's', 's', 'e', 'm', 'o', 's' };
+static const symbol s_6_104[8] = { 0xC3, 0xAA, 's', 's', 'e', 'm', 'o', 's' };
+static const symbol s_6_105[8] = { 0xC3, 0xAD, 's', 's', 'e', 'm', 'o', 's' };
+static const symbol s_6_106[4] = { 'i', 'm', 'o', 's' };
+static const symbol s_6_107[5] = { 'a', 'r', 'm', 'o', 's' };
+static const symbol s_6_108[5] = { 'e', 'r', 'm', 'o', 's' };
+static const symbol s_6_109[5] = { 'i', 'r', 'm', 'o', 's' };
+static const symbol s_6_110[5] = { 0xC3, 0xA1, 'm', 'o', 's' };
+static const symbol s_6_111[5] = { 'a', 'r', 0xC3, 0xA1, 's' };
+static const symbol s_6_112[5] = { 'e', 'r', 0xC3, 0xA1, 's' };
+static const symbol s_6_113[5] = { 'i', 'r', 0xC3, 0xA1, 's' };
+static const symbol s_6_114[2] = { 'e', 'u' };
+static const symbol s_6_115[2] = { 'i', 'u' };
+static const symbol s_6_116[2] = { 'o', 'u' };
+static const symbol s_6_117[4] = { 'a', 'r', 0xC3, 0xA1 };
+static const symbol s_6_118[4] = { 'e', 'r', 0xC3, 0xA1 };
+static const symbol s_6_119[4] = { 'i', 'r', 0xC3, 0xA1 };
+
+static const struct among a_6[120] =
+{
+/* 0 */ { 3, s_6_0, -1, 1, 0},
+/* 1 */ { 3, s_6_1, -1, 1, 0},
+/* 2 */ { 2, s_6_2, -1, 1, 0},
+/* 3 */ { 4, s_6_3, 2, 1, 0},
+/* 4 */ { 4, s_6_4, 2, 1, 0},
+/* 5 */ { 4, s_6_5, 2, 1, 0},
+/* 6 */ { 3, s_6_6, -1, 1, 0},
+/* 7 */ { 3, s_6_7, -1, 1, 0},
+/* 8 */ { 3, s_6_8, -1, 1, 0},
+/* 9 */ { 3, s_6_9, -1, 1, 0},
+/* 10 */ { 4, s_6_10, -1, 1, 0},
+/* 11 */ { 4, s_6_11, -1, 1, 0},
+/* 12 */ { 4, s_6_12, -1, 1, 0},
+/* 13 */ { 4, s_6_13, -1, 1, 0},
+/* 14 */ { 4, s_6_14, -1, 1, 0},
+/* 15 */ { 4, s_6_15, -1, 1, 0},
+/* 16 */ { 2, s_6_16, -1, 1, 0},
+/* 17 */ { 4, s_6_17, 16, 1, 0},
+/* 18 */ { 4, s_6_18, 16, 1, 0},
+/* 19 */ { 4, s_6_19, 16, 1, 0},
+/* 20 */ { 2, s_6_20, -1, 1, 0},
+/* 21 */ { 3, s_6_21, 20, 1, 0},
+/* 22 */ { 5, s_6_22, 21, 1, 0},
+/* 23 */ { 5, s_6_23, 21, 1, 0},
+/* 24 */ { 5, s_6_24, 21, 1, 0},
+/* 25 */ { 4, s_6_25, 20, 1, 0},
+/* 26 */ { 4, s_6_26, 20, 1, 0},
+/* 27 */ { 4, s_6_27, 20, 1, 0},
+/* 28 */ { 4, s_6_28, 20, 1, 0},
+/* 29 */ { 2, s_6_29, -1, 1, 0},
+/* 30 */ { 4, s_6_30, 29, 1, 0},
+/* 31 */ { 4, s_6_31, 29, 1, 0},
+/* 32 */ { 4, s_6_32, 29, 1, 0},
+/* 33 */ { 5, s_6_33, 29, 1, 0},
+/* 34 */ { 5, s_6_34, 29, 1, 0},
+/* 35 */ { 5, s_6_35, 29, 1, 0},
+/* 36 */ { 3, s_6_36, -1, 1, 0},
+/* 37 */ { 3, s_6_37, -1, 1, 0},
+/* 38 */ { 4, s_6_38, -1, 1, 0},
+/* 39 */ { 4, s_6_39, -1, 1, 0},
+/* 40 */ { 4, s_6_40, -1, 1, 0},
+/* 41 */ { 5, s_6_41, -1, 1, 0},
+/* 42 */ { 5, s_6_42, -1, 1, 0},
+/* 43 */ { 5, s_6_43, -1, 1, 0},
+/* 44 */ { 2, s_6_44, -1, 1, 0},
+/* 45 */ { 2, s_6_45, -1, 1, 0},
+/* 46 */ { 2, s_6_46, -1, 1, 0},
+/* 47 */ { 2, s_6_47, -1, 1, 0},
+/* 48 */ { 4, s_6_48, 47, 1, 0},
+/* 49 */ { 4, s_6_49, 47, 1, 0},
+/* 50 */ { 3, s_6_50, 47, 1, 0},
+/* 51 */ { 5, s_6_51, 50, 1, 0},
+/* 52 */ { 5, s_6_52, 50, 1, 0},
+/* 53 */ { 5, s_6_53, 50, 1, 0},
+/* 54 */ { 4, s_6_54, 47, 1, 0},
+/* 55 */ { 4, s_6_55, 47, 1, 0},
+/* 56 */ { 4, s_6_56, 47, 1, 0},
+/* 57 */ { 4, s_6_57, 47, 1, 0},
+/* 58 */ { 2, s_6_58, -1, 1, 0},
+/* 59 */ { 5, s_6_59, 58, 1, 0},
+/* 60 */ { 5, s_6_60, 58, 1, 0},
+/* 61 */ { 5, s_6_61, 58, 1, 0},
+/* 62 */ { 4, s_6_62, 58, 1, 0},
+/* 63 */ { 4, s_6_63, 58, 1, 0},
+/* 64 */ { 4, s_6_64, 58, 1, 0},
+/* 65 */ { 5, s_6_65, 58, 1, 0},
+/* 66 */ { 5, s_6_66, 58, 1, 0},
+/* 67 */ { 5, s_6_67, 58, 1, 0},
+/* 68 */ { 5, s_6_68, 58, 1, 0},
+/* 69 */ { 5, s_6_69, 58, 1, 0},
+/* 70 */ { 5, s_6_70, 58, 1, 0},
+/* 71 */ { 2, s_6_71, -1, 1, 0},
+/* 72 */ { 3, s_6_72, 71, 1, 0},
+/* 73 */ { 3, s_6_73, 71, 1, 0},
+/* 74 */ { 5, s_6_74, 73, 1, 0},
+/* 75 */ { 5, s_6_75, 73, 1, 0},
+/* 76 */ { 5, s_6_76, 73, 1, 0},
+/* 77 */ { 6, s_6_77, 73, 1, 0},
+/* 78 */ { 6, s_6_78, 73, 1, 0},
+/* 79 */ { 6, s_6_79, 73, 1, 0},
+/* 80 */ { 7, s_6_80, 73, 1, 0},
+/* 81 */ { 7, s_6_81, 73, 1, 0},
+/* 82 */ { 7, s_6_82, 73, 1, 0},
+/* 83 */ { 6, s_6_83, 73, 1, 0},
+/* 84 */ { 5, s_6_84, 73, 1, 0},
+/* 85 */ { 7, s_6_85, 84, 1, 0},
+/* 86 */ { 7, s_6_86, 84, 1, 0},
+/* 87 */ { 7, s_6_87, 84, 1, 0},
+/* 88 */ { 4, s_6_88, -1, 1, 0},
+/* 89 */ { 4, s_6_89, -1, 1, 0},
+/* 90 */ { 4, s_6_90, -1, 1, 0},
+/* 91 */ { 7, s_6_91, 90, 1, 0},
+/* 92 */ { 7, s_6_92, 90, 1, 0},
+/* 93 */ { 7, s_6_93, 90, 1, 0},
+/* 94 */ { 7, s_6_94, 90, 1, 0},
+/* 95 */ { 6, s_6_95, 90, 1, 0},
+/* 96 */ { 8, s_6_96, 95, 1, 0},
+/* 97 */ { 8, s_6_97, 95, 1, 0},
+/* 98 */ { 8, s_6_98, 95, 1, 0},
+/* 99 */ { 4, s_6_99, -1, 1, 0},
+/*100 */ { 6, s_6_100, 99, 1, 0},
+/*101 */ { 6, s_6_101, 99, 1, 0},
+/*102 */ { 6, s_6_102, 99, 1, 0},
+/*103 */ { 8, s_6_103, 99, 1, 0},
+/*104 */ { 8, s_6_104, 99, 1, 0},
+/*105 */ { 8, s_6_105, 99, 1, 0},
+/*106 */ { 4, s_6_106, -1, 1, 0},
+/*107 */ { 5, s_6_107, -1, 1, 0},
+/*108 */ { 5, s_6_108, -1, 1, 0},
+/*109 */ { 5, s_6_109, -1, 1, 0},
+/*110 */ { 5, s_6_110, -1, 1, 0},
+/*111 */ { 5, s_6_111, -1, 1, 0},
+/*112 */ { 5, s_6_112, -1, 1, 0},
+/*113 */ { 5, s_6_113, -1, 1, 0},
+/*114 */ { 2, s_6_114, -1, 1, 0},
+/*115 */ { 2, s_6_115, -1, 1, 0},
+/*116 */ { 2, s_6_116, -1, 1, 0},
+/*117 */ { 4, s_6_117, -1, 1, 0},
+/*118 */ { 4, s_6_118, -1, 1, 0},
+/*119 */ { 4, s_6_119, -1, 1, 0}
+};
+
+static const symbol s_7_0[1] = { 'a' };
+static const symbol s_7_1[1] = { 'i' };
+static const symbol s_7_2[1] = { 'o' };
+static const symbol s_7_3[2] = { 'o', 's' };
+static const symbol s_7_4[2] = { 0xC3, 0xA1 };
+static const symbol s_7_5[2] = { 0xC3, 0xAD };
+static const symbol s_7_6[2] = { 0xC3, 0xB3 };
+
+static const struct among a_7[7] =
+{
+/* 0 */ { 1, s_7_0, -1, 1, 0},
+/* 1 */ { 1, s_7_1, -1, 1, 0},
+/* 2 */ { 1, s_7_2, -1, 1, 0},
+/* 3 */ { 2, s_7_3, -1, 1, 0},
+/* 4 */ { 2, s_7_4, -1, 1, 0},
+/* 5 */ { 2, s_7_5, -1, 1, 0},
+/* 6 */ { 2, s_7_6, -1, 1, 0}
+};
+
+static const symbol s_8_0[1] = { 'e' };
+static const symbol s_8_1[2] = { 0xC3, 0xA7 };
+static const symbol s_8_2[2] = { 0xC3, 0xA9 };
+static const symbol s_8_3[2] = { 0xC3, 0xAA };
+
+static const struct among a_8[4] =
+{
+/* 0 */ { 1, s_8_0, -1, 1, 0},
+/* 1 */ { 2, s_8_1, -1, 2, 0},
+/* 2 */ { 2, s_8_2, -1, 1, 0},
+/* 3 */ { 2, s_8_3, -1, 1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 19, 12, 2 };
+
+static const symbol s_0[] = { 'a', '~' };
+static const symbol s_1[] = { 'o', '~' };
+static const symbol s_2[] = { 0xC3, 0xA3 };
+static const symbol s_3[] = { 0xC3, 0xB5 };
+static const symbol s_4[] = { 'l', 'o', 'g' };
+static const symbol s_5[] = { 'u' };
+static const symbol s_6[] = { 'e', 'n', 't', 'e' };
+static const symbol s_7[] = { 'a', 't' };
+static const symbol s_8[] = { 'a', 't' };
+static const symbol s_9[] = { 'e' };
+static const symbol s_10[] = { 'i', 'r' };
+static const symbol s_11[] = { 'u' };
+static const symbol s_12[] = { 'g' };
+static const symbol s_13[] = { 'i' };
+static const symbol s_14[] = { 'c' };
+static const symbol s_15[] = { 'c' };
+static const symbol s_16[] = { 'i' };
+static const symbol s_17[] = { 'c' };
+
+static int r_prelude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 36 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 37 */
+ if (z->c + 1 >= z->l || (z->p[z->c + 1] != 163 && z->p[z->c + 1] != 181)) among_var = 3; else
+ among_var = find_among(z, a_0, 3); /* substring, line 37 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 37 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 2, s_0); /* <-, line 38 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 2, s_1); /* <-, line 39 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 40 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ z->I[2] = z->l;
+ { int c1 = z->c; /* do, line 50 */
+ { int c2 = z->c; /* or, line 52 */
+ if (in_grouping_U(z, g_v, 97, 250, 0)) goto lab2;
+ { int c3 = z->c; /* or, line 51 */
+ if (out_grouping_U(z, g_v, 97, 250, 0)) goto lab4;
+ { /* gopast */ /* grouping v, line 51 */
+ int ret = out_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ goto lab3;
+ lab4:
+ z->c = c3;
+ if (in_grouping_U(z, g_v, 97, 250, 0)) goto lab2;
+ { /* gopast */ /* non v, line 51 */
+ int ret = in_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab2;
+ z->c += ret;
+ }
+ }
+ lab3:
+ goto lab1;
+ lab2:
+ z->c = c2;
+ if (out_grouping_U(z, g_v, 97, 250, 0)) goto lab0;
+ { int c4 = z->c; /* or, line 53 */
+ if (out_grouping_U(z, g_v, 97, 250, 0)) goto lab6;
+ { /* gopast */ /* grouping v, line 53 */
+ int ret = out_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab6;
+ z->c += ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = c4;
+ if (in_grouping_U(z, g_v, 97, 250, 0)) goto lab0;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 53 */
+ }
+ }
+ lab5:
+ ;
+ }
+ lab1:
+ z->I[0] = z->c; /* setmark pV, line 54 */
+ lab0:
+ z->c = c1;
+ }
+ { int c5 = z->c; /* do, line 56 */
+ { /* gopast */ /* grouping v, line 57 */
+ int ret = out_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 57 */
+ int ret = in_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p1, line 57 */
+ { /* gopast */ /* grouping v, line 58 */
+ int ret = out_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 58 */
+ int ret = in_grouping_U(z, g_v, 97, 250, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[2] = z->c; /* setmark p2, line 58 */
+ lab7:
+ z->c = c5;
+ }
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 62 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 63 */
+ if (z->c + 1 >= z->l || z->p[z->c + 1] != 126) among_var = 3; else
+ among_var = find_among(z, a_1, 3); /* substring, line 63 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 63 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 2, s_2); /* <-, line 64 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 2, s_3); /* <-, line 65 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 66 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_RV(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[2] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 77 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((839714 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_5, 45); /* substring, line 77 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 77 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 93 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 93 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 98 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_4); /* <-, line 98 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 102 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 1, s_5); /* <-, line 102 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 106 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 4, s_6); /* <-, line 106 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 110 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 110 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 111 */
+ z->ket = z->c; /* [, line 112 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4718616 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab0; }
+ among_var = find_among_b(z, a_2, 4); /* substring, line 112 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 112 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call R2, line 112 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 112 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab0; }
+ case 1:
+ z->ket = z->c; /* [, line 113 */
+ if (!(eq_s_b(z, 2, s_7))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 113 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call R2, line 113 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 113 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab0:
+ ;
+ }
+ break;
+ case 6:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 122 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 122 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 123 */
+ z->ket = z->c; /* [, line 124 */
+ if (z->c - 3 <= z->lb || (z->p[z->c - 1] != 101 && z->p[z->c - 1] != 108)) { z->c = z->l - m_keep; goto lab1; }
+ among_var = find_among_b(z, a_3, 3); /* substring, line 124 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 124 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab1; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab1; } /* call R2, line 127 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 127 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab1:
+ ;
+ }
+ break;
+ case 7:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 134 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 134 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 135 */
+ z->ket = z->c; /* [, line 136 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4198408 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab2; }
+ among_var = find_among_b(z, a_4, 3); /* substring, line 136 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab2; }
+ z->bra = z->c; /* ], line 136 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab2; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab2; } /* call R2, line 139 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 139 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab2:
+ ;
+ }
+ break;
+ case 8:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 146 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 146 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 147 */
+ z->ket = z->c; /* [, line 148 */
+ if (!(eq_s_b(z, 2, s_8))) { z->c = z->l - m_keep; goto lab3; }
+ z->bra = z->c; /* ], line 148 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 148 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 148 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ ;
+ }
+ break;
+ case 9:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 153 */
+ if (ret < 0) return ret;
+ }
+ if (!(eq_s_b(z, 1, s_9))) return 0;
+ { int ret = slice_from_s(z, 2, s_10); /* <-, line 154 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 159 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 159 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 160 */
+ among_var = find_among_b(z, a_6, 120); /* substring, line 160 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 160 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ { int ret = slice_del(z); /* delete, line 179 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_residual_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 184 */
+ among_var = find_among_b(z, a_7, 7); /* substring, line 184 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 184 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 187 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 187 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_residual_form(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 192 */
+ among_var = find_among_b(z, a_8, 4); /* substring, line 192 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 192 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 194 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 194 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 194 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 194 */
+ if (!(eq_s_b(z, 1, s_11))) goto lab1;
+ z->bra = z->c; /* ], line 194 */
+ { int m_test = z->l - z->c; /* test, line 194 */
+ if (!(eq_s_b(z, 1, s_12))) goto lab1;
+ z->c = z->l - m_test;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_13))) return 0;
+ z->bra = z->c; /* ], line 195 */
+ { int m_test = z->l - z->c; /* test, line 195 */
+ if (!(eq_s_b(z, 1, s_14))) return 0;
+ z->c = z->l - m_test;
+ }
+ }
+ lab0:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 195 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 195 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_15); /* <-, line 196 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+extern int portuguese_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 202 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab0; /* call prelude, line 202 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 203 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab1; /* call mark_regions, line 203 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = c2;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 204 */
+
+ { int m3 = z->l - z->c; (void)m3; /* do, line 205 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 209 */
+ { int m5 = z->l - z->c; (void)m5; /* and, line 207 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 206 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab6; /* call standard_suffix, line 206 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = z->l - m6;
+ { int ret = r_verb_suffix(z);
+ if (ret == 0) goto lab4; /* call verb_suffix, line 206 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ z->c = z->l - m5;
+ { int m7 = z->l - z->c; (void)m7; /* do, line 207 */
+ z->ket = z->c; /* [, line 207 */
+ if (!(eq_s_b(z, 1, s_16))) goto lab7;
+ z->bra = z->c; /* ], line 207 */
+ { int m_test = z->l - z->c; /* test, line 207 */
+ if (!(eq_s_b(z, 1, s_17))) goto lab7;
+ z->c = z->l - m_test;
+ }
+ { int ret = r_RV(z);
+ if (ret == 0) goto lab7; /* call RV, line 207 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 207 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = z->l - m7;
+ }
+ }
+ goto lab3;
+ lab4:
+ z->c = z->l - m4;
+ { int ret = r_residual_suffix(z);
+ if (ret == 0) goto lab2; /* call residual_suffix, line 209 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab3:
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m8 = z->l - z->c; (void)m8; /* do, line 211 */
+ { int ret = r_residual_form(z);
+ if (ret == 0) goto lab8; /* call residual_form, line 211 */
+ if (ret < 0) return ret;
+ }
+ lab8:
+ z->c = z->l - m8;
+ }
+ z->c = z->lb;
+ { int c9 = z->c; /* do, line 213 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab9; /* call postlude, line 213 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = c9;
+ }
+ return 1;
+}
+
+extern struct SN_env * portuguese_UTF_8_create_env(void) { return SN_create_env(0, 3, 0); }
+
+extern void portuguese_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_portuguese.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_portuguese.h
new file mode 100644
index 0000000..9fe7f9a
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_portuguese.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * portuguese_UTF_8_create_env(void);
+extern void portuguese_UTF_8_close_env(struct SN_env * z);
+
+extern int portuguese_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_romanian.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_romanian.c
new file mode 100644
index 0000000..5ca94d5
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_romanian.c
@@ -0,0 +1,1004 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int romanian_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_vowel_suffix(struct SN_env * z);
+static int r_verb_suffix(struct SN_env * z);
+static int r_combo_suffix(struct SN_env * z);
+static int r_standard_suffix(struct SN_env * z);
+static int r_step_0(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_RV(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_prelude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * romanian_UTF_8_create_env(void);
+extern void romanian_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_1[1] = { 'I' };
+static const symbol s_0_2[1] = { 'U' };
+
+static const struct among a_0[3] =
+{
+/* 0 */ { 0, 0, -1, 3, 0},
+/* 1 */ { 1, s_0_1, 0, 1, 0},
+/* 2 */ { 1, s_0_2, 0, 2, 0}
+};
+
+static const symbol s_1_0[2] = { 'e', 'a' };
+static const symbol s_1_1[5] = { 'a', 0xC5, 0xA3, 'i', 'a' };
+static const symbol s_1_2[3] = { 'a', 'u', 'a' };
+static const symbol s_1_3[3] = { 'i', 'u', 'a' };
+static const symbol s_1_4[5] = { 'a', 0xC5, 0xA3, 'i', 'e' };
+static const symbol s_1_5[3] = { 'e', 'l', 'e' };
+static const symbol s_1_6[3] = { 'i', 'l', 'e' };
+static const symbol s_1_7[4] = { 'i', 'i', 'l', 'e' };
+static const symbol s_1_8[3] = { 'i', 'e', 'i' };
+static const symbol s_1_9[4] = { 'a', 't', 'e', 'i' };
+static const symbol s_1_10[2] = { 'i', 'i' };
+static const symbol s_1_11[4] = { 'u', 'l', 'u', 'i' };
+static const symbol s_1_12[2] = { 'u', 'l' };
+static const symbol s_1_13[4] = { 'e', 'l', 'o', 'r' };
+static const symbol s_1_14[4] = { 'i', 'l', 'o', 'r' };
+static const symbol s_1_15[5] = { 'i', 'i', 'l', 'o', 'r' };
+
+static const struct among a_1[16] =
+{
+/* 0 */ { 2, s_1_0, -1, 3, 0},
+/* 1 */ { 5, s_1_1, -1, 7, 0},
+/* 2 */ { 3, s_1_2, -1, 2, 0},
+/* 3 */ { 3, s_1_3, -1, 4, 0},
+/* 4 */ { 5, s_1_4, -1, 7, 0},
+/* 5 */ { 3, s_1_5, -1, 3, 0},
+/* 6 */ { 3, s_1_6, -1, 5, 0},
+/* 7 */ { 4, s_1_7, 6, 4, 0},
+/* 8 */ { 3, s_1_8, -1, 4, 0},
+/* 9 */ { 4, s_1_9, -1, 6, 0},
+/* 10 */ { 2, s_1_10, -1, 4, 0},
+/* 11 */ { 4, s_1_11, -1, 1, 0},
+/* 12 */ { 2, s_1_12, -1, 1, 0},
+/* 13 */ { 4, s_1_13, -1, 3, 0},
+/* 14 */ { 4, s_1_14, -1, 4, 0},
+/* 15 */ { 5, s_1_15, 14, 4, 0}
+};
+
+static const symbol s_2_0[5] = { 'i', 'c', 'a', 'l', 'a' };
+static const symbol s_2_1[5] = { 'i', 'c', 'i', 'v', 'a' };
+static const symbol s_2_2[5] = { 'a', 't', 'i', 'v', 'a' };
+static const symbol s_2_3[5] = { 'i', 't', 'i', 'v', 'a' };
+static const symbol s_2_4[5] = { 'i', 'c', 'a', 'l', 'e' };
+static const symbol s_2_5[7] = { 'a', 0xC5, 0xA3, 'i', 'u', 'n', 'e' };
+static const symbol s_2_6[7] = { 'i', 0xC5, 0xA3, 'i', 'u', 'n', 'e' };
+static const symbol s_2_7[6] = { 'a', 't', 'o', 'a', 'r', 'e' };
+static const symbol s_2_8[6] = { 'i', 't', 'o', 'a', 'r', 'e' };
+static const symbol s_2_9[7] = { 0xC4, 0x83, 't', 'o', 'a', 'r', 'e' };
+static const symbol s_2_10[7] = { 'i', 'c', 'i', 't', 'a', 't', 'e' };
+static const symbol s_2_11[9] = { 'a', 'b', 'i', 'l', 'i', 't', 'a', 't', 'e' };
+static const symbol s_2_12[9] = { 'i', 'b', 'i', 'l', 'i', 't', 'a', 't', 'e' };
+static const symbol s_2_13[7] = { 'i', 'v', 'i', 't', 'a', 't', 'e' };
+static const symbol s_2_14[5] = { 'i', 'c', 'i', 'v', 'e' };
+static const symbol s_2_15[5] = { 'a', 't', 'i', 'v', 'e' };
+static const symbol s_2_16[5] = { 'i', 't', 'i', 'v', 'e' };
+static const symbol s_2_17[5] = { 'i', 'c', 'a', 'l', 'i' };
+static const symbol s_2_18[5] = { 'a', 't', 'o', 'r', 'i' };
+static const symbol s_2_19[7] = { 'i', 'c', 'a', 't', 'o', 'r', 'i' };
+static const symbol s_2_20[5] = { 'i', 't', 'o', 'r', 'i' };
+static const symbol s_2_21[6] = { 0xC4, 0x83, 't', 'o', 'r', 'i' };
+static const symbol s_2_22[7] = { 'i', 'c', 'i', 't', 'a', 't', 'i' };
+static const symbol s_2_23[9] = { 'a', 'b', 'i', 'l', 'i', 't', 'a', 't', 'i' };
+static const symbol s_2_24[7] = { 'i', 'v', 'i', 't', 'a', 't', 'i' };
+static const symbol s_2_25[5] = { 'i', 'c', 'i', 'v', 'i' };
+static const symbol s_2_26[5] = { 'a', 't', 'i', 'v', 'i' };
+static const symbol s_2_27[5] = { 'i', 't', 'i', 'v', 'i' };
+static const symbol s_2_28[7] = { 'i', 'c', 'i', 't', 0xC4, 0x83, 'i' };
+static const symbol s_2_29[9] = { 'a', 'b', 'i', 'l', 'i', 't', 0xC4, 0x83, 'i' };
+static const symbol s_2_30[7] = { 'i', 'v', 'i', 't', 0xC4, 0x83, 'i' };
+static const symbol s_2_31[9] = { 'i', 'c', 'i', 't', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_2_32[11] = { 'a', 'b', 'i', 'l', 'i', 't', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_2_33[9] = { 'i', 'v', 'i', 't', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_2_34[4] = { 'i', 'c', 'a', 'l' };
+static const symbol s_2_35[4] = { 'a', 't', 'o', 'r' };
+static const symbol s_2_36[6] = { 'i', 'c', 'a', 't', 'o', 'r' };
+static const symbol s_2_37[4] = { 'i', 't', 'o', 'r' };
+static const symbol s_2_38[5] = { 0xC4, 0x83, 't', 'o', 'r' };
+static const symbol s_2_39[4] = { 'i', 'c', 'i', 'v' };
+static const symbol s_2_40[4] = { 'a', 't', 'i', 'v' };
+static const symbol s_2_41[4] = { 'i', 't', 'i', 'v' };
+static const symbol s_2_42[6] = { 'i', 'c', 'a', 'l', 0xC4, 0x83 };
+static const symbol s_2_43[6] = { 'i', 'c', 'i', 'v', 0xC4, 0x83 };
+static const symbol s_2_44[6] = { 'a', 't', 'i', 'v', 0xC4, 0x83 };
+static const symbol s_2_45[6] = { 'i', 't', 'i', 'v', 0xC4, 0x83 };
+
+static const struct among a_2[46] =
+{
+/* 0 */ { 5, s_2_0, -1, 4, 0},
+/* 1 */ { 5, s_2_1, -1, 4, 0},
+/* 2 */ { 5, s_2_2, -1, 5, 0},
+/* 3 */ { 5, s_2_3, -1, 6, 0},
+/* 4 */ { 5, s_2_4, -1, 4, 0},
+/* 5 */ { 7, s_2_5, -1, 5, 0},
+/* 6 */ { 7, s_2_6, -1, 6, 0},
+/* 7 */ { 6, s_2_7, -1, 5, 0},
+/* 8 */ { 6, s_2_8, -1, 6, 0},
+/* 9 */ { 7, s_2_9, -1, 5, 0},
+/* 10 */ { 7, s_2_10, -1, 4, 0},
+/* 11 */ { 9, s_2_11, -1, 1, 0},
+/* 12 */ { 9, s_2_12, -1, 2, 0},
+/* 13 */ { 7, s_2_13, -1, 3, 0},
+/* 14 */ { 5, s_2_14, -1, 4, 0},
+/* 15 */ { 5, s_2_15, -1, 5, 0},
+/* 16 */ { 5, s_2_16, -1, 6, 0},
+/* 17 */ { 5, s_2_17, -1, 4, 0},
+/* 18 */ { 5, s_2_18, -1, 5, 0},
+/* 19 */ { 7, s_2_19, 18, 4, 0},
+/* 20 */ { 5, s_2_20, -1, 6, 0},
+/* 21 */ { 6, s_2_21, -1, 5, 0},
+/* 22 */ { 7, s_2_22, -1, 4, 0},
+/* 23 */ { 9, s_2_23, -1, 1, 0},
+/* 24 */ { 7, s_2_24, -1, 3, 0},
+/* 25 */ { 5, s_2_25, -1, 4, 0},
+/* 26 */ { 5, s_2_26, -1, 5, 0},
+/* 27 */ { 5, s_2_27, -1, 6, 0},
+/* 28 */ { 7, s_2_28, -1, 4, 0},
+/* 29 */ { 9, s_2_29, -1, 1, 0},
+/* 30 */ { 7, s_2_30, -1, 3, 0},
+/* 31 */ { 9, s_2_31, -1, 4, 0},
+/* 32 */ { 11, s_2_32, -1, 1, 0},
+/* 33 */ { 9, s_2_33, -1, 3, 0},
+/* 34 */ { 4, s_2_34, -1, 4, 0},
+/* 35 */ { 4, s_2_35, -1, 5, 0},
+/* 36 */ { 6, s_2_36, 35, 4, 0},
+/* 37 */ { 4, s_2_37, -1, 6, 0},
+/* 38 */ { 5, s_2_38, -1, 5, 0},
+/* 39 */ { 4, s_2_39, -1, 4, 0},
+/* 40 */ { 4, s_2_40, -1, 5, 0},
+/* 41 */ { 4, s_2_41, -1, 6, 0},
+/* 42 */ { 6, s_2_42, -1, 4, 0},
+/* 43 */ { 6, s_2_43, -1, 4, 0},
+/* 44 */ { 6, s_2_44, -1, 5, 0},
+/* 45 */ { 6, s_2_45, -1, 6, 0}
+};
+
+static const symbol s_3_0[3] = { 'i', 'c', 'a' };
+static const symbol s_3_1[5] = { 'a', 'b', 'i', 'l', 'a' };
+static const symbol s_3_2[5] = { 'i', 'b', 'i', 'l', 'a' };
+static const symbol s_3_3[4] = { 'o', 'a', 's', 'a' };
+static const symbol s_3_4[3] = { 'a', 't', 'a' };
+static const symbol s_3_5[3] = { 'i', 't', 'a' };
+static const symbol s_3_6[4] = { 'a', 'n', 't', 'a' };
+static const symbol s_3_7[4] = { 'i', 's', 't', 'a' };
+static const symbol s_3_8[3] = { 'u', 't', 'a' };
+static const symbol s_3_9[3] = { 'i', 'v', 'a' };
+static const symbol s_3_10[2] = { 'i', 'c' };
+static const symbol s_3_11[3] = { 'i', 'c', 'e' };
+static const symbol s_3_12[5] = { 'a', 'b', 'i', 'l', 'e' };
+static const symbol s_3_13[5] = { 'i', 'b', 'i', 'l', 'e' };
+static const symbol s_3_14[4] = { 'i', 's', 'm', 'e' };
+static const symbol s_3_15[4] = { 'i', 'u', 'n', 'e' };
+static const symbol s_3_16[4] = { 'o', 'a', 's', 'e' };
+static const symbol s_3_17[3] = { 'a', 't', 'e' };
+static const symbol s_3_18[5] = { 'i', 't', 'a', 't', 'e' };
+static const symbol s_3_19[3] = { 'i', 't', 'e' };
+static const symbol s_3_20[4] = { 'a', 'n', 't', 'e' };
+static const symbol s_3_21[4] = { 'i', 's', 't', 'e' };
+static const symbol s_3_22[3] = { 'u', 't', 'e' };
+static const symbol s_3_23[3] = { 'i', 'v', 'e' };
+static const symbol s_3_24[3] = { 'i', 'c', 'i' };
+static const symbol s_3_25[5] = { 'a', 'b', 'i', 'l', 'i' };
+static const symbol s_3_26[5] = { 'i', 'b', 'i', 'l', 'i' };
+static const symbol s_3_27[4] = { 'i', 'u', 'n', 'i' };
+static const symbol s_3_28[5] = { 'a', 't', 'o', 'r', 'i' };
+static const symbol s_3_29[3] = { 'o', 's', 'i' };
+static const symbol s_3_30[3] = { 'a', 't', 'i' };
+static const symbol s_3_31[5] = { 'i', 't', 'a', 't', 'i' };
+static const symbol s_3_32[3] = { 'i', 't', 'i' };
+static const symbol s_3_33[4] = { 'a', 'n', 't', 'i' };
+static const symbol s_3_34[4] = { 'i', 's', 't', 'i' };
+static const symbol s_3_35[3] = { 'u', 't', 'i' };
+static const symbol s_3_36[5] = { 'i', 0xC5, 0x9F, 't', 'i' };
+static const symbol s_3_37[3] = { 'i', 'v', 'i' };
+static const symbol s_3_38[5] = { 'i', 't', 0xC4, 0x83, 'i' };
+static const symbol s_3_39[4] = { 'o', 0xC5, 0x9F, 'i' };
+static const symbol s_3_40[7] = { 'i', 't', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_3_41[4] = { 'a', 'b', 'i', 'l' };
+static const symbol s_3_42[4] = { 'i', 'b', 'i', 'l' };
+static const symbol s_3_43[3] = { 'i', 's', 'm' };
+static const symbol s_3_44[4] = { 'a', 't', 'o', 'r' };
+static const symbol s_3_45[2] = { 'o', 's' };
+static const symbol s_3_46[2] = { 'a', 't' };
+static const symbol s_3_47[2] = { 'i', 't' };
+static const symbol s_3_48[3] = { 'a', 'n', 't' };
+static const symbol s_3_49[3] = { 'i', 's', 't' };
+static const symbol s_3_50[2] = { 'u', 't' };
+static const symbol s_3_51[2] = { 'i', 'v' };
+static const symbol s_3_52[4] = { 'i', 'c', 0xC4, 0x83 };
+static const symbol s_3_53[6] = { 'a', 'b', 'i', 'l', 0xC4, 0x83 };
+static const symbol s_3_54[6] = { 'i', 'b', 'i', 'l', 0xC4, 0x83 };
+static const symbol s_3_55[5] = { 'o', 'a', 's', 0xC4, 0x83 };
+static const symbol s_3_56[4] = { 'a', 't', 0xC4, 0x83 };
+static const symbol s_3_57[4] = { 'i', 't', 0xC4, 0x83 };
+static const symbol s_3_58[5] = { 'a', 'n', 't', 0xC4, 0x83 };
+static const symbol s_3_59[5] = { 'i', 's', 't', 0xC4, 0x83 };
+static const symbol s_3_60[4] = { 'u', 't', 0xC4, 0x83 };
+static const symbol s_3_61[4] = { 'i', 'v', 0xC4, 0x83 };
+
+static const struct among a_3[62] =
+{
+/* 0 */ { 3, s_3_0, -1, 1, 0},
+/* 1 */ { 5, s_3_1, -1, 1, 0},
+/* 2 */ { 5, s_3_2, -1, 1, 0},
+/* 3 */ { 4, s_3_3, -1, 1, 0},
+/* 4 */ { 3, s_3_4, -1, 1, 0},
+/* 5 */ { 3, s_3_5, -1, 1, 0},
+/* 6 */ { 4, s_3_6, -1, 1, 0},
+/* 7 */ { 4, s_3_7, -1, 3, 0},
+/* 8 */ { 3, s_3_8, -1, 1, 0},
+/* 9 */ { 3, s_3_9, -1, 1, 0},
+/* 10 */ { 2, s_3_10, -1, 1, 0},
+/* 11 */ { 3, s_3_11, -1, 1, 0},
+/* 12 */ { 5, s_3_12, -1, 1, 0},
+/* 13 */ { 5, s_3_13, -1, 1, 0},
+/* 14 */ { 4, s_3_14, -1, 3, 0},
+/* 15 */ { 4, s_3_15, -1, 2, 0},
+/* 16 */ { 4, s_3_16, -1, 1, 0},
+/* 17 */ { 3, s_3_17, -1, 1, 0},
+/* 18 */ { 5, s_3_18, 17, 1, 0},
+/* 19 */ { 3, s_3_19, -1, 1, 0},
+/* 20 */ { 4, s_3_20, -1, 1, 0},
+/* 21 */ { 4, s_3_21, -1, 3, 0},
+/* 22 */ { 3, s_3_22, -1, 1, 0},
+/* 23 */ { 3, s_3_23, -1, 1, 0},
+/* 24 */ { 3, s_3_24, -1, 1, 0},
+/* 25 */ { 5, s_3_25, -1, 1, 0},
+/* 26 */ { 5, s_3_26, -1, 1, 0},
+/* 27 */ { 4, s_3_27, -1, 2, 0},
+/* 28 */ { 5, s_3_28, -1, 1, 0},
+/* 29 */ { 3, s_3_29, -1, 1, 0},
+/* 30 */ { 3, s_3_30, -1, 1, 0},
+/* 31 */ { 5, s_3_31, 30, 1, 0},
+/* 32 */ { 3, s_3_32, -1, 1, 0},
+/* 33 */ { 4, s_3_33, -1, 1, 0},
+/* 34 */ { 4, s_3_34, -1, 3, 0},
+/* 35 */ { 3, s_3_35, -1, 1, 0},
+/* 36 */ { 5, s_3_36, -1, 3, 0},
+/* 37 */ { 3, s_3_37, -1, 1, 0},
+/* 38 */ { 5, s_3_38, -1, 1, 0},
+/* 39 */ { 4, s_3_39, -1, 1, 0},
+/* 40 */ { 7, s_3_40, -1, 1, 0},
+/* 41 */ { 4, s_3_41, -1, 1, 0},
+/* 42 */ { 4, s_3_42, -1, 1, 0},
+/* 43 */ { 3, s_3_43, -1, 3, 0},
+/* 44 */ { 4, s_3_44, -1, 1, 0},
+/* 45 */ { 2, s_3_45, -1, 1, 0},
+/* 46 */ { 2, s_3_46, -1, 1, 0},
+/* 47 */ { 2, s_3_47, -1, 1, 0},
+/* 48 */ { 3, s_3_48, -1, 1, 0},
+/* 49 */ { 3, s_3_49, -1, 3, 0},
+/* 50 */ { 2, s_3_50, -1, 1, 0},
+/* 51 */ { 2, s_3_51, -1, 1, 0},
+/* 52 */ { 4, s_3_52, -1, 1, 0},
+/* 53 */ { 6, s_3_53, -1, 1, 0},
+/* 54 */ { 6, s_3_54, -1, 1, 0},
+/* 55 */ { 5, s_3_55, -1, 1, 0},
+/* 56 */ { 4, s_3_56, -1, 1, 0},
+/* 57 */ { 4, s_3_57, -1, 1, 0},
+/* 58 */ { 5, s_3_58, -1, 1, 0},
+/* 59 */ { 5, s_3_59, -1, 3, 0},
+/* 60 */ { 4, s_3_60, -1, 1, 0},
+/* 61 */ { 4, s_3_61, -1, 1, 0}
+};
+
+static const symbol s_4_0[2] = { 'e', 'a' };
+static const symbol s_4_1[2] = { 'i', 'a' };
+static const symbol s_4_2[3] = { 'e', 's', 'c' };
+static const symbol s_4_3[4] = { 0xC4, 0x83, 's', 'c' };
+static const symbol s_4_4[3] = { 'i', 'n', 'd' };
+static const symbol s_4_5[4] = { 0xC3, 0xA2, 'n', 'd' };
+static const symbol s_4_6[3] = { 'a', 'r', 'e' };
+static const symbol s_4_7[3] = { 'e', 'r', 'e' };
+static const symbol s_4_8[3] = { 'i', 'r', 'e' };
+static const symbol s_4_9[4] = { 0xC3, 0xA2, 'r', 'e' };
+static const symbol s_4_10[2] = { 's', 'e' };
+static const symbol s_4_11[3] = { 'a', 's', 'e' };
+static const symbol s_4_12[4] = { 's', 'e', 's', 'e' };
+static const symbol s_4_13[3] = { 'i', 's', 'e' };
+static const symbol s_4_14[3] = { 'u', 's', 'e' };
+static const symbol s_4_15[4] = { 0xC3, 0xA2, 's', 'e' };
+static const symbol s_4_16[5] = { 'e', 0xC5, 0x9F, 't', 'e' };
+static const symbol s_4_17[6] = { 0xC4, 0x83, 0xC5, 0x9F, 't', 'e' };
+static const symbol s_4_18[3] = { 'e', 'z', 'e' };
+static const symbol s_4_19[2] = { 'a', 'i' };
+static const symbol s_4_20[3] = { 'e', 'a', 'i' };
+static const symbol s_4_21[3] = { 'i', 'a', 'i' };
+static const symbol s_4_22[3] = { 's', 'e', 'i' };
+static const symbol s_4_23[5] = { 'e', 0xC5, 0x9F, 't', 'i' };
+static const symbol s_4_24[6] = { 0xC4, 0x83, 0xC5, 0x9F, 't', 'i' };
+static const symbol s_4_25[2] = { 'u', 'i' };
+static const symbol s_4_26[3] = { 'e', 'z', 'i' };
+static const symbol s_4_27[4] = { 'a', 0xC5, 0x9F, 'i' };
+static const symbol s_4_28[5] = { 's', 'e', 0xC5, 0x9F, 'i' };
+static const symbol s_4_29[6] = { 'a', 's', 'e', 0xC5, 0x9F, 'i' };
+static const symbol s_4_30[7] = { 's', 'e', 's', 'e', 0xC5, 0x9F, 'i' };
+static const symbol s_4_31[6] = { 'i', 's', 'e', 0xC5, 0x9F, 'i' };
+static const symbol s_4_32[6] = { 'u', 's', 'e', 0xC5, 0x9F, 'i' };
+static const symbol s_4_33[7] = { 0xC3, 0xA2, 's', 'e', 0xC5, 0x9F, 'i' };
+static const symbol s_4_34[4] = { 'i', 0xC5, 0x9F, 'i' };
+static const symbol s_4_35[4] = { 'u', 0xC5, 0x9F, 'i' };
+static const symbol s_4_36[5] = { 0xC3, 0xA2, 0xC5, 0x9F, 'i' };
+static const symbol s_4_37[3] = { 0xC3, 0xA2, 'i' };
+static const symbol s_4_38[4] = { 'a', 0xC5, 0xA3, 'i' };
+static const symbol s_4_39[5] = { 'e', 'a', 0xC5, 0xA3, 'i' };
+static const symbol s_4_40[5] = { 'i', 'a', 0xC5, 0xA3, 'i' };
+static const symbol s_4_41[4] = { 'e', 0xC5, 0xA3, 'i' };
+static const symbol s_4_42[4] = { 'i', 0xC5, 0xA3, 'i' };
+static const symbol s_4_43[7] = { 'a', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_44[8] = { 's', 'e', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_45[9] = { 'a', 's', 'e', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_46[10] = { 's', 'e', 's', 'e', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_47[9] = { 'i', 's', 'e', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_48[9] = { 'u', 's', 'e', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_49[10] = { 0xC3, 0xA2, 's', 'e', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_50[7] = { 'i', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_51[7] = { 'u', 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_52[8] = { 0xC3, 0xA2, 'r', 0xC4, 0x83, 0xC5, 0xA3, 'i' };
+static const symbol s_4_53[5] = { 0xC3, 0xA2, 0xC5, 0xA3, 'i' };
+static const symbol s_4_54[2] = { 'a', 'm' };
+static const symbol s_4_55[3] = { 'e', 'a', 'm' };
+static const symbol s_4_56[3] = { 'i', 'a', 'm' };
+static const symbol s_4_57[2] = { 'e', 'm' };
+static const symbol s_4_58[4] = { 'a', 's', 'e', 'm' };
+static const symbol s_4_59[5] = { 's', 'e', 's', 'e', 'm' };
+static const symbol s_4_60[4] = { 'i', 's', 'e', 'm' };
+static const symbol s_4_61[4] = { 'u', 's', 'e', 'm' };
+static const symbol s_4_62[5] = { 0xC3, 0xA2, 's', 'e', 'm' };
+static const symbol s_4_63[2] = { 'i', 'm' };
+static const symbol s_4_64[3] = { 0xC4, 0x83, 'm' };
+static const symbol s_4_65[5] = { 'a', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_66[6] = { 's', 'e', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_67[7] = { 'a', 's', 'e', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_68[8] = { 's', 'e', 's', 'e', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_69[7] = { 'i', 's', 'e', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_70[7] = { 'u', 's', 'e', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_71[8] = { 0xC3, 0xA2, 's', 'e', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_72[5] = { 'i', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_73[5] = { 'u', 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_74[6] = { 0xC3, 0xA2, 'r', 0xC4, 0x83, 'm' };
+static const symbol s_4_75[3] = { 0xC3, 0xA2, 'm' };
+static const symbol s_4_76[2] = { 'a', 'u' };
+static const symbol s_4_77[3] = { 'e', 'a', 'u' };
+static const symbol s_4_78[3] = { 'i', 'a', 'u' };
+static const symbol s_4_79[4] = { 'i', 'n', 'd', 'u' };
+static const symbol s_4_80[5] = { 0xC3, 0xA2, 'n', 'd', 'u' };
+static const symbol s_4_81[2] = { 'e', 'z' };
+static const symbol s_4_82[6] = { 'e', 'a', 's', 'c', 0xC4, 0x83 };
+static const symbol s_4_83[4] = { 'a', 'r', 0xC4, 0x83 };
+static const symbol s_4_84[5] = { 's', 'e', 'r', 0xC4, 0x83 };
+static const symbol s_4_85[6] = { 'a', 's', 'e', 'r', 0xC4, 0x83 };
+static const symbol s_4_86[7] = { 's', 'e', 's', 'e', 'r', 0xC4, 0x83 };
+static const symbol s_4_87[6] = { 'i', 's', 'e', 'r', 0xC4, 0x83 };
+static const symbol s_4_88[6] = { 'u', 's', 'e', 'r', 0xC4, 0x83 };
+static const symbol s_4_89[7] = { 0xC3, 0xA2, 's', 'e', 'r', 0xC4, 0x83 };
+static const symbol s_4_90[4] = { 'i', 'r', 0xC4, 0x83 };
+static const symbol s_4_91[4] = { 'u', 'r', 0xC4, 0x83 };
+static const symbol s_4_92[5] = { 0xC3, 0xA2, 'r', 0xC4, 0x83 };
+static const symbol s_4_93[5] = { 'e', 'a', 'z', 0xC4, 0x83 };
+
+static const struct among a_4[94] =
+{
+/* 0 */ { 2, s_4_0, -1, 1, 0},
+/* 1 */ { 2, s_4_1, -1, 1, 0},
+/* 2 */ { 3, s_4_2, -1, 1, 0},
+/* 3 */ { 4, s_4_3, -1, 1, 0},
+/* 4 */ { 3, s_4_4, -1, 1, 0},
+/* 5 */ { 4, s_4_5, -1, 1, 0},
+/* 6 */ { 3, s_4_6, -1, 1, 0},
+/* 7 */ { 3, s_4_7, -1, 1, 0},
+/* 8 */ { 3, s_4_8, -1, 1, 0},
+/* 9 */ { 4, s_4_9, -1, 1, 0},
+/* 10 */ { 2, s_4_10, -1, 2, 0},
+/* 11 */ { 3, s_4_11, 10, 1, 0},
+/* 12 */ { 4, s_4_12, 10, 2, 0},
+/* 13 */ { 3, s_4_13, 10, 1, 0},
+/* 14 */ { 3, s_4_14, 10, 1, 0},
+/* 15 */ { 4, s_4_15, 10, 1, 0},
+/* 16 */ { 5, s_4_16, -1, 1, 0},
+/* 17 */ { 6, s_4_17, -1, 1, 0},
+/* 18 */ { 3, s_4_18, -1, 1, 0},
+/* 19 */ { 2, s_4_19, -1, 1, 0},
+/* 20 */ { 3, s_4_20, 19, 1, 0},
+/* 21 */ { 3, s_4_21, 19, 1, 0},
+/* 22 */ { 3, s_4_22, -1, 2, 0},
+/* 23 */ { 5, s_4_23, -1, 1, 0},
+/* 24 */ { 6, s_4_24, -1, 1, 0},
+/* 25 */ { 2, s_4_25, -1, 1, 0},
+/* 26 */ { 3, s_4_26, -1, 1, 0},
+/* 27 */ { 4, s_4_27, -1, 1, 0},
+/* 28 */ { 5, s_4_28, -1, 2, 0},
+/* 29 */ { 6, s_4_29, 28, 1, 0},
+/* 30 */ { 7, s_4_30, 28, 2, 0},
+/* 31 */ { 6, s_4_31, 28, 1, 0},
+/* 32 */ { 6, s_4_32, 28, 1, 0},
+/* 33 */ { 7, s_4_33, 28, 1, 0},
+/* 34 */ { 4, s_4_34, -1, 1, 0},
+/* 35 */ { 4, s_4_35, -1, 1, 0},
+/* 36 */ { 5, s_4_36, -1, 1, 0},
+/* 37 */ { 3, s_4_37, -1, 1, 0},
+/* 38 */ { 4, s_4_38, -1, 2, 0},
+/* 39 */ { 5, s_4_39, 38, 1, 0},
+/* 40 */ { 5, s_4_40, 38, 1, 0},
+/* 41 */ { 4, s_4_41, -1, 2, 0},
+/* 42 */ { 4, s_4_42, -1, 2, 0},
+/* 43 */ { 7, s_4_43, -1, 1, 0},
+/* 44 */ { 8, s_4_44, -1, 2, 0},
+/* 45 */ { 9, s_4_45, 44, 1, 0},
+/* 46 */ { 10, s_4_46, 44, 2, 0},
+/* 47 */ { 9, s_4_47, 44, 1, 0},
+/* 48 */ { 9, s_4_48, 44, 1, 0},
+/* 49 */ { 10, s_4_49, 44, 1, 0},
+/* 50 */ { 7, s_4_50, -1, 1, 0},
+/* 51 */ { 7, s_4_51, -1, 1, 0},
+/* 52 */ { 8, s_4_52, -1, 1, 0},
+/* 53 */ { 5, s_4_53, -1, 2, 0},
+/* 54 */ { 2, s_4_54, -1, 1, 0},
+/* 55 */ { 3, s_4_55, 54, 1, 0},
+/* 56 */ { 3, s_4_56, 54, 1, 0},
+/* 57 */ { 2, s_4_57, -1, 2, 0},
+/* 58 */ { 4, s_4_58, 57, 1, 0},
+/* 59 */ { 5, s_4_59, 57, 2, 0},
+/* 60 */ { 4, s_4_60, 57, 1, 0},
+/* 61 */ { 4, s_4_61, 57, 1, 0},
+/* 62 */ { 5, s_4_62, 57, 1, 0},
+/* 63 */ { 2, s_4_63, -1, 2, 0},
+/* 64 */ { 3, s_4_64, -1, 2, 0},
+/* 65 */ { 5, s_4_65, 64, 1, 0},
+/* 66 */ { 6, s_4_66, 64, 2, 0},
+/* 67 */ { 7, s_4_67, 66, 1, 0},
+/* 68 */ { 8, s_4_68, 66, 2, 0},
+/* 69 */ { 7, s_4_69, 66, 1, 0},
+/* 70 */ { 7, s_4_70, 66, 1, 0},
+/* 71 */ { 8, s_4_71, 66, 1, 0},
+/* 72 */ { 5, s_4_72, 64, 1, 0},
+/* 73 */ { 5, s_4_73, 64, 1, 0},
+/* 74 */ { 6, s_4_74, 64, 1, 0},
+/* 75 */ { 3, s_4_75, -1, 2, 0},
+/* 76 */ { 2, s_4_76, -1, 1, 0},
+/* 77 */ { 3, s_4_77, 76, 1, 0},
+/* 78 */ { 3, s_4_78, 76, 1, 0},
+/* 79 */ { 4, s_4_79, -1, 1, 0},
+/* 80 */ { 5, s_4_80, -1, 1, 0},
+/* 81 */ { 2, s_4_81, -1, 1, 0},
+/* 82 */ { 6, s_4_82, -1, 1, 0},
+/* 83 */ { 4, s_4_83, -1, 1, 0},
+/* 84 */ { 5, s_4_84, -1, 2, 0},
+/* 85 */ { 6, s_4_85, 84, 1, 0},
+/* 86 */ { 7, s_4_86, 84, 2, 0},
+/* 87 */ { 6, s_4_87, 84, 1, 0},
+/* 88 */ { 6, s_4_88, 84, 1, 0},
+/* 89 */ { 7, s_4_89, 84, 1, 0},
+/* 90 */ { 4, s_4_90, -1, 1, 0},
+/* 91 */ { 4, s_4_91, -1, 1, 0},
+/* 92 */ { 5, s_4_92, -1, 1, 0},
+/* 93 */ { 5, s_4_93, -1, 1, 0}
+};
+
+static const symbol s_5_0[1] = { 'a' };
+static const symbol s_5_1[1] = { 'e' };
+static const symbol s_5_2[2] = { 'i', 'e' };
+static const symbol s_5_3[1] = { 'i' };
+static const symbol s_5_4[2] = { 0xC4, 0x83 };
+
+static const struct among a_5[5] =
+{
+/* 0 */ { 1, s_5_0, -1, 1, 0},
+/* 1 */ { 1, s_5_1, -1, 1, 0},
+/* 2 */ { 2, s_5_2, 1, 1, 0},
+/* 3 */ { 1, s_5_3, -1, 1, 0},
+/* 4 */ { 2, s_5_4, -1, 1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 32, 0, 0, 4 };
+
+static const symbol s_0[] = { 'u' };
+static const symbol s_1[] = { 'U' };
+static const symbol s_2[] = { 'i' };
+static const symbol s_3[] = { 'I' };
+static const symbol s_4[] = { 'i' };
+static const symbol s_5[] = { 'u' };
+static const symbol s_6[] = { 'a' };
+static const symbol s_7[] = { 'e' };
+static const symbol s_8[] = { 'i' };
+static const symbol s_9[] = { 'a', 'b' };
+static const symbol s_10[] = { 'i' };
+static const symbol s_11[] = { 'a', 't' };
+static const symbol s_12[] = { 'a', 0xC5, 0xA3, 'i' };
+static const symbol s_13[] = { 'a', 'b', 'i', 'l' };
+static const symbol s_14[] = { 'i', 'b', 'i', 'l' };
+static const symbol s_15[] = { 'i', 'v' };
+static const symbol s_16[] = { 'i', 'c' };
+static const symbol s_17[] = { 'a', 't' };
+static const symbol s_18[] = { 'i', 't' };
+static const symbol s_19[] = { 0xC5, 0xA3 };
+static const symbol s_20[] = { 't' };
+static const symbol s_21[] = { 'i', 's', 't' };
+static const symbol s_22[] = { 'u' };
+
+static int r_prelude(struct SN_env * z) {
+ while(1) { /* repeat, line 32 */
+ int c1 = z->c;
+ while(1) { /* goto, line 32 */
+ int c2 = z->c;
+ if (in_grouping_U(z, g_v, 97, 259, 0)) goto lab1;
+ z->bra = z->c; /* [, line 33 */
+ { int c3 = z->c; /* or, line 33 */
+ if (!(eq_s(z, 1, s_0))) goto lab3;
+ z->ket = z->c; /* ], line 33 */
+ if (in_grouping_U(z, g_v, 97, 259, 0)) goto lab3;
+ { int ret = slice_from_s(z, 1, s_1); /* <-, line 33 */
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab3:
+ z->c = c3;
+ if (!(eq_s(z, 1, s_2))) goto lab1;
+ z->ket = z->c; /* ], line 34 */
+ if (in_grouping_U(z, g_v, 97, 259, 0)) goto lab1;
+ { int ret = slice_from_s(z, 1, s_3); /* <-, line 34 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab2:
+ z->c = c2;
+ break;
+ lab1:
+ z->c = c2;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* goto, line 32 */
+ }
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ z->I[2] = z->l;
+ { int c1 = z->c; /* do, line 44 */
+ { int c2 = z->c; /* or, line 46 */
+ if (in_grouping_U(z, g_v, 97, 259, 0)) goto lab2;
+ { int c3 = z->c; /* or, line 45 */
+ if (out_grouping_U(z, g_v, 97, 259, 0)) goto lab4;
+ { /* gopast */ /* grouping v, line 45 */
+ int ret = out_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ goto lab3;
+ lab4:
+ z->c = c3;
+ if (in_grouping_U(z, g_v, 97, 259, 0)) goto lab2;
+ { /* gopast */ /* non v, line 45 */
+ int ret = in_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab2;
+ z->c += ret;
+ }
+ }
+ lab3:
+ goto lab1;
+ lab2:
+ z->c = c2;
+ if (out_grouping_U(z, g_v, 97, 259, 0)) goto lab0;
+ { int c4 = z->c; /* or, line 47 */
+ if (out_grouping_U(z, g_v, 97, 259, 0)) goto lab6;
+ { /* gopast */ /* grouping v, line 47 */
+ int ret = out_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab6;
+ z->c += ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = c4;
+ if (in_grouping_U(z, g_v, 97, 259, 0)) goto lab0;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 47 */
+ }
+ }
+ lab5:
+ ;
+ }
+ lab1:
+ z->I[0] = z->c; /* setmark pV, line 48 */
+ lab0:
+ z->c = c1;
+ }
+ { int c5 = z->c; /* do, line 50 */
+ { /* gopast */ /* grouping v, line 51 */
+ int ret = out_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 51 */
+ int ret = in_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p1, line 51 */
+ { /* gopast */ /* grouping v, line 52 */
+ int ret = out_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 52 */
+ int ret = in_grouping_U(z, g_v, 97, 259, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[2] = z->c; /* setmark p2, line 52 */
+ lab7:
+ z->c = c5;
+ }
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 56 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 58 */
+ if (z->c >= z->l || (z->p[z->c + 0] != 73 && z->p[z->c + 0] != 85)) among_var = 3; else
+ among_var = find_among(z, a_0, 3); /* substring, line 58 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 58 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_4); /* <-, line 59 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_5); /* <-, line 60 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 61 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_RV(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[2] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_step_0(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 73 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((266786 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_1, 16); /* substring, line 73 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 73 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 73 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 75 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_6); /* <-, line 77 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_7); /* <-, line 79 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 1, s_8); /* <-, line 81 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int m1 = z->l - z->c; (void)m1; /* not, line 83 */
+ if (!(eq_s_b(z, 2, s_9))) goto lab0;
+ return 0;
+ lab0:
+ z->c = z->l - m1;
+ }
+ { int ret = slice_from_s(z, 1, s_10); /* <-, line 83 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 2, s_11); /* <-, line 85 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ { int ret = slice_from_s(z, 4, s_12); /* <-, line 87 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_combo_suffix(struct SN_env * z) {
+ int among_var;
+ { int m_test = z->l - z->c; /* test, line 91 */
+ z->ket = z->c; /* [, line 92 */
+ among_var = find_among_b(z, a_2, 46); /* substring, line 92 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 92 */
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 92 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 4, s_13); /* <-, line 101 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 4, s_14); /* <-, line 104 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 2, s_15); /* <-, line 107 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 2, s_16); /* <-, line 113 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 2, s_17); /* <-, line 118 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_from_s(z, 2, s_18); /* <-, line 122 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->B[0] = 1; /* set standard_suffix_removed, line 125 */
+ z->c = z->l - m_test;
+ }
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ z->B[0] = 0; /* unset standard_suffix_removed, line 130 */
+ while(1) { /* repeat, line 131 */
+ int m1 = z->l - z->c; (void)m1;
+ { int ret = r_combo_suffix(z);
+ if (ret == 0) goto lab0; /* call combo_suffix, line 131 */
+ if (ret < 0) return ret;
+ }
+ continue;
+ lab0:
+ z->c = z->l - m1;
+ break;
+ }
+ z->ket = z->c; /* [, line 132 */
+ among_var = find_among_b(z, a_3, 62); /* substring, line 132 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 132 */
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 132 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 149 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ if (!(eq_s_b(z, 2, s_19))) return 0;
+ z->bra = z->c; /* ], line 152 */
+ { int ret = slice_from_s(z, 1, s_20); /* <-, line 152 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 3, s_21); /* <-, line 156 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->B[0] = 1; /* set standard_suffix_removed, line 160 */
+ return 1;
+}
+
+static int r_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 164 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 164 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 165 */
+ among_var = find_among_b(z, a_4, 94); /* substring, line 165 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 165 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ { int m2 = z->l - z->c; (void)m2; /* or, line 200 */
+ if (out_grouping_b_U(z, g_v, 97, 259, 0)) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m2;
+ if (!(eq_s_b(z, 1, s_22))) { z->lb = mlimit; return 0; }
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 200 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 214 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_vowel_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 219 */
+ among_var = find_among_b(z, a_5, 5); /* substring, line 219 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 219 */
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 219 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 220 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+extern int romanian_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 226 */
+ { int ret = r_prelude(z);
+ if (ret == 0) goto lab0; /* call prelude, line 226 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ { int c2 = z->c; /* do, line 227 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab1; /* call mark_regions, line 227 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = c2;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 228 */
+
+ { int m3 = z->l - z->c; (void)m3; /* do, line 229 */
+ { int ret = r_step_0(z);
+ if (ret == 0) goto lab2; /* call step_0, line 229 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 230 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab3; /* call standard_suffix, line 230 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = z->l - m4;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 231 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 231 */
+ if (!(z->B[0])) goto lab6; /* Boolean test standard_suffix_removed, line 231 */
+ goto lab5;
+ lab6:
+ z->c = z->l - m6;
+ { int ret = r_verb_suffix(z);
+ if (ret == 0) goto lab4; /* call verb_suffix, line 231 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ lab4:
+ z->c = z->l - m5;
+ }
+ { int m7 = z->l - z->c; (void)m7; /* do, line 232 */
+ { int ret = r_vowel_suffix(z);
+ if (ret == 0) goto lab7; /* call vowel_suffix, line 232 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = z->l - m7;
+ }
+ z->c = z->lb;
+ { int c8 = z->c; /* do, line 234 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab8; /* call postlude, line 234 */
+ if (ret < 0) return ret;
+ }
+ lab8:
+ z->c = c8;
+ }
+ return 1;
+}
+
+extern struct SN_env * romanian_UTF_8_create_env(void) { return SN_create_env(0, 3, 1); }
+
+extern void romanian_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_romanian.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_romanian.h
new file mode 100644
index 0000000..d01e813
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_romanian.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * romanian_UTF_8_create_env(void);
+extern void romanian_UTF_8_close_env(struct SN_env * z);
+
+extern int romanian_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_russian.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_russian.c
new file mode 100644
index 0000000..6f0a964
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_russian.c
@@ -0,0 +1,694 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int russian_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_tidy_up(struct SN_env * z);
+static int r_derivational(struct SN_env * z);
+static int r_noun(struct SN_env * z);
+static int r_verb(struct SN_env * z);
+static int r_reflexive(struct SN_env * z);
+static int r_adjectival(struct SN_env * z);
+static int r_adjective(struct SN_env * z);
+static int r_perfective_gerund(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * russian_UTF_8_create_env(void);
+extern void russian_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[10] = { 0xD0, 0xB2, 0xD1, 0x88, 0xD0, 0xB8, 0xD1, 0x81, 0xD1, 0x8C };
+static const symbol s_0_1[12] = { 0xD1, 0x8B, 0xD0, 0xB2, 0xD1, 0x88, 0xD0, 0xB8, 0xD1, 0x81, 0xD1, 0x8C };
+static const symbol s_0_2[12] = { 0xD0, 0xB8, 0xD0, 0xB2, 0xD1, 0x88, 0xD0, 0xB8, 0xD1, 0x81, 0xD1, 0x8C };
+static const symbol s_0_3[2] = { 0xD0, 0xB2 };
+static const symbol s_0_4[4] = { 0xD1, 0x8B, 0xD0, 0xB2 };
+static const symbol s_0_5[4] = { 0xD0, 0xB8, 0xD0, 0xB2 };
+static const symbol s_0_6[6] = { 0xD0, 0xB2, 0xD1, 0x88, 0xD0, 0xB8 };
+static const symbol s_0_7[8] = { 0xD1, 0x8B, 0xD0, 0xB2, 0xD1, 0x88, 0xD0, 0xB8 };
+static const symbol s_0_8[8] = { 0xD0, 0xB8, 0xD0, 0xB2, 0xD1, 0x88, 0xD0, 0xB8 };
+
+static const struct among a_0[9] =
+{
+/* 0 */ { 10, s_0_0, -1, 1, 0},
+/* 1 */ { 12, s_0_1, 0, 2, 0},
+/* 2 */ { 12, s_0_2, 0, 2, 0},
+/* 3 */ { 2, s_0_3, -1, 1, 0},
+/* 4 */ { 4, s_0_4, 3, 2, 0},
+/* 5 */ { 4, s_0_5, 3, 2, 0},
+/* 6 */ { 6, s_0_6, -1, 1, 0},
+/* 7 */ { 8, s_0_7, 6, 2, 0},
+/* 8 */ { 8, s_0_8, 6, 2, 0}
+};
+
+static const symbol s_1_0[6] = { 0xD0, 0xB5, 0xD0, 0xBC, 0xD1, 0x83 };
+static const symbol s_1_1[6] = { 0xD0, 0xBE, 0xD0, 0xBC, 0xD1, 0x83 };
+static const symbol s_1_2[4] = { 0xD1, 0x8B, 0xD1, 0x85 };
+static const symbol s_1_3[4] = { 0xD0, 0xB8, 0xD1, 0x85 };
+static const symbol s_1_4[4] = { 0xD1, 0x83, 0xD1, 0x8E };
+static const symbol s_1_5[4] = { 0xD1, 0x8E, 0xD1, 0x8E };
+static const symbol s_1_6[4] = { 0xD0, 0xB5, 0xD1, 0x8E };
+static const symbol s_1_7[4] = { 0xD0, 0xBE, 0xD1, 0x8E };
+static const symbol s_1_8[4] = { 0xD1, 0x8F, 0xD1, 0x8F };
+static const symbol s_1_9[4] = { 0xD0, 0xB0, 0xD1, 0x8F };
+static const symbol s_1_10[4] = { 0xD1, 0x8B, 0xD0, 0xB5 };
+static const symbol s_1_11[4] = { 0xD0, 0xB5, 0xD0, 0xB5 };
+static const symbol s_1_12[4] = { 0xD0, 0xB8, 0xD0, 0xB5 };
+static const symbol s_1_13[4] = { 0xD0, 0xBE, 0xD0, 0xB5 };
+static const symbol s_1_14[6] = { 0xD1, 0x8B, 0xD0, 0xBC, 0xD0, 0xB8 };
+static const symbol s_1_15[6] = { 0xD0, 0xB8, 0xD0, 0xBC, 0xD0, 0xB8 };
+static const symbol s_1_16[4] = { 0xD1, 0x8B, 0xD0, 0xB9 };
+static const symbol s_1_17[4] = { 0xD0, 0xB5, 0xD0, 0xB9 };
+static const symbol s_1_18[4] = { 0xD0, 0xB8, 0xD0, 0xB9 };
+static const symbol s_1_19[4] = { 0xD0, 0xBE, 0xD0, 0xB9 };
+static const symbol s_1_20[4] = { 0xD1, 0x8B, 0xD0, 0xBC };
+static const symbol s_1_21[4] = { 0xD0, 0xB5, 0xD0, 0xBC };
+static const symbol s_1_22[4] = { 0xD0, 0xB8, 0xD0, 0xBC };
+static const symbol s_1_23[4] = { 0xD0, 0xBE, 0xD0, 0xBC };
+static const symbol s_1_24[6] = { 0xD0, 0xB5, 0xD0, 0xB3, 0xD0, 0xBE };
+static const symbol s_1_25[6] = { 0xD0, 0xBE, 0xD0, 0xB3, 0xD0, 0xBE };
+
+static const struct among a_1[26] =
+{
+/* 0 */ { 6, s_1_0, -1, 1, 0},
+/* 1 */ { 6, s_1_1, -1, 1, 0},
+/* 2 */ { 4, s_1_2, -1, 1, 0},
+/* 3 */ { 4, s_1_3, -1, 1, 0},
+/* 4 */ { 4, s_1_4, -1, 1, 0},
+/* 5 */ { 4, s_1_5, -1, 1, 0},
+/* 6 */ { 4, s_1_6, -1, 1, 0},
+/* 7 */ { 4, s_1_7, -1, 1, 0},
+/* 8 */ { 4, s_1_8, -1, 1, 0},
+/* 9 */ { 4, s_1_9, -1, 1, 0},
+/* 10 */ { 4, s_1_10, -1, 1, 0},
+/* 11 */ { 4, s_1_11, -1, 1, 0},
+/* 12 */ { 4, s_1_12, -1, 1, 0},
+/* 13 */ { 4, s_1_13, -1, 1, 0},
+/* 14 */ { 6, s_1_14, -1, 1, 0},
+/* 15 */ { 6, s_1_15, -1, 1, 0},
+/* 16 */ { 4, s_1_16, -1, 1, 0},
+/* 17 */ { 4, s_1_17, -1, 1, 0},
+/* 18 */ { 4, s_1_18, -1, 1, 0},
+/* 19 */ { 4, s_1_19, -1, 1, 0},
+/* 20 */ { 4, s_1_20, -1, 1, 0},
+/* 21 */ { 4, s_1_21, -1, 1, 0},
+/* 22 */ { 4, s_1_22, -1, 1, 0},
+/* 23 */ { 4, s_1_23, -1, 1, 0},
+/* 24 */ { 6, s_1_24, -1, 1, 0},
+/* 25 */ { 6, s_1_25, -1, 1, 0}
+};
+
+static const symbol s_2_0[4] = { 0xD0, 0xB2, 0xD1, 0x88 };
+static const symbol s_2_1[6] = { 0xD1, 0x8B, 0xD0, 0xB2, 0xD1, 0x88 };
+static const symbol s_2_2[6] = { 0xD0, 0xB8, 0xD0, 0xB2, 0xD1, 0x88 };
+static const symbol s_2_3[2] = { 0xD1, 0x89 };
+static const symbol s_2_4[4] = { 0xD1, 0x8E, 0xD1, 0x89 };
+static const symbol s_2_5[6] = { 0xD1, 0x83, 0xD1, 0x8E, 0xD1, 0x89 };
+static const symbol s_2_6[4] = { 0xD0, 0xB5, 0xD0, 0xBC };
+static const symbol s_2_7[4] = { 0xD0, 0xBD, 0xD0, 0xBD };
+
+static const struct among a_2[8] =
+{
+/* 0 */ { 4, s_2_0, -1, 1, 0},
+/* 1 */ { 6, s_2_1, 0, 2, 0},
+/* 2 */ { 6, s_2_2, 0, 2, 0},
+/* 3 */ { 2, s_2_3, -1, 1, 0},
+/* 4 */ { 4, s_2_4, 3, 1, 0},
+/* 5 */ { 6, s_2_5, 4, 2, 0},
+/* 6 */ { 4, s_2_6, -1, 1, 0},
+/* 7 */ { 4, s_2_7, -1, 1, 0}
+};
+
+static const symbol s_3_0[4] = { 0xD1, 0x81, 0xD1, 0x8C };
+static const symbol s_3_1[4] = { 0xD1, 0x81, 0xD1, 0x8F };
+
+static const struct among a_3[2] =
+{
+/* 0 */ { 4, s_3_0, -1, 1, 0},
+/* 1 */ { 4, s_3_1, -1, 1, 0}
+};
+
+static const symbol s_4_0[4] = { 0xD1, 0x8B, 0xD1, 0x82 };
+static const symbol s_4_1[4] = { 0xD1, 0x8E, 0xD1, 0x82 };
+static const symbol s_4_2[6] = { 0xD1, 0x83, 0xD1, 0x8E, 0xD1, 0x82 };
+static const symbol s_4_3[4] = { 0xD1, 0x8F, 0xD1, 0x82 };
+static const symbol s_4_4[4] = { 0xD0, 0xB5, 0xD1, 0x82 };
+static const symbol s_4_5[6] = { 0xD1, 0x83, 0xD0, 0xB5, 0xD1, 0x82 };
+static const symbol s_4_6[4] = { 0xD0, 0xB8, 0xD1, 0x82 };
+static const symbol s_4_7[4] = { 0xD0, 0xBD, 0xD1, 0x8B };
+static const symbol s_4_8[6] = { 0xD0, 0xB5, 0xD0, 0xBD, 0xD1, 0x8B };
+static const symbol s_4_9[4] = { 0xD1, 0x82, 0xD1, 0x8C };
+static const symbol s_4_10[6] = { 0xD1, 0x8B, 0xD1, 0x82, 0xD1, 0x8C };
+static const symbol s_4_11[6] = { 0xD0, 0xB8, 0xD1, 0x82, 0xD1, 0x8C };
+static const symbol s_4_12[6] = { 0xD0, 0xB5, 0xD1, 0x88, 0xD1, 0x8C };
+static const symbol s_4_13[6] = { 0xD0, 0xB8, 0xD1, 0x88, 0xD1, 0x8C };
+static const symbol s_4_14[2] = { 0xD1, 0x8E };
+static const symbol s_4_15[4] = { 0xD1, 0x83, 0xD1, 0x8E };
+static const symbol s_4_16[4] = { 0xD0, 0xBB, 0xD0, 0xB0 };
+static const symbol s_4_17[6] = { 0xD1, 0x8B, 0xD0, 0xBB, 0xD0, 0xB0 };
+static const symbol s_4_18[6] = { 0xD0, 0xB8, 0xD0, 0xBB, 0xD0, 0xB0 };
+static const symbol s_4_19[4] = { 0xD0, 0xBD, 0xD0, 0xB0 };
+static const symbol s_4_20[6] = { 0xD0, 0xB5, 0xD0, 0xBD, 0xD0, 0xB0 };
+static const symbol s_4_21[6] = { 0xD0, 0xB5, 0xD1, 0x82, 0xD0, 0xB5 };
+static const symbol s_4_22[6] = { 0xD0, 0xB8, 0xD1, 0x82, 0xD0, 0xB5 };
+static const symbol s_4_23[6] = { 0xD0, 0xB9, 0xD1, 0x82, 0xD0, 0xB5 };
+static const symbol s_4_24[8] = { 0xD1, 0x83, 0xD0, 0xB9, 0xD1, 0x82, 0xD0, 0xB5 };
+static const symbol s_4_25[8] = { 0xD0, 0xB5, 0xD0, 0xB9, 0xD1, 0x82, 0xD0, 0xB5 };
+static const symbol s_4_26[4] = { 0xD0, 0xBB, 0xD0, 0xB8 };
+static const symbol s_4_27[6] = { 0xD1, 0x8B, 0xD0, 0xBB, 0xD0, 0xB8 };
+static const symbol s_4_28[6] = { 0xD0, 0xB8, 0xD0, 0xBB, 0xD0, 0xB8 };
+static const symbol s_4_29[2] = { 0xD0, 0xB9 };
+static const symbol s_4_30[4] = { 0xD1, 0x83, 0xD0, 0xB9 };
+static const symbol s_4_31[4] = { 0xD0, 0xB5, 0xD0, 0xB9 };
+static const symbol s_4_32[2] = { 0xD0, 0xBB };
+static const symbol s_4_33[4] = { 0xD1, 0x8B, 0xD0, 0xBB };
+static const symbol s_4_34[4] = { 0xD0, 0xB8, 0xD0, 0xBB };
+static const symbol s_4_35[4] = { 0xD1, 0x8B, 0xD0, 0xBC };
+static const symbol s_4_36[4] = { 0xD0, 0xB5, 0xD0, 0xBC };
+static const symbol s_4_37[4] = { 0xD0, 0xB8, 0xD0, 0xBC };
+static const symbol s_4_38[2] = { 0xD0, 0xBD };
+static const symbol s_4_39[4] = { 0xD0, 0xB5, 0xD0, 0xBD };
+static const symbol s_4_40[4] = { 0xD0, 0xBB, 0xD0, 0xBE };
+static const symbol s_4_41[6] = { 0xD1, 0x8B, 0xD0, 0xBB, 0xD0, 0xBE };
+static const symbol s_4_42[6] = { 0xD0, 0xB8, 0xD0, 0xBB, 0xD0, 0xBE };
+static const symbol s_4_43[4] = { 0xD0, 0xBD, 0xD0, 0xBE };
+static const symbol s_4_44[6] = { 0xD0, 0xB5, 0xD0, 0xBD, 0xD0, 0xBE };
+static const symbol s_4_45[6] = { 0xD0, 0xBD, 0xD0, 0xBD, 0xD0, 0xBE };
+
+static const struct among a_4[46] =
+{
+/* 0 */ { 4, s_4_0, -1, 2, 0},
+/* 1 */ { 4, s_4_1, -1, 1, 0},
+/* 2 */ { 6, s_4_2, 1, 2, 0},
+/* 3 */ { 4, s_4_3, -1, 2, 0},
+/* 4 */ { 4, s_4_4, -1, 1, 0},
+/* 5 */ { 6, s_4_5, 4, 2, 0},
+/* 6 */ { 4, s_4_6, -1, 2, 0},
+/* 7 */ { 4, s_4_7, -1, 1, 0},
+/* 8 */ { 6, s_4_8, 7, 2, 0},
+/* 9 */ { 4, s_4_9, -1, 1, 0},
+/* 10 */ { 6, s_4_10, 9, 2, 0},
+/* 11 */ { 6, s_4_11, 9, 2, 0},
+/* 12 */ { 6, s_4_12, -1, 1, 0},
+/* 13 */ { 6, s_4_13, -1, 2, 0},
+/* 14 */ { 2, s_4_14, -1, 2, 0},
+/* 15 */ { 4, s_4_15, 14, 2, 0},
+/* 16 */ { 4, s_4_16, -1, 1, 0},
+/* 17 */ { 6, s_4_17, 16, 2, 0},
+/* 18 */ { 6, s_4_18, 16, 2, 0},
+/* 19 */ { 4, s_4_19, -1, 1, 0},
+/* 20 */ { 6, s_4_20, 19, 2, 0},
+/* 21 */ { 6, s_4_21, -1, 1, 0},
+/* 22 */ { 6, s_4_22, -1, 2, 0},
+/* 23 */ { 6, s_4_23, -1, 1, 0},
+/* 24 */ { 8, s_4_24, 23, 2, 0},
+/* 25 */ { 8, s_4_25, 23, 2, 0},
+/* 26 */ { 4, s_4_26, -1, 1, 0},
+/* 27 */ { 6, s_4_27, 26, 2, 0},
+/* 28 */ { 6, s_4_28, 26, 2, 0},
+/* 29 */ { 2, s_4_29, -1, 1, 0},
+/* 30 */ { 4, s_4_30, 29, 2, 0},
+/* 31 */ { 4, s_4_31, 29, 2, 0},
+/* 32 */ { 2, s_4_32, -1, 1, 0},
+/* 33 */ { 4, s_4_33, 32, 2, 0},
+/* 34 */ { 4, s_4_34, 32, 2, 0},
+/* 35 */ { 4, s_4_35, -1, 2, 0},
+/* 36 */ { 4, s_4_36, -1, 1, 0},
+/* 37 */ { 4, s_4_37, -1, 2, 0},
+/* 38 */ { 2, s_4_38, -1, 1, 0},
+/* 39 */ { 4, s_4_39, 38, 2, 0},
+/* 40 */ { 4, s_4_40, -1, 1, 0},
+/* 41 */ { 6, s_4_41, 40, 2, 0},
+/* 42 */ { 6, s_4_42, 40, 2, 0},
+/* 43 */ { 4, s_4_43, -1, 1, 0},
+/* 44 */ { 6, s_4_44, 43, 2, 0},
+/* 45 */ { 6, s_4_45, 43, 1, 0}
+};
+
+static const symbol s_5_0[2] = { 0xD1, 0x83 };
+static const symbol s_5_1[4] = { 0xD1, 0x8F, 0xD1, 0x85 };
+static const symbol s_5_2[6] = { 0xD0, 0xB8, 0xD1, 0x8F, 0xD1, 0x85 };
+static const symbol s_5_3[4] = { 0xD0, 0xB0, 0xD1, 0x85 };
+static const symbol s_5_4[2] = { 0xD1, 0x8B };
+static const symbol s_5_5[2] = { 0xD1, 0x8C };
+static const symbol s_5_6[2] = { 0xD1, 0x8E };
+static const symbol s_5_7[4] = { 0xD1, 0x8C, 0xD1, 0x8E };
+static const symbol s_5_8[4] = { 0xD0, 0xB8, 0xD1, 0x8E };
+static const symbol s_5_9[2] = { 0xD1, 0x8F };
+static const symbol s_5_10[4] = { 0xD1, 0x8C, 0xD1, 0x8F };
+static const symbol s_5_11[4] = { 0xD0, 0xB8, 0xD1, 0x8F };
+static const symbol s_5_12[2] = { 0xD0, 0xB0 };
+static const symbol s_5_13[4] = { 0xD0, 0xB5, 0xD0, 0xB2 };
+static const symbol s_5_14[4] = { 0xD0, 0xBE, 0xD0, 0xB2 };
+static const symbol s_5_15[2] = { 0xD0, 0xB5 };
+static const symbol s_5_16[4] = { 0xD1, 0x8C, 0xD0, 0xB5 };
+static const symbol s_5_17[4] = { 0xD0, 0xB8, 0xD0, 0xB5 };
+static const symbol s_5_18[2] = { 0xD0, 0xB8 };
+static const symbol s_5_19[4] = { 0xD0, 0xB5, 0xD0, 0xB8 };
+static const symbol s_5_20[4] = { 0xD0, 0xB8, 0xD0, 0xB8 };
+static const symbol s_5_21[6] = { 0xD1, 0x8F, 0xD0, 0xBC, 0xD0, 0xB8 };
+static const symbol s_5_22[8] = { 0xD0, 0xB8, 0xD1, 0x8F, 0xD0, 0xBC, 0xD0, 0xB8 };
+static const symbol s_5_23[6] = { 0xD0, 0xB0, 0xD0, 0xBC, 0xD0, 0xB8 };
+static const symbol s_5_24[2] = { 0xD0, 0xB9 };
+static const symbol s_5_25[4] = { 0xD0, 0xB5, 0xD0, 0xB9 };
+static const symbol s_5_26[6] = { 0xD0, 0xB8, 0xD0, 0xB5, 0xD0, 0xB9 };
+static const symbol s_5_27[4] = { 0xD0, 0xB8, 0xD0, 0xB9 };
+static const symbol s_5_28[4] = { 0xD0, 0xBE, 0xD0, 0xB9 };
+static const symbol s_5_29[4] = { 0xD1, 0x8F, 0xD0, 0xBC };
+static const symbol s_5_30[6] = { 0xD0, 0xB8, 0xD1, 0x8F, 0xD0, 0xBC };
+static const symbol s_5_31[4] = { 0xD0, 0xB0, 0xD0, 0xBC };
+static const symbol s_5_32[4] = { 0xD0, 0xB5, 0xD0, 0xBC };
+static const symbol s_5_33[6] = { 0xD0, 0xB8, 0xD0, 0xB5, 0xD0, 0xBC };
+static const symbol s_5_34[4] = { 0xD0, 0xBE, 0xD0, 0xBC };
+static const symbol s_5_35[2] = { 0xD0, 0xBE };
+
+static const struct among a_5[36] =
+{
+/* 0 */ { 2, s_5_0, -1, 1, 0},
+/* 1 */ { 4, s_5_1, -1, 1, 0},
+/* 2 */ { 6, s_5_2, 1, 1, 0},
+/* 3 */ { 4, s_5_3, -1, 1, 0},
+/* 4 */ { 2, s_5_4, -1, 1, 0},
+/* 5 */ { 2, s_5_5, -1, 1, 0},
+/* 6 */ { 2, s_5_6, -1, 1, 0},
+/* 7 */ { 4, s_5_7, 6, 1, 0},
+/* 8 */ { 4, s_5_8, 6, 1, 0},
+/* 9 */ { 2, s_5_9, -1, 1, 0},
+/* 10 */ { 4, s_5_10, 9, 1, 0},
+/* 11 */ { 4, s_5_11, 9, 1, 0},
+/* 12 */ { 2, s_5_12, -1, 1, 0},
+/* 13 */ { 4, s_5_13, -1, 1, 0},
+/* 14 */ { 4, s_5_14, -1, 1, 0},
+/* 15 */ { 2, s_5_15, -1, 1, 0},
+/* 16 */ { 4, s_5_16, 15, 1, 0},
+/* 17 */ { 4, s_5_17, 15, 1, 0},
+/* 18 */ { 2, s_5_18, -1, 1, 0},
+/* 19 */ { 4, s_5_19, 18, 1, 0},
+/* 20 */ { 4, s_5_20, 18, 1, 0},
+/* 21 */ { 6, s_5_21, 18, 1, 0},
+/* 22 */ { 8, s_5_22, 21, 1, 0},
+/* 23 */ { 6, s_5_23, 18, 1, 0},
+/* 24 */ { 2, s_5_24, -1, 1, 0},
+/* 25 */ { 4, s_5_25, 24, 1, 0},
+/* 26 */ { 6, s_5_26, 25, 1, 0},
+/* 27 */ { 4, s_5_27, 24, 1, 0},
+/* 28 */ { 4, s_5_28, 24, 1, 0},
+/* 29 */ { 4, s_5_29, -1, 1, 0},
+/* 30 */ { 6, s_5_30, 29, 1, 0},
+/* 31 */ { 4, s_5_31, -1, 1, 0},
+/* 32 */ { 4, s_5_32, -1, 1, 0},
+/* 33 */ { 6, s_5_33, 32, 1, 0},
+/* 34 */ { 4, s_5_34, -1, 1, 0},
+/* 35 */ { 2, s_5_35, -1, 1, 0}
+};
+
+static const symbol s_6_0[6] = { 0xD0, 0xBE, 0xD1, 0x81, 0xD1, 0x82 };
+static const symbol s_6_1[8] = { 0xD0, 0xBE, 0xD1, 0x81, 0xD1, 0x82, 0xD1, 0x8C };
+
+static const struct among a_6[2] =
+{
+/* 0 */ { 6, s_6_0, -1, 1, 0},
+/* 1 */ { 8, s_6_1, -1, 1, 0}
+};
+
+static const symbol s_7_0[6] = { 0xD0, 0xB5, 0xD0, 0xB9, 0xD1, 0x88 };
+static const symbol s_7_1[2] = { 0xD1, 0x8C };
+static const symbol s_7_2[8] = { 0xD0, 0xB5, 0xD0, 0xB9, 0xD1, 0x88, 0xD0, 0xB5 };
+static const symbol s_7_3[2] = { 0xD0, 0xBD };
+
+static const struct among a_7[4] =
+{
+/* 0 */ { 6, s_7_0, -1, 1, 0},
+/* 1 */ { 2, s_7_1, -1, 3, 0},
+/* 2 */ { 8, s_7_2, -1, 1, 0},
+/* 3 */ { 2, s_7_3, -1, 2, 0}
+};
+
+static const unsigned char g_v[] = { 33, 65, 8, 232 };
+
+static const symbol s_0[] = { 0xD0, 0xB0 };
+static const symbol s_1[] = { 0xD1, 0x8F };
+static const symbol s_2[] = { 0xD0, 0xB0 };
+static const symbol s_3[] = { 0xD1, 0x8F };
+static const symbol s_4[] = { 0xD0, 0xB0 };
+static const symbol s_5[] = { 0xD1, 0x8F };
+static const symbol s_6[] = { 0xD0, 0xBD };
+static const symbol s_7[] = { 0xD0, 0xBD };
+static const symbol s_8[] = { 0xD0, 0xBD };
+static const symbol s_9[] = { 0xD0, 0xB8 };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ { int c1 = z->c; /* do, line 61 */
+ { /* gopast */ /* grouping v, line 62 */
+ int ret = out_grouping_U(z, g_v, 1072, 1103, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark pV, line 62 */
+ { /* gopast */ /* non v, line 62 */
+ int ret = in_grouping_U(z, g_v, 1072, 1103, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ { /* gopast */ /* grouping v, line 63 */
+ int ret = out_grouping_U(z, g_v, 1072, 1103, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 63 */
+ int ret = in_grouping_U(z, g_v, 1072, 1103, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p2, line 63 */
+ lab0:
+ z->c = c1;
+ }
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_perfective_gerund(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 72 */
+ among_var = find_among_b(z, a_0, 9); /* substring, line 72 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 72 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 76 */
+ if (!(eq_s_b(z, 2, s_0))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_1))) return 0;
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 76 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 83 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_adjective(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 88 */
+ among_var = find_among_b(z, a_1, 26); /* substring, line 88 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 88 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 97 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_adjectival(struct SN_env * z) {
+ int among_var;
+ { int ret = r_adjective(z);
+ if (ret == 0) return 0; /* call adjective, line 102 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 109 */
+ z->ket = z->c; /* [, line 110 */
+ among_var = find_among_b(z, a_2, 8); /* substring, line 110 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 110 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab0; }
+ case 1:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 115 */
+ if (!(eq_s_b(z, 2, s_2))) goto lab2;
+ goto lab1;
+ lab2:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_3))) { z->c = z->l - m_keep; goto lab0; }
+ }
+ lab1:
+ { int ret = slice_del(z); /* delete, line 115 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 122 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab0:
+ ;
+ }
+ return 1;
+}
+
+static int r_reflexive(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 129 */
+ if (z->c - 3 <= z->lb || (z->p[z->c - 1] != 140 && z->p[z->c - 1] != 143)) return 0;
+ among_var = find_among_b(z, a_3, 2); /* substring, line 129 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 129 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 132 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_verb(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 137 */
+ among_var = find_among_b(z, a_4, 46); /* substring, line 137 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 137 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m1 = z->l - z->c; (void)m1; /* or, line 143 */
+ if (!(eq_s_b(z, 2, s_4))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_5))) return 0;
+ }
+ lab0:
+ { int ret = slice_del(z); /* delete, line 143 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 151 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_noun(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 160 */
+ among_var = find_among_b(z, a_5, 36); /* substring, line 160 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 160 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 167 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_derivational(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 176 */
+ if (z->c - 5 <= z->lb || (z->p[z->c - 1] != 130 && z->p[z->c - 1] != 140)) return 0;
+ among_var = find_among_b(z, a_6, 2); /* substring, line 176 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 176 */
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 176 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 179 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_tidy_up(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 184 */
+ among_var = find_among_b(z, a_7, 4); /* substring, line 184 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 184 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 188 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 189 */
+ if (!(eq_s_b(z, 2, s_6))) return 0;
+ z->bra = z->c; /* ], line 189 */
+ if (!(eq_s_b(z, 2, s_7))) return 0;
+ { int ret = slice_del(z); /* delete, line 189 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ if (!(eq_s_b(z, 2, s_8))) return 0;
+ { int ret = slice_del(z); /* delete, line 192 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_del(z); /* delete, line 194 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+extern int russian_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 201 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 201 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 202 */
+
+ { int mlimit; /* setlimit, line 202 */
+ int m2 = z->l - z->c; (void)m2;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 202 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m2;
+ { int m3 = z->l - z->c; (void)m3; /* do, line 203 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 204 */
+ { int ret = r_perfective_gerund(z);
+ if (ret == 0) goto lab3; /* call perfective_gerund, line 204 */
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab3:
+ z->c = z->l - m4;
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 205 */
+ { int ret = r_reflexive(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab4; } /* call reflexive, line 205 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ ;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* or, line 206 */
+ { int ret = r_adjectival(z);
+ if (ret == 0) goto lab6; /* call adjectival, line 206 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = z->l - m5;
+ { int ret = r_verb(z);
+ if (ret == 0) goto lab7; /* call verb, line 206 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab7:
+ z->c = z->l - m5;
+ { int ret = r_noun(z);
+ if (ret == 0) goto lab1; /* call noun, line 206 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ ;
+ }
+ lab2:
+ lab1:
+ z->c = z->l - m3;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 209 */
+ z->ket = z->c; /* [, line 209 */
+ if (!(eq_s_b(z, 2, s_9))) { z->c = z->l - m_keep; goto lab8; }
+ z->bra = z->c; /* ], line 209 */
+ { int ret = slice_del(z); /* delete, line 209 */
+ if (ret < 0) return ret;
+ }
+ lab8:
+ ;
+ }
+ { int m6 = z->l - z->c; (void)m6; /* do, line 212 */
+ { int ret = r_derivational(z);
+ if (ret == 0) goto lab9; /* call derivational, line 212 */
+ if (ret < 0) return ret;
+ }
+ lab9:
+ z->c = z->l - m6;
+ }
+ { int m7 = z->l - z->c; (void)m7; /* do, line 213 */
+ { int ret = r_tidy_up(z);
+ if (ret == 0) goto lab10; /* call tidy_up, line 213 */
+ if (ret < 0) return ret;
+ }
+ lab10:
+ z->c = z->l - m7;
+ }
+ z->lb = mlimit;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern struct SN_env * russian_UTF_8_create_env(void) { return SN_create_env(0, 2, 0); }
+
+extern void russian_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_russian.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_russian.h
new file mode 100644
index 0000000..4ef774d
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_russian.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * russian_UTF_8_create_env(void);
+extern void russian_UTF_8_close_env(struct SN_env * z);
+
+extern int russian_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_spanish.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_spanish.c
new file mode 100644
index 0000000..9550d67
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_spanish.c
@@ -0,0 +1,1097 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int spanish_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_residual_suffix(struct SN_env * z);
+static int r_verb_suffix(struct SN_env * z);
+static int r_y_verb_suffix(struct SN_env * z);
+static int r_standard_suffix(struct SN_env * z);
+static int r_attached_pronoun(struct SN_env * z);
+static int r_R2(struct SN_env * z);
+static int r_R1(struct SN_env * z);
+static int r_RV(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * spanish_UTF_8_create_env(void);
+extern void spanish_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_1[2] = { 0xC3, 0xA1 };
+static const symbol s_0_2[2] = { 0xC3, 0xA9 };
+static const symbol s_0_3[2] = { 0xC3, 0xAD };
+static const symbol s_0_4[2] = { 0xC3, 0xB3 };
+static const symbol s_0_5[2] = { 0xC3, 0xBA };
+
+static const struct among a_0[6] =
+{
+/* 0 */ { 0, 0, -1, 6, 0},
+/* 1 */ { 2, s_0_1, 0, 1, 0},
+/* 2 */ { 2, s_0_2, 0, 2, 0},
+/* 3 */ { 2, s_0_3, 0, 3, 0},
+/* 4 */ { 2, s_0_4, 0, 4, 0},
+/* 5 */ { 2, s_0_5, 0, 5, 0}
+};
+
+static const symbol s_1_0[2] = { 'l', 'a' };
+static const symbol s_1_1[4] = { 's', 'e', 'l', 'a' };
+static const symbol s_1_2[2] = { 'l', 'e' };
+static const symbol s_1_3[2] = { 'm', 'e' };
+static const symbol s_1_4[2] = { 's', 'e' };
+static const symbol s_1_5[2] = { 'l', 'o' };
+static const symbol s_1_6[4] = { 's', 'e', 'l', 'o' };
+static const symbol s_1_7[3] = { 'l', 'a', 's' };
+static const symbol s_1_8[5] = { 's', 'e', 'l', 'a', 's' };
+static const symbol s_1_9[3] = { 'l', 'e', 's' };
+static const symbol s_1_10[3] = { 'l', 'o', 's' };
+static const symbol s_1_11[5] = { 's', 'e', 'l', 'o', 's' };
+static const symbol s_1_12[3] = { 'n', 'o', 's' };
+
+static const struct among a_1[13] =
+{
+/* 0 */ { 2, s_1_0, -1, -1, 0},
+/* 1 */ { 4, s_1_1, 0, -1, 0},
+/* 2 */ { 2, s_1_2, -1, -1, 0},
+/* 3 */ { 2, s_1_3, -1, -1, 0},
+/* 4 */ { 2, s_1_4, -1, -1, 0},
+/* 5 */ { 2, s_1_5, -1, -1, 0},
+/* 6 */ { 4, s_1_6, 5, -1, 0},
+/* 7 */ { 3, s_1_7, -1, -1, 0},
+/* 8 */ { 5, s_1_8, 7, -1, 0},
+/* 9 */ { 3, s_1_9, -1, -1, 0},
+/* 10 */ { 3, s_1_10, -1, -1, 0},
+/* 11 */ { 5, s_1_11, 10, -1, 0},
+/* 12 */ { 3, s_1_12, -1, -1, 0}
+};
+
+static const symbol s_2_0[4] = { 'a', 'n', 'd', 'o' };
+static const symbol s_2_1[5] = { 'i', 'e', 'n', 'd', 'o' };
+static const symbol s_2_2[5] = { 'y', 'e', 'n', 'd', 'o' };
+static const symbol s_2_3[5] = { 0xC3, 0xA1, 'n', 'd', 'o' };
+static const symbol s_2_4[6] = { 'i', 0xC3, 0xA9, 'n', 'd', 'o' };
+static const symbol s_2_5[2] = { 'a', 'r' };
+static const symbol s_2_6[2] = { 'e', 'r' };
+static const symbol s_2_7[2] = { 'i', 'r' };
+static const symbol s_2_8[3] = { 0xC3, 0xA1, 'r' };
+static const symbol s_2_9[3] = { 0xC3, 0xA9, 'r' };
+static const symbol s_2_10[3] = { 0xC3, 0xAD, 'r' };
+
+static const struct among a_2[11] =
+{
+/* 0 */ { 4, s_2_0, -1, 6, 0},
+/* 1 */ { 5, s_2_1, -1, 6, 0},
+/* 2 */ { 5, s_2_2, -1, 7, 0},
+/* 3 */ { 5, s_2_3, -1, 2, 0},
+/* 4 */ { 6, s_2_4, -1, 1, 0},
+/* 5 */ { 2, s_2_5, -1, 6, 0},
+/* 6 */ { 2, s_2_6, -1, 6, 0},
+/* 7 */ { 2, s_2_7, -1, 6, 0},
+/* 8 */ { 3, s_2_8, -1, 3, 0},
+/* 9 */ { 3, s_2_9, -1, 4, 0},
+/* 10 */ { 3, s_2_10, -1, 5, 0}
+};
+
+static const symbol s_3_0[2] = { 'i', 'c' };
+static const symbol s_3_1[2] = { 'a', 'd' };
+static const symbol s_3_2[2] = { 'o', 's' };
+static const symbol s_3_3[2] = { 'i', 'v' };
+
+static const struct among a_3[4] =
+{
+/* 0 */ { 2, s_3_0, -1, -1, 0},
+/* 1 */ { 2, s_3_1, -1, -1, 0},
+/* 2 */ { 2, s_3_2, -1, -1, 0},
+/* 3 */ { 2, s_3_3, -1, 1, 0}
+};
+
+static const symbol s_4_0[4] = { 'a', 'b', 'l', 'e' };
+static const symbol s_4_1[4] = { 'i', 'b', 'l', 'e' };
+static const symbol s_4_2[4] = { 'a', 'n', 't', 'e' };
+
+static const struct among a_4[3] =
+{
+/* 0 */ { 4, s_4_0, -1, 1, 0},
+/* 1 */ { 4, s_4_1, -1, 1, 0},
+/* 2 */ { 4, s_4_2, -1, 1, 0}
+};
+
+static const symbol s_5_0[2] = { 'i', 'c' };
+static const symbol s_5_1[4] = { 'a', 'b', 'i', 'l' };
+static const symbol s_5_2[2] = { 'i', 'v' };
+
+static const struct among a_5[3] =
+{
+/* 0 */ { 2, s_5_0, -1, 1, 0},
+/* 1 */ { 4, s_5_1, -1, 1, 0},
+/* 2 */ { 2, s_5_2, -1, 1, 0}
+};
+
+static const symbol s_6_0[3] = { 'i', 'c', 'a' };
+static const symbol s_6_1[5] = { 'a', 'n', 'c', 'i', 'a' };
+static const symbol s_6_2[5] = { 'e', 'n', 'c', 'i', 'a' };
+static const symbol s_6_3[5] = { 'a', 'd', 'o', 'r', 'a' };
+static const symbol s_6_4[3] = { 'o', 's', 'a' };
+static const symbol s_6_5[4] = { 'i', 's', 't', 'a' };
+static const symbol s_6_6[3] = { 'i', 'v', 'a' };
+static const symbol s_6_7[4] = { 'a', 'n', 'z', 'a' };
+static const symbol s_6_8[6] = { 'l', 'o', 'g', 0xC3, 0xAD, 'a' };
+static const symbol s_6_9[4] = { 'i', 'd', 'a', 'd' };
+static const symbol s_6_10[4] = { 'a', 'b', 'l', 'e' };
+static const symbol s_6_11[4] = { 'i', 'b', 'l', 'e' };
+static const symbol s_6_12[4] = { 'a', 'n', 't', 'e' };
+static const symbol s_6_13[5] = { 'm', 'e', 'n', 't', 'e' };
+static const symbol s_6_14[6] = { 'a', 'm', 'e', 'n', 't', 'e' };
+static const symbol s_6_15[6] = { 'a', 'c', 'i', 0xC3, 0xB3, 'n' };
+static const symbol s_6_16[6] = { 'u', 'c', 'i', 0xC3, 0xB3, 'n' };
+static const symbol s_6_17[3] = { 'i', 'c', 'o' };
+static const symbol s_6_18[4] = { 'i', 's', 'm', 'o' };
+static const symbol s_6_19[3] = { 'o', 's', 'o' };
+static const symbol s_6_20[7] = { 'a', 'm', 'i', 'e', 'n', 't', 'o' };
+static const symbol s_6_21[7] = { 'i', 'm', 'i', 'e', 'n', 't', 'o' };
+static const symbol s_6_22[3] = { 'i', 'v', 'o' };
+static const symbol s_6_23[4] = { 'a', 'd', 'o', 'r' };
+static const symbol s_6_24[4] = { 'i', 'c', 'a', 's' };
+static const symbol s_6_25[6] = { 'a', 'n', 'c', 'i', 'a', 's' };
+static const symbol s_6_26[6] = { 'e', 'n', 'c', 'i', 'a', 's' };
+static const symbol s_6_27[6] = { 'a', 'd', 'o', 'r', 'a', 's' };
+static const symbol s_6_28[4] = { 'o', 's', 'a', 's' };
+static const symbol s_6_29[5] = { 'i', 's', 't', 'a', 's' };
+static const symbol s_6_30[4] = { 'i', 'v', 'a', 's' };
+static const symbol s_6_31[5] = { 'a', 'n', 'z', 'a', 's' };
+static const symbol s_6_32[7] = { 'l', 'o', 'g', 0xC3, 0xAD, 'a', 's' };
+static const symbol s_6_33[6] = { 'i', 'd', 'a', 'd', 'e', 's' };
+static const symbol s_6_34[5] = { 'a', 'b', 'l', 'e', 's' };
+static const symbol s_6_35[5] = { 'i', 'b', 'l', 'e', 's' };
+static const symbol s_6_36[7] = { 'a', 'c', 'i', 'o', 'n', 'e', 's' };
+static const symbol s_6_37[7] = { 'u', 'c', 'i', 'o', 'n', 'e', 's' };
+static const symbol s_6_38[6] = { 'a', 'd', 'o', 'r', 'e', 's' };
+static const symbol s_6_39[5] = { 'a', 'n', 't', 'e', 's' };
+static const symbol s_6_40[4] = { 'i', 'c', 'o', 's' };
+static const symbol s_6_41[5] = { 'i', 's', 'm', 'o', 's' };
+static const symbol s_6_42[4] = { 'o', 's', 'o', 's' };
+static const symbol s_6_43[8] = { 'a', 'm', 'i', 'e', 'n', 't', 'o', 's' };
+static const symbol s_6_44[8] = { 'i', 'm', 'i', 'e', 'n', 't', 'o', 's' };
+static const symbol s_6_45[4] = { 'i', 'v', 'o', 's' };
+
+static const struct among a_6[46] =
+{
+/* 0 */ { 3, s_6_0, -1, 1, 0},
+/* 1 */ { 5, s_6_1, -1, 2, 0},
+/* 2 */ { 5, s_6_2, -1, 5, 0},
+/* 3 */ { 5, s_6_3, -1, 2, 0},
+/* 4 */ { 3, s_6_4, -1, 1, 0},
+/* 5 */ { 4, s_6_5, -1, 1, 0},
+/* 6 */ { 3, s_6_6, -1, 9, 0},
+/* 7 */ { 4, s_6_7, -1, 1, 0},
+/* 8 */ { 6, s_6_8, -1, 3, 0},
+/* 9 */ { 4, s_6_9, -1, 8, 0},
+/* 10 */ { 4, s_6_10, -1, 1, 0},
+/* 11 */ { 4, s_6_11, -1, 1, 0},
+/* 12 */ { 4, s_6_12, -1, 2, 0},
+/* 13 */ { 5, s_6_13, -1, 7, 0},
+/* 14 */ { 6, s_6_14, 13, 6, 0},
+/* 15 */ { 6, s_6_15, -1, 2, 0},
+/* 16 */ { 6, s_6_16, -1, 4, 0},
+/* 17 */ { 3, s_6_17, -1, 1, 0},
+/* 18 */ { 4, s_6_18, -1, 1, 0},
+/* 19 */ { 3, s_6_19, -1, 1, 0},
+/* 20 */ { 7, s_6_20, -1, 1, 0},
+/* 21 */ { 7, s_6_21, -1, 1, 0},
+/* 22 */ { 3, s_6_22, -1, 9, 0},
+/* 23 */ { 4, s_6_23, -1, 2, 0},
+/* 24 */ { 4, s_6_24, -1, 1, 0},
+/* 25 */ { 6, s_6_25, -1, 2, 0},
+/* 26 */ { 6, s_6_26, -1, 5, 0},
+/* 27 */ { 6, s_6_27, -1, 2, 0},
+/* 28 */ { 4, s_6_28, -1, 1, 0},
+/* 29 */ { 5, s_6_29, -1, 1, 0},
+/* 30 */ { 4, s_6_30, -1, 9, 0},
+/* 31 */ { 5, s_6_31, -1, 1, 0},
+/* 32 */ { 7, s_6_32, -1, 3, 0},
+/* 33 */ { 6, s_6_33, -1, 8, 0},
+/* 34 */ { 5, s_6_34, -1, 1, 0},
+/* 35 */ { 5, s_6_35, -1, 1, 0},
+/* 36 */ { 7, s_6_36, -1, 2, 0},
+/* 37 */ { 7, s_6_37, -1, 4, 0},
+/* 38 */ { 6, s_6_38, -1, 2, 0},
+/* 39 */ { 5, s_6_39, -1, 2, 0},
+/* 40 */ { 4, s_6_40, -1, 1, 0},
+/* 41 */ { 5, s_6_41, -1, 1, 0},
+/* 42 */ { 4, s_6_42, -1, 1, 0},
+/* 43 */ { 8, s_6_43, -1, 1, 0},
+/* 44 */ { 8, s_6_44, -1, 1, 0},
+/* 45 */ { 4, s_6_45, -1, 9, 0}
+};
+
+static const symbol s_7_0[2] = { 'y', 'a' };
+static const symbol s_7_1[2] = { 'y', 'e' };
+static const symbol s_7_2[3] = { 'y', 'a', 'n' };
+static const symbol s_7_3[3] = { 'y', 'e', 'n' };
+static const symbol s_7_4[5] = { 'y', 'e', 'r', 'o', 'n' };
+static const symbol s_7_5[5] = { 'y', 'e', 'n', 'd', 'o' };
+static const symbol s_7_6[2] = { 'y', 'o' };
+static const symbol s_7_7[3] = { 'y', 'a', 's' };
+static const symbol s_7_8[3] = { 'y', 'e', 's' };
+static const symbol s_7_9[4] = { 'y', 'a', 'i', 's' };
+static const symbol s_7_10[5] = { 'y', 'a', 'm', 'o', 's' };
+static const symbol s_7_11[3] = { 'y', 0xC3, 0xB3 };
+
+static const struct among a_7[12] =
+{
+/* 0 */ { 2, s_7_0, -1, 1, 0},
+/* 1 */ { 2, s_7_1, -1, 1, 0},
+/* 2 */ { 3, s_7_2, -1, 1, 0},
+/* 3 */ { 3, s_7_3, -1, 1, 0},
+/* 4 */ { 5, s_7_4, -1, 1, 0},
+/* 5 */ { 5, s_7_5, -1, 1, 0},
+/* 6 */ { 2, s_7_6, -1, 1, 0},
+/* 7 */ { 3, s_7_7, -1, 1, 0},
+/* 8 */ { 3, s_7_8, -1, 1, 0},
+/* 9 */ { 4, s_7_9, -1, 1, 0},
+/* 10 */ { 5, s_7_10, -1, 1, 0},
+/* 11 */ { 3, s_7_11, -1, 1, 0}
+};
+
+static const symbol s_8_0[3] = { 'a', 'b', 'a' };
+static const symbol s_8_1[3] = { 'a', 'd', 'a' };
+static const symbol s_8_2[3] = { 'i', 'd', 'a' };
+static const symbol s_8_3[3] = { 'a', 'r', 'a' };
+static const symbol s_8_4[4] = { 'i', 'e', 'r', 'a' };
+static const symbol s_8_5[3] = { 0xC3, 0xAD, 'a' };
+static const symbol s_8_6[5] = { 'a', 'r', 0xC3, 0xAD, 'a' };
+static const symbol s_8_7[5] = { 'e', 'r', 0xC3, 0xAD, 'a' };
+static const symbol s_8_8[5] = { 'i', 'r', 0xC3, 0xAD, 'a' };
+static const symbol s_8_9[2] = { 'a', 'd' };
+static const symbol s_8_10[2] = { 'e', 'd' };
+static const symbol s_8_11[2] = { 'i', 'd' };
+static const symbol s_8_12[3] = { 'a', 's', 'e' };
+static const symbol s_8_13[4] = { 'i', 'e', 's', 'e' };
+static const symbol s_8_14[4] = { 'a', 's', 't', 'e' };
+static const symbol s_8_15[4] = { 'i', 's', 't', 'e' };
+static const symbol s_8_16[2] = { 'a', 'n' };
+static const symbol s_8_17[4] = { 'a', 'b', 'a', 'n' };
+static const symbol s_8_18[4] = { 'a', 'r', 'a', 'n' };
+static const symbol s_8_19[5] = { 'i', 'e', 'r', 'a', 'n' };
+static const symbol s_8_20[4] = { 0xC3, 0xAD, 'a', 'n' };
+static const symbol s_8_21[6] = { 'a', 'r', 0xC3, 0xAD, 'a', 'n' };
+static const symbol s_8_22[6] = { 'e', 'r', 0xC3, 0xAD, 'a', 'n' };
+static const symbol s_8_23[6] = { 'i', 'r', 0xC3, 0xAD, 'a', 'n' };
+static const symbol s_8_24[2] = { 'e', 'n' };
+static const symbol s_8_25[4] = { 'a', 's', 'e', 'n' };
+static const symbol s_8_26[5] = { 'i', 'e', 's', 'e', 'n' };
+static const symbol s_8_27[4] = { 'a', 'r', 'o', 'n' };
+static const symbol s_8_28[5] = { 'i', 'e', 'r', 'o', 'n' };
+static const symbol s_8_29[5] = { 'a', 'r', 0xC3, 0xA1, 'n' };
+static const symbol s_8_30[5] = { 'e', 'r', 0xC3, 0xA1, 'n' };
+static const symbol s_8_31[5] = { 'i', 'r', 0xC3, 0xA1, 'n' };
+static const symbol s_8_32[3] = { 'a', 'd', 'o' };
+static const symbol s_8_33[3] = { 'i', 'd', 'o' };
+static const symbol s_8_34[4] = { 'a', 'n', 'd', 'o' };
+static const symbol s_8_35[5] = { 'i', 'e', 'n', 'd', 'o' };
+static const symbol s_8_36[2] = { 'a', 'r' };
+static const symbol s_8_37[2] = { 'e', 'r' };
+static const symbol s_8_38[2] = { 'i', 'r' };
+static const symbol s_8_39[2] = { 'a', 's' };
+static const symbol s_8_40[4] = { 'a', 'b', 'a', 's' };
+static const symbol s_8_41[4] = { 'a', 'd', 'a', 's' };
+static const symbol s_8_42[4] = { 'i', 'd', 'a', 's' };
+static const symbol s_8_43[4] = { 'a', 'r', 'a', 's' };
+static const symbol s_8_44[5] = { 'i', 'e', 'r', 'a', 's' };
+static const symbol s_8_45[4] = { 0xC3, 0xAD, 'a', 's' };
+static const symbol s_8_46[6] = { 'a', 'r', 0xC3, 0xAD, 'a', 's' };
+static const symbol s_8_47[6] = { 'e', 'r', 0xC3, 0xAD, 'a', 's' };
+static const symbol s_8_48[6] = { 'i', 'r', 0xC3, 0xAD, 'a', 's' };
+static const symbol s_8_49[2] = { 'e', 's' };
+static const symbol s_8_50[4] = { 'a', 's', 'e', 's' };
+static const symbol s_8_51[5] = { 'i', 'e', 's', 'e', 's' };
+static const symbol s_8_52[5] = { 'a', 'b', 'a', 'i', 's' };
+static const symbol s_8_53[5] = { 'a', 'r', 'a', 'i', 's' };
+static const symbol s_8_54[6] = { 'i', 'e', 'r', 'a', 'i', 's' };
+static const symbol s_8_55[5] = { 0xC3, 0xAD, 'a', 'i', 's' };
+static const symbol s_8_56[7] = { 'a', 'r', 0xC3, 0xAD, 'a', 'i', 's' };
+static const symbol s_8_57[7] = { 'e', 'r', 0xC3, 0xAD, 'a', 'i', 's' };
+static const symbol s_8_58[7] = { 'i', 'r', 0xC3, 0xAD, 'a', 'i', 's' };
+static const symbol s_8_59[5] = { 'a', 's', 'e', 'i', 's' };
+static const symbol s_8_60[6] = { 'i', 'e', 's', 'e', 'i', 's' };
+static const symbol s_8_61[6] = { 'a', 's', 't', 'e', 'i', 's' };
+static const symbol s_8_62[6] = { 'i', 's', 't', 'e', 'i', 's' };
+static const symbol s_8_63[4] = { 0xC3, 0xA1, 'i', 's' };
+static const symbol s_8_64[4] = { 0xC3, 0xA9, 'i', 's' };
+static const symbol s_8_65[6] = { 'a', 'r', 0xC3, 0xA9, 'i', 's' };
+static const symbol s_8_66[6] = { 'e', 'r', 0xC3, 0xA9, 'i', 's' };
+static const symbol s_8_67[6] = { 'i', 'r', 0xC3, 0xA9, 'i', 's' };
+static const symbol s_8_68[4] = { 'a', 'd', 'o', 's' };
+static const symbol s_8_69[4] = { 'i', 'd', 'o', 's' };
+static const symbol s_8_70[4] = { 'a', 'm', 'o', 's' };
+static const symbol s_8_71[7] = { 0xC3, 0xA1, 'b', 'a', 'm', 'o', 's' };
+static const symbol s_8_72[7] = { 0xC3, 0xA1, 'r', 'a', 'm', 'o', 's' };
+static const symbol s_8_73[8] = { 'i', 0xC3, 0xA9, 'r', 'a', 'm', 'o', 's' };
+static const symbol s_8_74[6] = { 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_8_75[8] = { 'a', 'r', 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_8_76[8] = { 'e', 'r', 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_8_77[8] = { 'i', 'r', 0xC3, 0xAD, 'a', 'm', 'o', 's' };
+static const symbol s_8_78[4] = { 'e', 'm', 'o', 's' };
+static const symbol s_8_79[6] = { 'a', 'r', 'e', 'm', 'o', 's' };
+static const symbol s_8_80[6] = { 'e', 'r', 'e', 'm', 'o', 's' };
+static const symbol s_8_81[6] = { 'i', 'r', 'e', 'm', 'o', 's' };
+static const symbol s_8_82[7] = { 0xC3, 0xA1, 's', 'e', 'm', 'o', 's' };
+static const symbol s_8_83[8] = { 'i', 0xC3, 0xA9, 's', 'e', 'm', 'o', 's' };
+static const symbol s_8_84[4] = { 'i', 'm', 'o', 's' };
+static const symbol s_8_85[5] = { 'a', 'r', 0xC3, 0xA1, 's' };
+static const symbol s_8_86[5] = { 'e', 'r', 0xC3, 0xA1, 's' };
+static const symbol s_8_87[5] = { 'i', 'r', 0xC3, 0xA1, 's' };
+static const symbol s_8_88[3] = { 0xC3, 0xAD, 's' };
+static const symbol s_8_89[4] = { 'a', 'r', 0xC3, 0xA1 };
+static const symbol s_8_90[4] = { 'e', 'r', 0xC3, 0xA1 };
+static const symbol s_8_91[4] = { 'i', 'r', 0xC3, 0xA1 };
+static const symbol s_8_92[4] = { 'a', 'r', 0xC3, 0xA9 };
+static const symbol s_8_93[4] = { 'e', 'r', 0xC3, 0xA9 };
+static const symbol s_8_94[4] = { 'i', 'r', 0xC3, 0xA9 };
+static const symbol s_8_95[3] = { 'i', 0xC3, 0xB3 };
+
+static const struct among a_8[96] =
+{
+/* 0 */ { 3, s_8_0, -1, 2, 0},
+/* 1 */ { 3, s_8_1, -1, 2, 0},
+/* 2 */ { 3, s_8_2, -1, 2, 0},
+/* 3 */ { 3, s_8_3, -1, 2, 0},
+/* 4 */ { 4, s_8_4, -1, 2, 0},
+/* 5 */ { 3, s_8_5, -1, 2, 0},
+/* 6 */ { 5, s_8_6, 5, 2, 0},
+/* 7 */ { 5, s_8_7, 5, 2, 0},
+/* 8 */ { 5, s_8_8, 5, 2, 0},
+/* 9 */ { 2, s_8_9, -1, 2, 0},
+/* 10 */ { 2, s_8_10, -1, 2, 0},
+/* 11 */ { 2, s_8_11, -1, 2, 0},
+/* 12 */ { 3, s_8_12, -1, 2, 0},
+/* 13 */ { 4, s_8_13, -1, 2, 0},
+/* 14 */ { 4, s_8_14, -1, 2, 0},
+/* 15 */ { 4, s_8_15, -1, 2, 0},
+/* 16 */ { 2, s_8_16, -1, 2, 0},
+/* 17 */ { 4, s_8_17, 16, 2, 0},
+/* 18 */ { 4, s_8_18, 16, 2, 0},
+/* 19 */ { 5, s_8_19, 16, 2, 0},
+/* 20 */ { 4, s_8_20, 16, 2, 0},
+/* 21 */ { 6, s_8_21, 20, 2, 0},
+/* 22 */ { 6, s_8_22, 20, 2, 0},
+/* 23 */ { 6, s_8_23, 20, 2, 0},
+/* 24 */ { 2, s_8_24, -1, 1, 0},
+/* 25 */ { 4, s_8_25, 24, 2, 0},
+/* 26 */ { 5, s_8_26, 24, 2, 0},
+/* 27 */ { 4, s_8_27, -1, 2, 0},
+/* 28 */ { 5, s_8_28, -1, 2, 0},
+/* 29 */ { 5, s_8_29, -1, 2, 0},
+/* 30 */ { 5, s_8_30, -1, 2, 0},
+/* 31 */ { 5, s_8_31, -1, 2, 0},
+/* 32 */ { 3, s_8_32, -1, 2, 0},
+/* 33 */ { 3, s_8_33, -1, 2, 0},
+/* 34 */ { 4, s_8_34, -1, 2, 0},
+/* 35 */ { 5, s_8_35, -1, 2, 0},
+/* 36 */ { 2, s_8_36, -1, 2, 0},
+/* 37 */ { 2, s_8_37, -1, 2, 0},
+/* 38 */ { 2, s_8_38, -1, 2, 0},
+/* 39 */ { 2, s_8_39, -1, 2, 0},
+/* 40 */ { 4, s_8_40, 39, 2, 0},
+/* 41 */ { 4, s_8_41, 39, 2, 0},
+/* 42 */ { 4, s_8_42, 39, 2, 0},
+/* 43 */ { 4, s_8_43, 39, 2, 0},
+/* 44 */ { 5, s_8_44, 39, 2, 0},
+/* 45 */ { 4, s_8_45, 39, 2, 0},
+/* 46 */ { 6, s_8_46, 45, 2, 0},
+/* 47 */ { 6, s_8_47, 45, 2, 0},
+/* 48 */ { 6, s_8_48, 45, 2, 0},
+/* 49 */ { 2, s_8_49, -1, 1, 0},
+/* 50 */ { 4, s_8_50, 49, 2, 0},
+/* 51 */ { 5, s_8_51, 49, 2, 0},
+/* 52 */ { 5, s_8_52, -1, 2, 0},
+/* 53 */ { 5, s_8_53, -1, 2, 0},
+/* 54 */ { 6, s_8_54, -1, 2, 0},
+/* 55 */ { 5, s_8_55, -1, 2, 0},
+/* 56 */ { 7, s_8_56, 55, 2, 0},
+/* 57 */ { 7, s_8_57, 55, 2, 0},
+/* 58 */ { 7, s_8_58, 55, 2, 0},
+/* 59 */ { 5, s_8_59, -1, 2, 0},
+/* 60 */ { 6, s_8_60, -1, 2, 0},
+/* 61 */ { 6, s_8_61, -1, 2, 0},
+/* 62 */ { 6, s_8_62, -1, 2, 0},
+/* 63 */ { 4, s_8_63, -1, 2, 0},
+/* 64 */ { 4, s_8_64, -1, 1, 0},
+/* 65 */ { 6, s_8_65, 64, 2, 0},
+/* 66 */ { 6, s_8_66, 64, 2, 0},
+/* 67 */ { 6, s_8_67, 64, 2, 0},
+/* 68 */ { 4, s_8_68, -1, 2, 0},
+/* 69 */ { 4, s_8_69, -1, 2, 0},
+/* 70 */ { 4, s_8_70, -1, 2, 0},
+/* 71 */ { 7, s_8_71, 70, 2, 0},
+/* 72 */ { 7, s_8_72, 70, 2, 0},
+/* 73 */ { 8, s_8_73, 70, 2, 0},
+/* 74 */ { 6, s_8_74, 70, 2, 0},
+/* 75 */ { 8, s_8_75, 74, 2, 0},
+/* 76 */ { 8, s_8_76, 74, 2, 0},
+/* 77 */ { 8, s_8_77, 74, 2, 0},
+/* 78 */ { 4, s_8_78, -1, 1, 0},
+/* 79 */ { 6, s_8_79, 78, 2, 0},
+/* 80 */ { 6, s_8_80, 78, 2, 0},
+/* 81 */ { 6, s_8_81, 78, 2, 0},
+/* 82 */ { 7, s_8_82, 78, 2, 0},
+/* 83 */ { 8, s_8_83, 78, 2, 0},
+/* 84 */ { 4, s_8_84, -1, 2, 0},
+/* 85 */ { 5, s_8_85, -1, 2, 0},
+/* 86 */ { 5, s_8_86, -1, 2, 0},
+/* 87 */ { 5, s_8_87, -1, 2, 0},
+/* 88 */ { 3, s_8_88, -1, 2, 0},
+/* 89 */ { 4, s_8_89, -1, 2, 0},
+/* 90 */ { 4, s_8_90, -1, 2, 0},
+/* 91 */ { 4, s_8_91, -1, 2, 0},
+/* 92 */ { 4, s_8_92, -1, 2, 0},
+/* 93 */ { 4, s_8_93, -1, 2, 0},
+/* 94 */ { 4, s_8_94, -1, 2, 0},
+/* 95 */ { 3, s_8_95, -1, 2, 0}
+};
+
+static const symbol s_9_0[1] = { 'a' };
+static const symbol s_9_1[1] = { 'e' };
+static const symbol s_9_2[1] = { 'o' };
+static const symbol s_9_3[2] = { 'o', 's' };
+static const symbol s_9_4[2] = { 0xC3, 0xA1 };
+static const symbol s_9_5[2] = { 0xC3, 0xA9 };
+static const symbol s_9_6[2] = { 0xC3, 0xAD };
+static const symbol s_9_7[2] = { 0xC3, 0xB3 };
+
+static const struct among a_9[8] =
+{
+/* 0 */ { 1, s_9_0, -1, 1, 0},
+/* 1 */ { 1, s_9_1, -1, 2, 0},
+/* 2 */ { 1, s_9_2, -1, 1, 0},
+/* 3 */ { 2, s_9_3, -1, 1, 0},
+/* 4 */ { 2, s_9_4, -1, 1, 0},
+/* 5 */ { 2, s_9_5, -1, 2, 0},
+/* 6 */ { 2, s_9_6, -1, 1, 0},
+/* 7 */ { 2, s_9_7, -1, 1, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 17, 4, 10 };
+
+static const symbol s_0[] = { 'a' };
+static const symbol s_1[] = { 'e' };
+static const symbol s_2[] = { 'i' };
+static const symbol s_3[] = { 'o' };
+static const symbol s_4[] = { 'u' };
+static const symbol s_5[] = { 'i', 'e', 'n', 'd', 'o' };
+static const symbol s_6[] = { 'a', 'n', 'd', 'o' };
+static const symbol s_7[] = { 'a', 'r' };
+static const symbol s_8[] = { 'e', 'r' };
+static const symbol s_9[] = { 'i', 'r' };
+static const symbol s_10[] = { 'u' };
+static const symbol s_11[] = { 'i', 'c' };
+static const symbol s_12[] = { 'l', 'o', 'g' };
+static const symbol s_13[] = { 'u' };
+static const symbol s_14[] = { 'e', 'n', 't', 'e' };
+static const symbol s_15[] = { 'a', 't' };
+static const symbol s_16[] = { 'a', 't' };
+static const symbol s_17[] = { 'u' };
+static const symbol s_18[] = { 'u' };
+static const symbol s_19[] = { 'g' };
+static const symbol s_20[] = { 'u' };
+static const symbol s_21[] = { 'g' };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ z->I[1] = z->l;
+ z->I[2] = z->l;
+ { int c1 = z->c; /* do, line 37 */
+ { int c2 = z->c; /* or, line 39 */
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab2;
+ { int c3 = z->c; /* or, line 38 */
+ if (out_grouping_U(z, g_v, 97, 252, 0)) goto lab4;
+ { /* gopast */ /* grouping v, line 38 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab4;
+ z->c += ret;
+ }
+ goto lab3;
+ lab4:
+ z->c = c3;
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab2;
+ { /* gopast */ /* non v, line 38 */
+ int ret = in_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab2;
+ z->c += ret;
+ }
+ }
+ lab3:
+ goto lab1;
+ lab2:
+ z->c = c2;
+ if (out_grouping_U(z, g_v, 97, 252, 0)) goto lab0;
+ { int c4 = z->c; /* or, line 40 */
+ if (out_grouping_U(z, g_v, 97, 252, 0)) goto lab6;
+ { /* gopast */ /* grouping v, line 40 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab6;
+ z->c += ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = c4;
+ if (in_grouping_U(z, g_v, 97, 252, 0)) goto lab0;
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 40 */
+ }
+ }
+ lab5:
+ ;
+ }
+ lab1:
+ z->I[0] = z->c; /* setmark pV, line 41 */
+ lab0:
+ z->c = c1;
+ }
+ { int c5 = z->c; /* do, line 43 */
+ { /* gopast */ /* grouping v, line 44 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 44 */
+ int ret = in_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[1] = z->c; /* setmark p1, line 44 */
+ { /* gopast */ /* grouping v, line 45 */
+ int ret = out_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ { /* gopast */ /* non v, line 45 */
+ int ret = in_grouping_U(z, g_v, 97, 252, 1);
+ if (ret < 0) goto lab7;
+ z->c += ret;
+ }
+ z->I[2] = z->c; /* setmark p2, line 45 */
+ lab7:
+ z->c = c5;
+ }
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ int among_var;
+ while(1) { /* repeat, line 49 */
+ int c1 = z->c;
+ z->bra = z->c; /* [, line 50 */
+ if (z->c + 1 >= z->l || z->p[z->c + 1] >> 5 != 5 || !((67641858 >> (z->p[z->c + 1] & 0x1f)) & 1)) among_var = 6; else
+ among_var = find_among(z, a_0, 6); /* substring, line 50 */
+ if (!(among_var)) goto lab0;
+ z->ket = z->c; /* ], line 50 */
+ switch(among_var) {
+ case 0: goto lab0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_0); /* <-, line 51 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 1, s_1); /* <-, line 52 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_2); /* <-, line 53 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 1, s_3); /* <-, line 54 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = slice_from_s(z, 1, s_4); /* <-, line 55 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab0;
+ z->c = ret; /* next, line 57 */
+ }
+ break;
+ }
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ return 1;
+}
+
+static int r_RV(struct SN_env * z) {
+ if (!(z->I[0] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R1(struct SN_env * z) {
+ if (!(z->I[1] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_R2(struct SN_env * z) {
+ if (!(z->I[2] <= z->c)) return 0;
+ return 1;
+}
+
+static int r_attached_pronoun(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 68 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((557090 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_1, 13))) return 0; /* substring, line 68 */
+ z->bra = z->c; /* ], line 68 */
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 111 && z->p[z->c - 1] != 114)) return 0;
+ among_var = find_among_b(z, a_2, 11); /* substring, line 72 */
+ if (!(among_var)) return 0;
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 72 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ z->bra = z->c; /* ], line 73 */
+ { int ret = slice_from_s(z, 5, s_5); /* <-, line 73 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ z->bra = z->c; /* ], line 74 */
+ { int ret = slice_from_s(z, 4, s_6); /* <-, line 74 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ z->bra = z->c; /* ], line 75 */
+ { int ret = slice_from_s(z, 2, s_7); /* <-, line 75 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ z->bra = z->c; /* ], line 76 */
+ { int ret = slice_from_s(z, 2, s_8); /* <-, line 76 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ z->bra = z->c; /* ], line 77 */
+ { int ret = slice_from_s(z, 2, s_9); /* <-, line 77 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = slice_del(z); /* delete, line 81 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 7:
+ if (!(eq_s_b(z, 1, s_10))) return 0;
+ { int ret = slice_del(z); /* delete, line 82 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_standard_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 87 */
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((835634 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ among_var = find_among_b(z, a_6, 46); /* substring, line 87 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 87 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 99 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 99 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 105 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 105 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 106 */
+ z->ket = z->c; /* [, line 106 */
+ if (!(eq_s_b(z, 2, s_11))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 106 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call R2, line 106 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 106 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ ;
+ }
+ break;
+ case 3:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 111 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 3, s_12); /* <-, line 111 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 115 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 1, s_13); /* <-, line 115 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 5:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 119 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_from_s(z, 4, s_14); /* <-, line 119 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 6:
+ { int ret = r_R1(z);
+ if (ret == 0) return 0; /* call R1, line 123 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 123 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 124 */
+ z->ket = z->c; /* [, line 125 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4718616 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab1; }
+ among_var = find_among_b(z, a_3, 4); /* substring, line 125 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 125 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab1; } /* call R2, line 125 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 125 */
+ if (ret < 0) return ret;
+ }
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab1; }
+ case 1:
+ z->ket = z->c; /* [, line 126 */
+ if (!(eq_s_b(z, 2, s_15))) { z->c = z->l - m_keep; goto lab1; }
+ z->bra = z->c; /* ], line 126 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab1; } /* call R2, line 126 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 126 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab1:
+ ;
+ }
+ break;
+ case 7:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 135 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 135 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 136 */
+ z->ket = z->c; /* [, line 137 */
+ if (z->c - 3 <= z->lb || z->p[z->c - 1] != 101) { z->c = z->l - m_keep; goto lab2; }
+ among_var = find_among_b(z, a_4, 3); /* substring, line 137 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab2; }
+ z->bra = z->c; /* ], line 137 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab2; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab2; } /* call R2, line 140 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 140 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab2:
+ ;
+ }
+ break;
+ case 8:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 147 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 147 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 148 */
+ z->ket = z->c; /* [, line 149 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((4198408 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->c = z->l - m_keep; goto lab3; }
+ among_var = find_among_b(z, a_5, 3); /* substring, line 149 */
+ if (!(among_var)) { z->c = z->l - m_keep; goto lab3; }
+ z->bra = z->c; /* ], line 149 */
+ switch(among_var) {
+ case 0: { z->c = z->l - m_keep; goto lab3; }
+ case 1:
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab3; } /* call R2, line 152 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 152 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ lab3:
+ ;
+ }
+ break;
+ case 9:
+ { int ret = r_R2(z);
+ if (ret == 0) return 0; /* call R2, line 159 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 159 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 160 */
+ z->ket = z->c; /* [, line 161 */
+ if (!(eq_s_b(z, 2, s_16))) { z->c = z->l - m_keep; goto lab4; }
+ z->bra = z->c; /* ], line 161 */
+ { int ret = r_R2(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab4; } /* call R2, line 161 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 161 */
+ if (ret < 0) return ret;
+ }
+ lab4:
+ ;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_y_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 168 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 168 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 168 */
+ among_var = find_among_b(z, a_7, 12); /* substring, line 168 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 168 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ if (!(eq_s_b(z, 1, s_17))) return 0;
+ { int ret = slice_del(z); /* delete, line 171 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_verb_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 176 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 176 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 176 */
+ among_var = find_among_b(z, a_8, 96); /* substring, line 176 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 176 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 179 */
+ if (!(eq_s_b(z, 1, s_18))) { z->c = z->l - m_keep; goto lab0; }
+ { int m_test = z->l - z->c; /* test, line 179 */
+ if (!(eq_s_b(z, 1, s_19))) { z->c = z->l - m_keep; goto lab0; }
+ z->c = z->l - m_test;
+ }
+ lab0:
+ ;
+ }
+ z->bra = z->c; /* ], line 179 */
+ { int ret = slice_del(z); /* delete, line 179 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_del(z); /* delete, line 200 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_residual_suffix(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 205 */
+ among_var = find_among_b(z, a_9, 8); /* substring, line 205 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 205 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 208 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 208 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = r_RV(z);
+ if (ret == 0) return 0; /* call RV, line 210 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 210 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 210 */
+ z->ket = z->c; /* [, line 210 */
+ if (!(eq_s_b(z, 1, s_20))) { z->c = z->l - m_keep; goto lab0; }
+ z->bra = z->c; /* ], line 210 */
+ { int m_test = z->l - z->c; /* test, line 210 */
+ if (!(eq_s_b(z, 1, s_21))) { z->c = z->l - m_keep; goto lab0; }
+ z->c = z->l - m_test;
+ }
+ { int ret = r_RV(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab0; } /* call RV, line 210 */
+ if (ret < 0) return ret;
+ }
+ { int ret = slice_del(z); /* delete, line 210 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ ;
+ }
+ break;
+ }
+ return 1;
+}
+
+extern int spanish_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 216 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 216 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 217 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 218 */
+ { int ret = r_attached_pronoun(z);
+ if (ret == 0) goto lab1; /* call attached_pronoun, line 218 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 219 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 219 */
+ { int ret = r_standard_suffix(z);
+ if (ret == 0) goto lab4; /* call standard_suffix, line 219 */
+ if (ret < 0) return ret;
+ }
+ goto lab3;
+ lab4:
+ z->c = z->l - m4;
+ { int ret = r_y_verb_suffix(z);
+ if (ret == 0) goto lab5; /* call y_verb_suffix, line 220 */
+ if (ret < 0) return ret;
+ }
+ goto lab3;
+ lab5:
+ z->c = z->l - m4;
+ { int ret = r_verb_suffix(z);
+ if (ret == 0) goto lab2; /* call verb_suffix, line 221 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab3:
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* do, line 223 */
+ { int ret = r_residual_suffix(z);
+ if (ret == 0) goto lab6; /* call residual_suffix, line 223 */
+ if (ret < 0) return ret;
+ }
+ lab6:
+ z->c = z->l - m5;
+ }
+ z->c = z->lb;
+ { int c6 = z->c; /* do, line 225 */
+ { int ret = r_postlude(z);
+ if (ret == 0) goto lab7; /* call postlude, line 225 */
+ if (ret < 0) return ret;
+ }
+ lab7:
+ z->c = c6;
+ }
+ return 1;
+}
+
+extern struct SN_env * spanish_UTF_8_create_env(void) { return SN_create_env(0, 3, 0); }
+
+extern void spanish_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_spanish.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_spanish.h
new file mode 100644
index 0000000..10572ec
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_spanish.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * spanish_UTF_8_create_env(void);
+extern void spanish_UTF_8_close_env(struct SN_env * z);
+
+extern int spanish_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_swedish.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_swedish.c
new file mode 100644
index 0000000..21a2353
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_swedish.c
@@ -0,0 +1,309 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int swedish_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_other_suffix(struct SN_env * z);
+static int r_consonant_pair(struct SN_env * z);
+static int r_main_suffix(struct SN_env * z);
+static int r_mark_regions(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * swedish_UTF_8_create_env(void);
+extern void swedish_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[1] = { 'a' };
+static const symbol s_0_1[4] = { 'a', 'r', 'n', 'a' };
+static const symbol s_0_2[4] = { 'e', 'r', 'n', 'a' };
+static const symbol s_0_3[7] = { 'h', 'e', 't', 'e', 'r', 'n', 'a' };
+static const symbol s_0_4[4] = { 'o', 'r', 'n', 'a' };
+static const symbol s_0_5[2] = { 'a', 'd' };
+static const symbol s_0_6[1] = { 'e' };
+static const symbol s_0_7[3] = { 'a', 'd', 'e' };
+static const symbol s_0_8[4] = { 'a', 'n', 'd', 'e' };
+static const symbol s_0_9[4] = { 'a', 'r', 'n', 'e' };
+static const symbol s_0_10[3] = { 'a', 'r', 'e' };
+static const symbol s_0_11[4] = { 'a', 's', 't', 'e' };
+static const symbol s_0_12[2] = { 'e', 'n' };
+static const symbol s_0_13[5] = { 'a', 'n', 'd', 'e', 'n' };
+static const symbol s_0_14[4] = { 'a', 'r', 'e', 'n' };
+static const symbol s_0_15[5] = { 'h', 'e', 't', 'e', 'n' };
+static const symbol s_0_16[3] = { 'e', 'r', 'n' };
+static const symbol s_0_17[2] = { 'a', 'r' };
+static const symbol s_0_18[2] = { 'e', 'r' };
+static const symbol s_0_19[5] = { 'h', 'e', 't', 'e', 'r' };
+static const symbol s_0_20[2] = { 'o', 'r' };
+static const symbol s_0_21[1] = { 's' };
+static const symbol s_0_22[2] = { 'a', 's' };
+static const symbol s_0_23[5] = { 'a', 'r', 'n', 'a', 's' };
+static const symbol s_0_24[5] = { 'e', 'r', 'n', 'a', 's' };
+static const symbol s_0_25[5] = { 'o', 'r', 'n', 'a', 's' };
+static const symbol s_0_26[2] = { 'e', 's' };
+static const symbol s_0_27[4] = { 'a', 'd', 'e', 's' };
+static const symbol s_0_28[5] = { 'a', 'n', 'd', 'e', 's' };
+static const symbol s_0_29[3] = { 'e', 'n', 's' };
+static const symbol s_0_30[5] = { 'a', 'r', 'e', 'n', 's' };
+static const symbol s_0_31[6] = { 'h', 'e', 't', 'e', 'n', 's' };
+static const symbol s_0_32[4] = { 'e', 'r', 'n', 's' };
+static const symbol s_0_33[2] = { 'a', 't' };
+static const symbol s_0_34[5] = { 'a', 'n', 'd', 'e', 't' };
+static const symbol s_0_35[3] = { 'h', 'e', 't' };
+static const symbol s_0_36[3] = { 'a', 's', 't' };
+
+static const struct among a_0[37] =
+{
+/* 0 */ { 1, s_0_0, -1, 1, 0},
+/* 1 */ { 4, s_0_1, 0, 1, 0},
+/* 2 */ { 4, s_0_2, 0, 1, 0},
+/* 3 */ { 7, s_0_3, 2, 1, 0},
+/* 4 */ { 4, s_0_4, 0, 1, 0},
+/* 5 */ { 2, s_0_5, -1, 1, 0},
+/* 6 */ { 1, s_0_6, -1, 1, 0},
+/* 7 */ { 3, s_0_7, 6, 1, 0},
+/* 8 */ { 4, s_0_8, 6, 1, 0},
+/* 9 */ { 4, s_0_9, 6, 1, 0},
+/* 10 */ { 3, s_0_10, 6, 1, 0},
+/* 11 */ { 4, s_0_11, 6, 1, 0},
+/* 12 */ { 2, s_0_12, -1, 1, 0},
+/* 13 */ { 5, s_0_13, 12, 1, 0},
+/* 14 */ { 4, s_0_14, 12, 1, 0},
+/* 15 */ { 5, s_0_15, 12, 1, 0},
+/* 16 */ { 3, s_0_16, -1, 1, 0},
+/* 17 */ { 2, s_0_17, -1, 1, 0},
+/* 18 */ { 2, s_0_18, -1, 1, 0},
+/* 19 */ { 5, s_0_19, 18, 1, 0},
+/* 20 */ { 2, s_0_20, -1, 1, 0},
+/* 21 */ { 1, s_0_21, -1, 2, 0},
+/* 22 */ { 2, s_0_22, 21, 1, 0},
+/* 23 */ { 5, s_0_23, 22, 1, 0},
+/* 24 */ { 5, s_0_24, 22, 1, 0},
+/* 25 */ { 5, s_0_25, 22, 1, 0},
+/* 26 */ { 2, s_0_26, 21, 1, 0},
+/* 27 */ { 4, s_0_27, 26, 1, 0},
+/* 28 */ { 5, s_0_28, 26, 1, 0},
+/* 29 */ { 3, s_0_29, 21, 1, 0},
+/* 30 */ { 5, s_0_30, 29, 1, 0},
+/* 31 */ { 6, s_0_31, 29, 1, 0},
+/* 32 */ { 4, s_0_32, 21, 1, 0},
+/* 33 */ { 2, s_0_33, -1, 1, 0},
+/* 34 */ { 5, s_0_34, -1, 1, 0},
+/* 35 */ { 3, s_0_35, -1, 1, 0},
+/* 36 */ { 3, s_0_36, -1, 1, 0}
+};
+
+static const symbol s_1_0[2] = { 'd', 'd' };
+static const symbol s_1_1[2] = { 'g', 'd' };
+static const symbol s_1_2[2] = { 'n', 'n' };
+static const symbol s_1_3[2] = { 'd', 't' };
+static const symbol s_1_4[2] = { 'g', 't' };
+static const symbol s_1_5[2] = { 'k', 't' };
+static const symbol s_1_6[2] = { 't', 't' };
+
+static const struct among a_1[7] =
+{
+/* 0 */ { 2, s_1_0, -1, -1, 0},
+/* 1 */ { 2, s_1_1, -1, -1, 0},
+/* 2 */ { 2, s_1_2, -1, -1, 0},
+/* 3 */ { 2, s_1_3, -1, -1, 0},
+/* 4 */ { 2, s_1_4, -1, -1, 0},
+/* 5 */ { 2, s_1_5, -1, -1, 0},
+/* 6 */ { 2, s_1_6, -1, -1, 0}
+};
+
+static const symbol s_2_0[2] = { 'i', 'g' };
+static const symbol s_2_1[3] = { 'l', 'i', 'g' };
+static const symbol s_2_2[3] = { 'e', 'l', 's' };
+static const symbol s_2_3[5] = { 'f', 'u', 'l', 'l', 't' };
+static const symbol s_2_4[5] = { 'l', 0xC3, 0xB6, 's', 't' };
+
+static const struct among a_2[5] =
+{
+/* 0 */ { 2, s_2_0, -1, 1, 0},
+/* 1 */ { 3, s_2_1, 0, 1, 0},
+/* 2 */ { 3, s_2_2, -1, 1, 0},
+/* 3 */ { 5, s_2_3, -1, 3, 0},
+/* 4 */ { 5, s_2_4, -1, 2, 0}
+};
+
+static const unsigned char g_v[] = { 17, 65, 16, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 32 };
+
+static const unsigned char g_s_ending[] = { 119, 127, 149 };
+
+static const symbol s_0[] = { 'l', 0xC3, 0xB6, 's' };
+static const symbol s_1[] = { 'f', 'u', 'l', 'l' };
+
+static int r_mark_regions(struct SN_env * z) {
+ z->I[0] = z->l;
+ { int c_test = z->c; /* test, line 29 */
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, + 3);
+ if (ret < 0) return 0;
+ z->c = ret; /* hop, line 29 */
+ }
+ z->I[1] = z->c; /* setmark x, line 29 */
+ z->c = c_test;
+ }
+ if (out_grouping_U(z, g_v, 97, 246, 1) < 0) return 0; /* goto */ /* grouping v, line 30 */
+ { /* gopast */ /* non v, line 30 */
+ int ret = in_grouping_U(z, g_v, 97, 246, 1);
+ if (ret < 0) return 0;
+ z->c += ret;
+ }
+ z->I[0] = z->c; /* setmark p1, line 30 */
+ /* try, line 31 */
+ if (!(z->I[0] < z->I[1])) goto lab0;
+ z->I[0] = z->I[1];
+lab0:
+ return 1;
+}
+
+static int r_main_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 37 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 37 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 37 */
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1851442 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_0, 37); /* substring, line 37 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 37 */
+ z->lb = mlimit;
+ }
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_del(z); /* delete, line 44 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ if (in_grouping_b_U(z, g_s_ending, 98, 121, 0)) return 0;
+ { int ret = slice_del(z); /* delete, line 46 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_consonant_pair(struct SN_env * z) {
+ { int mlimit; /* setlimit, line 50 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 50 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* and, line 52 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1064976 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ if (!(find_among_b(z, a_1, 7))) { z->lb = mlimit; return 0; } /* among, line 51 */
+ z->c = z->l - m2;
+ z->ket = z->c; /* [, line 52 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) { z->lb = mlimit; return 0; }
+ z->c = ret; /* next, line 52 */
+ }
+ z->bra = z->c; /* ], line 52 */
+ { int ret = slice_del(z); /* delete, line 52 */
+ if (ret < 0) return ret;
+ }
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+static int r_other_suffix(struct SN_env * z) {
+ int among_var;
+ { int mlimit; /* setlimit, line 55 */
+ int m1 = z->l - z->c; (void)m1;
+ if (z->c < z->I[0]) return 0;
+ z->c = z->I[0]; /* tomark, line 55 */
+ mlimit = z->lb; z->lb = z->c;
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 56 */
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((1572992 >> (z->p[z->c - 1] & 0x1f)) & 1)) { z->lb = mlimit; return 0; }
+ among_var = find_among_b(z, a_2, 5); /* substring, line 56 */
+ if (!(among_var)) { z->lb = mlimit; return 0; }
+ z->bra = z->c; /* ], line 56 */
+ switch(among_var) {
+ case 0: { z->lb = mlimit; return 0; }
+ case 1:
+ { int ret = slice_del(z); /* delete, line 57 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 4, s_0); /* <-, line 58 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 4, s_1); /* <-, line 59 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ z->lb = mlimit;
+ }
+ return 1;
+}
+
+extern int swedish_UTF_8_stem(struct SN_env * z) {
+ { int c1 = z->c; /* do, line 66 */
+ { int ret = r_mark_regions(z);
+ if (ret == 0) goto lab0; /* call mark_regions, line 66 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 67 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 68 */
+ { int ret = r_main_suffix(z);
+ if (ret == 0) goto lab1; /* call main_suffix, line 68 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 69 */
+ { int ret = r_consonant_pair(z);
+ if (ret == 0) goto lab2; /* call consonant_pair, line 69 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ { int m4 = z->l - z->c; (void)m4; /* do, line 70 */
+ { int ret = r_other_suffix(z);
+ if (ret == 0) goto lab3; /* call other_suffix, line 70 */
+ if (ret < 0) return ret;
+ }
+ lab3:
+ z->c = z->l - m4;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern struct SN_env * swedish_UTF_8_create_env(void) { return SN_create_env(0, 2, 0); }
+
+extern void swedish_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_swedish.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_swedish.h
new file mode 100644
index 0000000..1444ebb
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_swedish.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * swedish_UTF_8_create_env(void);
+extern void swedish_UTF_8_close_env(struct SN_env * z);
+
+extern int swedish_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_turkish.c b/modules/analysis/snowstem/source/src_c/stem_UTF_8_turkish.c
new file mode 100644
index 0000000..ae3cc76
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_turkish.c
@@ -0,0 +1,2205 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#include "../runtime/header.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+extern int turkish_UTF_8_stem(struct SN_env * z);
+#ifdef __cplusplus
+}
+#endif
+static int r_stem_suffix_chain_before_ki(struct SN_env * z);
+static int r_stem_noun_suffixes(struct SN_env * z);
+static int r_stem_nominal_verb_suffixes(struct SN_env * z);
+static int r_postlude(struct SN_env * z);
+static int r_post_process_last_consonants(struct SN_env * z);
+static int r_more_than_one_syllable_word(struct SN_env * z);
+static int r_mark_suffix_with_optional_s_consonant(struct SN_env * z);
+static int r_mark_suffix_with_optional_n_consonant(struct SN_env * z);
+static int r_mark_suffix_with_optional_U_vowel(struct SN_env * z);
+static int r_mark_suffix_with_optional_y_consonant(struct SN_env * z);
+static int r_mark_ysA(struct SN_env * z);
+static int r_mark_ymUs_(struct SN_env * z);
+static int r_mark_yken(struct SN_env * z);
+static int r_mark_yDU(struct SN_env * z);
+static int r_mark_yUz(struct SN_env * z);
+static int r_mark_yUm(struct SN_env * z);
+static int r_mark_yU(struct SN_env * z);
+static int r_mark_ylA(struct SN_env * z);
+static int r_mark_yA(struct SN_env * z);
+static int r_mark_possessives(struct SN_env * z);
+static int r_mark_sUnUz(struct SN_env * z);
+static int r_mark_sUn(struct SN_env * z);
+static int r_mark_sU(struct SN_env * z);
+static int r_mark_nUz(struct SN_env * z);
+static int r_mark_nUn(struct SN_env * z);
+static int r_mark_nU(struct SN_env * z);
+static int r_mark_ndAn(struct SN_env * z);
+static int r_mark_ndA(struct SN_env * z);
+static int r_mark_ncA(struct SN_env * z);
+static int r_mark_nA(struct SN_env * z);
+static int r_mark_lArI(struct SN_env * z);
+static int r_mark_lAr(struct SN_env * z);
+static int r_mark_ki(struct SN_env * z);
+static int r_mark_DUr(struct SN_env * z);
+static int r_mark_DAn(struct SN_env * z);
+static int r_mark_DA(struct SN_env * z);
+static int r_mark_cAsInA(struct SN_env * z);
+static int r_is_reserved_word(struct SN_env * z);
+static int r_check_vowel_harmony(struct SN_env * z);
+static int r_append_U_to_stems_ending_with_d_or_g(struct SN_env * z);
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+
+extern struct SN_env * turkish_UTF_8_create_env(void);
+extern void turkish_UTF_8_close_env(struct SN_env * z);
+
+
+#ifdef __cplusplus
+}
+#endif
+static const symbol s_0_0[1] = { 'm' };
+static const symbol s_0_1[1] = { 'n' };
+static const symbol s_0_2[3] = { 'm', 'i', 'z' };
+static const symbol s_0_3[3] = { 'n', 'i', 'z' };
+static const symbol s_0_4[3] = { 'm', 'u', 'z' };
+static const symbol s_0_5[3] = { 'n', 'u', 'z' };
+static const symbol s_0_6[4] = { 'm', 0xC4, 0xB1, 'z' };
+static const symbol s_0_7[4] = { 'n', 0xC4, 0xB1, 'z' };
+static const symbol s_0_8[4] = { 'm', 0xC3, 0xBC, 'z' };
+static const symbol s_0_9[4] = { 'n', 0xC3, 0xBC, 'z' };
+
+static const struct among a_0[10] =
+{
+/* 0 */ { 1, s_0_0, -1, -1, 0},
+/* 1 */ { 1, s_0_1, -1, -1, 0},
+/* 2 */ { 3, s_0_2, -1, -1, 0},
+/* 3 */ { 3, s_0_3, -1, -1, 0},
+/* 4 */ { 3, s_0_4, -1, -1, 0},
+/* 5 */ { 3, s_0_5, -1, -1, 0},
+/* 6 */ { 4, s_0_6, -1, -1, 0},
+/* 7 */ { 4, s_0_7, -1, -1, 0},
+/* 8 */ { 4, s_0_8, -1, -1, 0},
+/* 9 */ { 4, s_0_9, -1, -1, 0}
+};
+
+static const symbol s_1_0[4] = { 'l', 'e', 'r', 'i' };
+static const symbol s_1_1[5] = { 'l', 'a', 'r', 0xC4, 0xB1 };
+
+static const struct among a_1[2] =
+{
+/* 0 */ { 4, s_1_0, -1, -1, 0},
+/* 1 */ { 5, s_1_1, -1, -1, 0}
+};
+
+static const symbol s_2_0[2] = { 'n', 'i' };
+static const symbol s_2_1[2] = { 'n', 'u' };
+static const symbol s_2_2[3] = { 'n', 0xC4, 0xB1 };
+static const symbol s_2_3[3] = { 'n', 0xC3, 0xBC };
+
+static const struct among a_2[4] =
+{
+/* 0 */ { 2, s_2_0, -1, -1, 0},
+/* 1 */ { 2, s_2_1, -1, -1, 0},
+/* 2 */ { 3, s_2_2, -1, -1, 0},
+/* 3 */ { 3, s_2_3, -1, -1, 0}
+};
+
+static const symbol s_3_0[2] = { 'i', 'n' };
+static const symbol s_3_1[2] = { 'u', 'n' };
+static const symbol s_3_2[3] = { 0xC4, 0xB1, 'n' };
+static const symbol s_3_3[3] = { 0xC3, 0xBC, 'n' };
+
+static const struct among a_3[4] =
+{
+/* 0 */ { 2, s_3_0, -1, -1, 0},
+/* 1 */ { 2, s_3_1, -1, -1, 0},
+/* 2 */ { 3, s_3_2, -1, -1, 0},
+/* 3 */ { 3, s_3_3, -1, -1, 0}
+};
+
+static const symbol s_4_0[1] = { 'a' };
+static const symbol s_4_1[1] = { 'e' };
+
+static const struct among a_4[2] =
+{
+/* 0 */ { 1, s_4_0, -1, -1, 0},
+/* 1 */ { 1, s_4_1, -1, -1, 0}
+};
+
+static const symbol s_5_0[2] = { 'n', 'a' };
+static const symbol s_5_1[2] = { 'n', 'e' };
+
+static const struct among a_5[2] =
+{
+/* 0 */ { 2, s_5_0, -1, -1, 0},
+/* 1 */ { 2, s_5_1, -1, -1, 0}
+};
+
+static const symbol s_6_0[2] = { 'd', 'a' };
+static const symbol s_6_1[2] = { 't', 'a' };
+static const symbol s_6_2[2] = { 'd', 'e' };
+static const symbol s_6_3[2] = { 't', 'e' };
+
+static const struct among a_6[4] =
+{
+/* 0 */ { 2, s_6_0, -1, -1, 0},
+/* 1 */ { 2, s_6_1, -1, -1, 0},
+/* 2 */ { 2, s_6_2, -1, -1, 0},
+/* 3 */ { 2, s_6_3, -1, -1, 0}
+};
+
+static const symbol s_7_0[3] = { 'n', 'd', 'a' };
+static const symbol s_7_1[3] = { 'n', 'd', 'e' };
+
+static const struct among a_7[2] =
+{
+/* 0 */ { 3, s_7_0, -1, -1, 0},
+/* 1 */ { 3, s_7_1, -1, -1, 0}
+};
+
+static const symbol s_8_0[3] = { 'd', 'a', 'n' };
+static const symbol s_8_1[3] = { 't', 'a', 'n' };
+static const symbol s_8_2[3] = { 'd', 'e', 'n' };
+static const symbol s_8_3[3] = { 't', 'e', 'n' };
+
+static const struct among a_8[4] =
+{
+/* 0 */ { 3, s_8_0, -1, -1, 0},
+/* 1 */ { 3, s_8_1, -1, -1, 0},
+/* 2 */ { 3, s_8_2, -1, -1, 0},
+/* 3 */ { 3, s_8_3, -1, -1, 0}
+};
+
+static const symbol s_9_0[4] = { 'n', 'd', 'a', 'n' };
+static const symbol s_9_1[4] = { 'n', 'd', 'e', 'n' };
+
+static const struct among a_9[2] =
+{
+/* 0 */ { 4, s_9_0, -1, -1, 0},
+/* 1 */ { 4, s_9_1, -1, -1, 0}
+};
+
+static const symbol s_10_0[2] = { 'l', 'a' };
+static const symbol s_10_1[2] = { 'l', 'e' };
+
+static const struct among a_10[2] =
+{
+/* 0 */ { 2, s_10_0, -1, -1, 0},
+/* 1 */ { 2, s_10_1, -1, -1, 0}
+};
+
+static const symbol s_11_0[2] = { 'c', 'a' };
+static const symbol s_11_1[2] = { 'c', 'e' };
+
+static const struct among a_11[2] =
+{
+/* 0 */ { 2, s_11_0, -1, -1, 0},
+/* 1 */ { 2, s_11_1, -1, -1, 0}
+};
+
+static const symbol s_12_0[2] = { 'i', 'm' };
+static const symbol s_12_1[2] = { 'u', 'm' };
+static const symbol s_12_2[3] = { 0xC4, 0xB1, 'm' };
+static const symbol s_12_3[3] = { 0xC3, 0xBC, 'm' };
+
+static const struct among a_12[4] =
+{
+/* 0 */ { 2, s_12_0, -1, -1, 0},
+/* 1 */ { 2, s_12_1, -1, -1, 0},
+/* 2 */ { 3, s_12_2, -1, -1, 0},
+/* 3 */ { 3, s_12_3, -1, -1, 0}
+};
+
+static const symbol s_13_0[3] = { 's', 'i', 'n' };
+static const symbol s_13_1[3] = { 's', 'u', 'n' };
+static const symbol s_13_2[4] = { 's', 0xC4, 0xB1, 'n' };
+static const symbol s_13_3[4] = { 's', 0xC3, 0xBC, 'n' };
+
+static const struct among a_13[4] =
+{
+/* 0 */ { 3, s_13_0, -1, -1, 0},
+/* 1 */ { 3, s_13_1, -1, -1, 0},
+/* 2 */ { 4, s_13_2, -1, -1, 0},
+/* 3 */ { 4, s_13_3, -1, -1, 0}
+};
+
+static const symbol s_14_0[2] = { 'i', 'z' };
+static const symbol s_14_1[2] = { 'u', 'z' };
+static const symbol s_14_2[3] = { 0xC4, 0xB1, 'z' };
+static const symbol s_14_3[3] = { 0xC3, 0xBC, 'z' };
+
+static const struct among a_14[4] =
+{
+/* 0 */ { 2, s_14_0, -1, -1, 0},
+/* 1 */ { 2, s_14_1, -1, -1, 0},
+/* 2 */ { 3, s_14_2, -1, -1, 0},
+/* 3 */ { 3, s_14_3, -1, -1, 0}
+};
+
+static const symbol s_15_0[5] = { 's', 'i', 'n', 'i', 'z' };
+static const symbol s_15_1[5] = { 's', 'u', 'n', 'u', 'z' };
+static const symbol s_15_2[7] = { 's', 0xC4, 0xB1, 'n', 0xC4, 0xB1, 'z' };
+static const symbol s_15_3[7] = { 's', 0xC3, 0xBC, 'n', 0xC3, 0xBC, 'z' };
+
+static const struct among a_15[4] =
+{
+/* 0 */ { 5, s_15_0, -1, -1, 0},
+/* 1 */ { 5, s_15_1, -1, -1, 0},
+/* 2 */ { 7, s_15_2, -1, -1, 0},
+/* 3 */ { 7, s_15_3, -1, -1, 0}
+};
+
+static const symbol s_16_0[3] = { 'l', 'a', 'r' };
+static const symbol s_16_1[3] = { 'l', 'e', 'r' };
+
+static const struct among a_16[2] =
+{
+/* 0 */ { 3, s_16_0, -1, -1, 0},
+/* 1 */ { 3, s_16_1, -1, -1, 0}
+};
+
+static const symbol s_17_0[3] = { 'n', 'i', 'z' };
+static const symbol s_17_1[3] = { 'n', 'u', 'z' };
+static const symbol s_17_2[4] = { 'n', 0xC4, 0xB1, 'z' };
+static const symbol s_17_3[4] = { 'n', 0xC3, 0xBC, 'z' };
+
+static const struct among a_17[4] =
+{
+/* 0 */ { 3, s_17_0, -1, -1, 0},
+/* 1 */ { 3, s_17_1, -1, -1, 0},
+/* 2 */ { 4, s_17_2, -1, -1, 0},
+/* 3 */ { 4, s_17_3, -1, -1, 0}
+};
+
+static const symbol s_18_0[3] = { 'd', 'i', 'r' };
+static const symbol s_18_1[3] = { 't', 'i', 'r' };
+static const symbol s_18_2[3] = { 'd', 'u', 'r' };
+static const symbol s_18_3[3] = { 't', 'u', 'r' };
+static const symbol s_18_4[4] = { 'd', 0xC4, 0xB1, 'r' };
+static const symbol s_18_5[4] = { 't', 0xC4, 0xB1, 'r' };
+static const symbol s_18_6[4] = { 'd', 0xC3, 0xBC, 'r' };
+static const symbol s_18_7[4] = { 't', 0xC3, 0xBC, 'r' };
+
+static const struct among a_18[8] =
+{
+/* 0 */ { 3, s_18_0, -1, -1, 0},
+/* 1 */ { 3, s_18_1, -1, -1, 0},
+/* 2 */ { 3, s_18_2, -1, -1, 0},
+/* 3 */ { 3, s_18_3, -1, -1, 0},
+/* 4 */ { 4, s_18_4, -1, -1, 0},
+/* 5 */ { 4, s_18_5, -1, -1, 0},
+/* 6 */ { 4, s_18_6, -1, -1, 0},
+/* 7 */ { 4, s_18_7, -1, -1, 0}
+};
+
+static const symbol s_19_0[7] = { 'c', 'a', 's', 0xC4, 0xB1, 'n', 'a' };
+static const symbol s_19_1[6] = { 'c', 'e', 's', 'i', 'n', 'e' };
+
+static const struct among a_19[2] =
+{
+/* 0 */ { 7, s_19_0, -1, -1, 0},
+/* 1 */ { 6, s_19_1, -1, -1, 0}
+};
+
+static const symbol s_20_0[2] = { 'd', 'i' };
+static const symbol s_20_1[2] = { 't', 'i' };
+static const symbol s_20_2[3] = { 'd', 'i', 'k' };
+static const symbol s_20_3[3] = { 't', 'i', 'k' };
+static const symbol s_20_4[3] = { 'd', 'u', 'k' };
+static const symbol s_20_5[3] = { 't', 'u', 'k' };
+static const symbol s_20_6[4] = { 'd', 0xC4, 0xB1, 'k' };
+static const symbol s_20_7[4] = { 't', 0xC4, 0xB1, 'k' };
+static const symbol s_20_8[4] = { 'd', 0xC3, 0xBC, 'k' };
+static const symbol s_20_9[4] = { 't', 0xC3, 0xBC, 'k' };
+static const symbol s_20_10[3] = { 'd', 'i', 'm' };
+static const symbol s_20_11[3] = { 't', 'i', 'm' };
+static const symbol s_20_12[3] = { 'd', 'u', 'm' };
+static const symbol s_20_13[3] = { 't', 'u', 'm' };
+static const symbol s_20_14[4] = { 'd', 0xC4, 0xB1, 'm' };
+static const symbol s_20_15[4] = { 't', 0xC4, 0xB1, 'm' };
+static const symbol s_20_16[4] = { 'd', 0xC3, 0xBC, 'm' };
+static const symbol s_20_17[4] = { 't', 0xC3, 0xBC, 'm' };
+static const symbol s_20_18[3] = { 'd', 'i', 'n' };
+static const symbol s_20_19[3] = { 't', 'i', 'n' };
+static const symbol s_20_20[3] = { 'd', 'u', 'n' };
+static const symbol s_20_21[3] = { 't', 'u', 'n' };
+static const symbol s_20_22[4] = { 'd', 0xC4, 0xB1, 'n' };
+static const symbol s_20_23[4] = { 't', 0xC4, 0xB1, 'n' };
+static const symbol s_20_24[4] = { 'd', 0xC3, 0xBC, 'n' };
+static const symbol s_20_25[4] = { 't', 0xC3, 0xBC, 'n' };
+static const symbol s_20_26[2] = { 'd', 'u' };
+static const symbol s_20_27[2] = { 't', 'u' };
+static const symbol s_20_28[3] = { 'd', 0xC4, 0xB1 };
+static const symbol s_20_29[3] = { 't', 0xC4, 0xB1 };
+static const symbol s_20_30[3] = { 'd', 0xC3, 0xBC };
+static const symbol s_20_31[3] = { 't', 0xC3, 0xBC };
+
+static const struct among a_20[32] =
+{
+/* 0 */ { 2, s_20_0, -1, -1, 0},
+/* 1 */ { 2, s_20_1, -1, -1, 0},
+/* 2 */ { 3, s_20_2, -1, -1, 0},
+/* 3 */ { 3, s_20_3, -1, -1, 0},
+/* 4 */ { 3, s_20_4, -1, -1, 0},
+/* 5 */ { 3, s_20_5, -1, -1, 0},
+/* 6 */ { 4, s_20_6, -1, -1, 0},
+/* 7 */ { 4, s_20_7, -1, -1, 0},
+/* 8 */ { 4, s_20_8, -1, -1, 0},
+/* 9 */ { 4, s_20_9, -1, -1, 0},
+/* 10 */ { 3, s_20_10, -1, -1, 0},
+/* 11 */ { 3, s_20_11, -1, -1, 0},
+/* 12 */ { 3, s_20_12, -1, -1, 0},
+/* 13 */ { 3, s_20_13, -1, -1, 0},
+/* 14 */ { 4, s_20_14, -1, -1, 0},
+/* 15 */ { 4, s_20_15, -1, -1, 0},
+/* 16 */ { 4, s_20_16, -1, -1, 0},
+/* 17 */ { 4, s_20_17, -1, -1, 0},
+/* 18 */ { 3, s_20_18, -1, -1, 0},
+/* 19 */ { 3, s_20_19, -1, -1, 0},
+/* 20 */ { 3, s_20_20, -1, -1, 0},
+/* 21 */ { 3, s_20_21, -1, -1, 0},
+/* 22 */ { 4, s_20_22, -1, -1, 0},
+/* 23 */ { 4, s_20_23, -1, -1, 0},
+/* 24 */ { 4, s_20_24, -1, -1, 0},
+/* 25 */ { 4, s_20_25, -1, -1, 0},
+/* 26 */ { 2, s_20_26, -1, -1, 0},
+/* 27 */ { 2, s_20_27, -1, -1, 0},
+/* 28 */ { 3, s_20_28, -1, -1, 0},
+/* 29 */ { 3, s_20_29, -1, -1, 0},
+/* 30 */ { 3, s_20_30, -1, -1, 0},
+/* 31 */ { 3, s_20_31, -1, -1, 0}
+};
+
+static const symbol s_21_0[2] = { 's', 'a' };
+static const symbol s_21_1[2] = { 's', 'e' };
+static const symbol s_21_2[3] = { 's', 'a', 'k' };
+static const symbol s_21_3[3] = { 's', 'e', 'k' };
+static const symbol s_21_4[3] = { 's', 'a', 'm' };
+static const symbol s_21_5[3] = { 's', 'e', 'm' };
+static const symbol s_21_6[3] = { 's', 'a', 'n' };
+static const symbol s_21_7[3] = { 's', 'e', 'n' };
+
+static const struct among a_21[8] =
+{
+/* 0 */ { 2, s_21_0, -1, -1, 0},
+/* 1 */ { 2, s_21_1, -1, -1, 0},
+/* 2 */ { 3, s_21_2, -1, -1, 0},
+/* 3 */ { 3, s_21_3, -1, -1, 0},
+/* 4 */ { 3, s_21_4, -1, -1, 0},
+/* 5 */ { 3, s_21_5, -1, -1, 0},
+/* 6 */ { 3, s_21_6, -1, -1, 0},
+/* 7 */ { 3, s_21_7, -1, -1, 0}
+};
+
+static const symbol s_22_0[4] = { 'm', 'i', 0xC5, 0x9F };
+static const symbol s_22_1[4] = { 'm', 'u', 0xC5, 0x9F };
+static const symbol s_22_2[5] = { 'm', 0xC4, 0xB1, 0xC5, 0x9F };
+static const symbol s_22_3[5] = { 'm', 0xC3, 0xBC, 0xC5, 0x9F };
+
+static const struct among a_22[4] =
+{
+/* 0 */ { 4, s_22_0, -1, -1, 0},
+/* 1 */ { 4, s_22_1, -1, -1, 0},
+/* 2 */ { 5, s_22_2, -1, -1, 0},
+/* 3 */ { 5, s_22_3, -1, -1, 0}
+};
+
+static const symbol s_23_0[1] = { 'b' };
+static const symbol s_23_1[1] = { 'c' };
+static const symbol s_23_2[1] = { 'd' };
+static const symbol s_23_3[2] = { 0xC4, 0x9F };
+
+static const struct among a_23[4] =
+{
+/* 0 */ { 1, s_23_0, -1, 1, 0},
+/* 1 */ { 1, s_23_1, -1, 2, 0},
+/* 2 */ { 1, s_23_2, -1, 3, 0},
+/* 3 */ { 2, s_23_3, -1, 4, 0}
+};
+
+static const unsigned char g_vowel[] = { 17, 65, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 8, 0, 0, 0, 0, 0, 0, 1 };
+
+static const unsigned char g_U[] = { 1, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 1 };
+
+static const unsigned char g_vowel1[] = { 1, 64, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+
+static const unsigned char g_vowel2[] = { 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130 };
+
+static const unsigned char g_vowel3[] = { 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 };
+
+static const unsigned char g_vowel4[] = { 17 };
+
+static const unsigned char g_vowel5[] = { 65 };
+
+static const unsigned char g_vowel6[] = { 65 };
+
+static const symbol s_0[] = { 'a' };
+static const symbol s_1[] = { 'e' };
+static const symbol s_2[] = { 0xC4, 0xB1 };
+static const symbol s_3[] = { 'i' };
+static const symbol s_4[] = { 'o' };
+static const symbol s_5[] = { 0xC3, 0xB6 };
+static const symbol s_6[] = { 'u' };
+static const symbol s_7[] = { 0xC3, 0xBC };
+static const symbol s_8[] = { 'n' };
+static const symbol s_9[] = { 'n' };
+static const symbol s_10[] = { 's' };
+static const symbol s_11[] = { 's' };
+static const symbol s_12[] = { 'y' };
+static const symbol s_13[] = { 'y' };
+static const symbol s_14[] = { 'k', 'i' };
+static const symbol s_15[] = { 'k', 'e', 'n' };
+static const symbol s_16[] = { 'p' };
+static const symbol s_17[] = { 0xC3, 0xA7 };
+static const symbol s_18[] = { 't' };
+static const symbol s_19[] = { 'k' };
+static const symbol s_20[] = { 'd' };
+static const symbol s_21[] = { 'g' };
+static const symbol s_22[] = { 'a' };
+static const symbol s_23[] = { 0xC4, 0xB1 };
+static const symbol s_24[] = { 0xC4, 0xB1 };
+static const symbol s_25[] = { 'e' };
+static const symbol s_26[] = { 'i' };
+static const symbol s_27[] = { 'i' };
+static const symbol s_28[] = { 'o' };
+static const symbol s_29[] = { 'u' };
+static const symbol s_30[] = { 'u' };
+static const symbol s_31[] = { 0xC3, 0xB6 };
+static const symbol s_32[] = { 0xC3, 0xBC };
+static const symbol s_33[] = { 0xC3, 0xBC };
+static const symbol s_34[] = { 'a', 'd' };
+static const symbol s_35[] = { 's', 'o', 'y', 'a', 'd' };
+
+static int r_check_vowel_harmony(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 112 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 1) < 0) return 0; /* goto */ /* grouping vowel, line 114 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 116 */
+ if (!(eq_s_b(z, 1, s_0))) goto lab1;
+ if (out_grouping_b_U(z, g_vowel1, 97, 305, 1) < 0) goto lab1; /* goto */ /* grouping vowel1, line 116 */
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_1))) goto lab2;
+ if (out_grouping_b_U(z, g_vowel2, 101, 252, 1) < 0) goto lab2; /* goto */ /* grouping vowel2, line 117 */
+ goto lab0;
+ lab2:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_2))) goto lab3;
+ if (out_grouping_b_U(z, g_vowel3, 97, 305, 1) < 0) goto lab3; /* goto */ /* grouping vowel3, line 118 */
+ goto lab0;
+ lab3:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_3))) goto lab4;
+ if (out_grouping_b_U(z, g_vowel4, 101, 105, 1) < 0) goto lab4; /* goto */ /* grouping vowel4, line 119 */
+ goto lab0;
+ lab4:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_4))) goto lab5;
+ if (out_grouping_b_U(z, g_vowel5, 111, 117, 1) < 0) goto lab5; /* goto */ /* grouping vowel5, line 120 */
+ goto lab0;
+ lab5:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_5))) goto lab6;
+ if (out_grouping_b_U(z, g_vowel6, 246, 252, 1) < 0) goto lab6; /* goto */ /* grouping vowel6, line 121 */
+ goto lab0;
+ lab6:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_6))) goto lab7;
+ if (out_grouping_b_U(z, g_vowel5, 111, 117, 1) < 0) goto lab7; /* goto */ /* grouping vowel5, line 122 */
+ goto lab0;
+ lab7:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 2, s_7))) return 0;
+ if (out_grouping_b_U(z, g_vowel6, 246, 252, 1) < 0) return 0; /* goto */ /* grouping vowel6, line 123 */
+ }
+ lab0:
+ z->c = z->l - m_test;
+ }
+ return 1;
+}
+
+static int r_mark_suffix_with_optional_n_consonant(struct SN_env * z) {
+ { int m1 = z->l - z->c; (void)m1; /* or, line 134 */
+ { int m_test = z->l - z->c; /* test, line 133 */
+ if (!(eq_s_b(z, 1, s_8))) goto lab1;
+ z->c = z->l - m_test;
+ }
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* next, line 133 */
+ }
+ { int m_test = z->l - z->c; /* test, line 133 */
+ if (in_grouping_b_U(z, g_vowel, 97, 305, 0)) goto lab1;
+ z->c = z->l - m_test;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* not, line 135 */
+ { int m_test = z->l - z->c; /* test, line 135 */
+ if (!(eq_s_b(z, 1, s_9))) goto lab2;
+ z->c = z->l - m_test;
+ }
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ { int m_test = z->l - z->c; /* test, line 135 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 135 */
+ }
+ { int m_test = z->l - z->c; /* test, line 135 */
+ if (in_grouping_b_U(z, g_vowel, 97, 305, 0)) return 0;
+ z->c = z->l - m_test;
+ }
+ z->c = z->l - m_test;
+ }
+ }
+lab0:
+ return 1;
+}
+
+static int r_mark_suffix_with_optional_s_consonant(struct SN_env * z) {
+ { int m1 = z->l - z->c; (void)m1; /* or, line 145 */
+ { int m_test = z->l - z->c; /* test, line 144 */
+ if (!(eq_s_b(z, 1, s_10))) goto lab1;
+ z->c = z->l - m_test;
+ }
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* next, line 144 */
+ }
+ { int m_test = z->l - z->c; /* test, line 144 */
+ if (in_grouping_b_U(z, g_vowel, 97, 305, 0)) goto lab1;
+ z->c = z->l - m_test;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* not, line 146 */
+ { int m_test = z->l - z->c; /* test, line 146 */
+ if (!(eq_s_b(z, 1, s_11))) goto lab2;
+ z->c = z->l - m_test;
+ }
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ { int m_test = z->l - z->c; /* test, line 146 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 146 */
+ }
+ { int m_test = z->l - z->c; /* test, line 146 */
+ if (in_grouping_b_U(z, g_vowel, 97, 305, 0)) return 0;
+ z->c = z->l - m_test;
+ }
+ z->c = z->l - m_test;
+ }
+ }
+lab0:
+ return 1;
+}
+
+static int r_mark_suffix_with_optional_y_consonant(struct SN_env * z) {
+ { int m1 = z->l - z->c; (void)m1; /* or, line 155 */
+ { int m_test = z->l - z->c; /* test, line 154 */
+ if (!(eq_s_b(z, 1, s_12))) goto lab1;
+ z->c = z->l - m_test;
+ }
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* next, line 154 */
+ }
+ { int m_test = z->l - z->c; /* test, line 154 */
+ if (in_grouping_b_U(z, g_vowel, 97, 305, 0)) goto lab1;
+ z->c = z->l - m_test;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* not, line 156 */
+ { int m_test = z->l - z->c; /* test, line 156 */
+ if (!(eq_s_b(z, 1, s_13))) goto lab2;
+ z->c = z->l - m_test;
+ }
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ { int m_test = z->l - z->c; /* test, line 156 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 156 */
+ }
+ { int m_test = z->l - z->c; /* test, line 156 */
+ if (in_grouping_b_U(z, g_vowel, 97, 305, 0)) return 0;
+ z->c = z->l - m_test;
+ }
+ z->c = z->l - m_test;
+ }
+ }
+lab0:
+ return 1;
+}
+
+static int r_mark_suffix_with_optional_U_vowel(struct SN_env * z) {
+ { int m1 = z->l - z->c; (void)m1; /* or, line 161 */
+ { int m_test = z->l - z->c; /* test, line 160 */
+ if (in_grouping_b_U(z, g_U, 105, 305, 0)) goto lab1;
+ z->c = z->l - m_test;
+ }
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* next, line 160 */
+ }
+ { int m_test = z->l - z->c; /* test, line 160 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 0)) goto lab1;
+ z->c = z->l - m_test;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int m2 = z->l - z->c; (void)m2; /* not, line 162 */
+ { int m_test = z->l - z->c; /* test, line 162 */
+ if (in_grouping_b_U(z, g_U, 105, 305, 0)) goto lab2;
+ z->c = z->l - m_test;
+ }
+ return 0;
+ lab2:
+ z->c = z->l - m2;
+ }
+ { int m_test = z->l - z->c; /* test, line 162 */
+ { int ret = skip_utf8(z->p, z->c, z->lb, 0, -1);
+ if (ret < 0) return 0;
+ z->c = ret; /* next, line 162 */
+ }
+ { int m_test = z->l - z->c; /* test, line 162 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 0)) return 0;
+ z->c = z->l - m_test;
+ }
+ z->c = z->l - m_test;
+ }
+ }
+lab0:
+ return 1;
+}
+
+static int r_mark_possessives(struct SN_env * z) {
+ if (z->c <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((67133440 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_0, 10))) return 0; /* among, line 167 */
+ { int ret = r_mark_suffix_with_optional_U_vowel(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_U_vowel, line 169 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_sU(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 173 */
+ if (ret < 0) return ret;
+ }
+ if (in_grouping_b_U(z, g_U, 105, 305, 0)) return 0;
+ { int ret = r_mark_suffix_with_optional_s_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_s_consonant, line 175 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_lArI(struct SN_env * z) {
+ if (z->c - 3 <= z->lb || (z->p[z->c - 1] != 105 && z->p[z->c - 1] != 177)) return 0;
+ if (!(find_among_b(z, a_1, 2))) return 0; /* among, line 179 */
+ return 1;
+}
+
+static int r_mark_yU(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 183 */
+ if (ret < 0) return ret;
+ }
+ if (in_grouping_b_U(z, g_U, 105, 305, 0)) return 0;
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 185 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_nU(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 189 */
+ if (ret < 0) return ret;
+ }
+ if (!(find_among_b(z, a_2, 4))) return 0; /* among, line 190 */
+ return 1;
+}
+
+static int r_mark_nUn(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 194 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] != 110) return 0;
+ if (!(find_among_b(z, a_3, 4))) return 0; /* among, line 195 */
+ { int ret = r_mark_suffix_with_optional_n_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_n_consonant, line 196 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_yA(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 200 */
+ if (ret < 0) return ret;
+ }
+ if (z->c <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_4, 2))) return 0; /* among, line 201 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 202 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_nA(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 206 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_5, 2))) return 0; /* among, line 207 */
+ return 1;
+}
+
+static int r_mark_DA(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 211 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_6, 4))) return 0; /* among, line 212 */
+ return 1;
+}
+
+static int r_mark_ndA(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 216 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 2 <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_7, 2))) return 0; /* among, line 217 */
+ return 1;
+}
+
+static int r_mark_DAn(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 221 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 110) return 0;
+ if (!(find_among_b(z, a_8, 4))) return 0; /* among, line 222 */
+ return 1;
+}
+
+static int r_mark_ndAn(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 226 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 3 <= z->lb || z->p[z->c - 1] != 110) return 0;
+ if (!(find_among_b(z, a_9, 2))) return 0; /* among, line 227 */
+ return 1;
+}
+
+static int r_mark_ylA(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 231 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_10, 2))) return 0; /* among, line 232 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 233 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_ki(struct SN_env * z) {
+ if (!(eq_s_b(z, 2, s_14))) return 0;
+ return 1;
+}
+
+static int r_mark_ncA(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 241 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_11, 2))) return 0; /* among, line 242 */
+ { int ret = r_mark_suffix_with_optional_n_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_n_consonant, line 243 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_yUm(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 247 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] != 109) return 0;
+ if (!(find_among_b(z, a_12, 4))) return 0; /* among, line 248 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 249 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_sUn(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 253 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 110) return 0;
+ if (!(find_among_b(z, a_13, 4))) return 0; /* among, line 254 */
+ return 1;
+}
+
+static int r_mark_yUz(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 258 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] != 122) return 0;
+ if (!(find_among_b(z, a_14, 4))) return 0; /* among, line 259 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 260 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_sUnUz(struct SN_env * z) {
+ if (z->c - 4 <= z->lb || z->p[z->c - 1] != 122) return 0;
+ if (!(find_among_b(z, a_15, 4))) return 0; /* among, line 264 */
+ return 1;
+}
+
+static int r_mark_lAr(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 268 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 114) return 0;
+ if (!(find_among_b(z, a_16, 2))) return 0; /* among, line 269 */
+ return 1;
+}
+
+static int r_mark_nUz(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 273 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 122) return 0;
+ if (!(find_among_b(z, a_17, 4))) return 0; /* among, line 274 */
+ return 1;
+}
+
+static int r_mark_DUr(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 278 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 2 <= z->lb || z->p[z->c - 1] != 114) return 0;
+ if (!(find_among_b(z, a_18, 8))) return 0; /* among, line 279 */
+ return 1;
+}
+
+static int r_mark_cAsInA(struct SN_env * z) {
+ if (z->c - 5 <= z->lb || (z->p[z->c - 1] != 97 && z->p[z->c - 1] != 101)) return 0;
+ if (!(find_among_b(z, a_19, 2))) return 0; /* among, line 283 */
+ return 1;
+}
+
+static int r_mark_yDU(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 287 */
+ if (ret < 0) return ret;
+ }
+ if (!(find_among_b(z, a_20, 32))) return 0; /* among, line 288 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 292 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_ysA(struct SN_env * z) {
+ if (z->c - 1 <= z->lb || z->p[z->c - 1] >> 5 != 3 || !((26658 >> (z->p[z->c - 1] & 0x1f)) & 1)) return 0;
+ if (!(find_among_b(z, a_21, 8))) return 0; /* among, line 297 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 298 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_ymUs_(struct SN_env * z) {
+ { int ret = r_check_vowel_harmony(z);
+ if (ret == 0) return 0; /* call check_vowel_harmony, line 302 */
+ if (ret < 0) return ret;
+ }
+ if (z->c - 3 <= z->lb || z->p[z->c - 1] != 159) return 0;
+ if (!(find_among_b(z, a_22, 4))) return 0; /* among, line 303 */
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 304 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_mark_yken(struct SN_env * z) {
+ if (!(eq_s_b(z, 3, s_15))) return 0;
+ { int ret = r_mark_suffix_with_optional_y_consonant(z);
+ if (ret == 0) return 0; /* call mark_suffix_with_optional_y_consonant, line 308 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_stem_nominal_verb_suffixes(struct SN_env * z) {
+ z->ket = z->c; /* [, line 312 */
+ z->B[0] = 1; /* set continue_stemming_noun_suffixes, line 313 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 315 */
+ { int m2 = z->l - z->c; (void)m2; /* or, line 314 */
+ { int ret = r_mark_ymUs_(z);
+ if (ret == 0) goto lab3; /* call mark_ymUs_, line 314 */
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab3:
+ z->c = z->l - m2;
+ { int ret = r_mark_yDU(z);
+ if (ret == 0) goto lab4; /* call mark_yDU, line 314 */
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab4:
+ z->c = z->l - m2;
+ { int ret = r_mark_ysA(z);
+ if (ret == 0) goto lab5; /* call mark_ysA, line 314 */
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab5:
+ z->c = z->l - m2;
+ { int ret = r_mark_yken(z);
+ if (ret == 0) goto lab1; /* call mark_yken, line 314 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab2:
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int ret = r_mark_cAsInA(z);
+ if (ret == 0) goto lab6; /* call mark_cAsInA, line 316 */
+ if (ret < 0) return ret;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* or, line 316 */
+ { int ret = r_mark_sUnUz(z);
+ if (ret == 0) goto lab8; /* call mark_sUnUz, line 316 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab8:
+ z->c = z->l - m3;
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab9; /* call mark_lAr, line 316 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab9:
+ z->c = z->l - m3;
+ { int ret = r_mark_yUm(z);
+ if (ret == 0) goto lab10; /* call mark_yUm, line 316 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab10:
+ z->c = z->l - m3;
+ { int ret = r_mark_sUn(z);
+ if (ret == 0) goto lab11; /* call mark_sUn, line 316 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab11:
+ z->c = z->l - m3;
+ { int ret = r_mark_yUz(z);
+ if (ret == 0) goto lab12; /* call mark_yUz, line 316 */
+ if (ret < 0) return ret;
+ }
+ goto lab7;
+ lab12:
+ z->c = z->l - m3;
+ }
+ lab7:
+ { int ret = r_mark_ymUs_(z);
+ if (ret == 0) goto lab6; /* call mark_ymUs_, line 316 */
+ if (ret < 0) return ret;
+ }
+ goto lab0;
+ lab6:
+ z->c = z->l - m1;
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab13; /* call mark_lAr, line 319 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 319 */
+ { int ret = slice_del(z); /* delete, line 319 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 319 */
+ z->ket = z->c; /* [, line 319 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 319 */
+ { int ret = r_mark_DUr(z);
+ if (ret == 0) goto lab16; /* call mark_DUr, line 319 */
+ if (ret < 0) return ret;
+ }
+ goto lab15;
+ lab16:
+ z->c = z->l - m4;
+ { int ret = r_mark_yDU(z);
+ if (ret == 0) goto lab17; /* call mark_yDU, line 319 */
+ if (ret < 0) return ret;
+ }
+ goto lab15;
+ lab17:
+ z->c = z->l - m4;
+ { int ret = r_mark_ysA(z);
+ if (ret == 0) goto lab18; /* call mark_ysA, line 319 */
+ if (ret < 0) return ret;
+ }
+ goto lab15;
+ lab18:
+ z->c = z->l - m4;
+ { int ret = r_mark_ymUs_(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab14; } /* call mark_ymUs_, line 319 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab15:
+ lab14:
+ ;
+ }
+ z->B[0] = 0; /* unset continue_stemming_noun_suffixes, line 320 */
+ goto lab0;
+ lab13:
+ z->c = z->l - m1;
+ { int ret = r_mark_nUz(z);
+ if (ret == 0) goto lab19; /* call mark_nUz, line 323 */
+ if (ret < 0) return ret;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* or, line 323 */
+ { int ret = r_mark_yDU(z);
+ if (ret == 0) goto lab21; /* call mark_yDU, line 323 */
+ if (ret < 0) return ret;
+ }
+ goto lab20;
+ lab21:
+ z->c = z->l - m5;
+ { int ret = r_mark_ysA(z);
+ if (ret == 0) goto lab19; /* call mark_ysA, line 323 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab20:
+ goto lab0;
+ lab19:
+ z->c = z->l - m1;
+ { int m6 = z->l - z->c; (void)m6; /* or, line 325 */
+ { int ret = r_mark_sUnUz(z);
+ if (ret == 0) goto lab24; /* call mark_sUnUz, line 325 */
+ if (ret < 0) return ret;
+ }
+ goto lab23;
+ lab24:
+ z->c = z->l - m6;
+ { int ret = r_mark_yUz(z);
+ if (ret == 0) goto lab25; /* call mark_yUz, line 325 */
+ if (ret < 0) return ret;
+ }
+ goto lab23;
+ lab25:
+ z->c = z->l - m6;
+ { int ret = r_mark_sUn(z);
+ if (ret == 0) goto lab26; /* call mark_sUn, line 325 */
+ if (ret < 0) return ret;
+ }
+ goto lab23;
+ lab26:
+ z->c = z->l - m6;
+ { int ret = r_mark_yUm(z);
+ if (ret == 0) goto lab22; /* call mark_yUm, line 325 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab23:
+ z->bra = z->c; /* ], line 325 */
+ { int ret = slice_del(z); /* delete, line 325 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 325 */
+ z->ket = z->c; /* [, line 325 */
+ { int ret = r_mark_ymUs_(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab27; } /* call mark_ymUs_, line 325 */
+ if (ret < 0) return ret;
+ }
+ lab27:
+ ;
+ }
+ goto lab0;
+ lab22:
+ z->c = z->l - m1;
+ { int ret = r_mark_DUr(z);
+ if (ret == 0) return 0; /* call mark_DUr, line 327 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 327 */
+ { int ret = slice_del(z); /* delete, line 327 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 327 */
+ z->ket = z->c; /* [, line 327 */
+ { int m7 = z->l - z->c; (void)m7; /* or, line 327 */
+ { int ret = r_mark_sUnUz(z);
+ if (ret == 0) goto lab30; /* call mark_sUnUz, line 327 */
+ if (ret < 0) return ret;
+ }
+ goto lab29;
+ lab30:
+ z->c = z->l - m7;
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab31; /* call mark_lAr, line 327 */
+ if (ret < 0) return ret;
+ }
+ goto lab29;
+ lab31:
+ z->c = z->l - m7;
+ { int ret = r_mark_yUm(z);
+ if (ret == 0) goto lab32; /* call mark_yUm, line 327 */
+ if (ret < 0) return ret;
+ }
+ goto lab29;
+ lab32:
+ z->c = z->l - m7;
+ { int ret = r_mark_sUn(z);
+ if (ret == 0) goto lab33; /* call mark_sUn, line 327 */
+ if (ret < 0) return ret;
+ }
+ goto lab29;
+ lab33:
+ z->c = z->l - m7;
+ { int ret = r_mark_yUz(z);
+ if (ret == 0) goto lab34; /* call mark_yUz, line 327 */
+ if (ret < 0) return ret;
+ }
+ goto lab29;
+ lab34:
+ z->c = z->l - m7;
+ }
+ lab29:
+ { int ret = r_mark_ymUs_(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab28; } /* call mark_ymUs_, line 327 */
+ if (ret < 0) return ret;
+ }
+ lab28:
+ ;
+ }
+ }
+lab0:
+ z->bra = z->c; /* ], line 328 */
+ { int ret = slice_del(z); /* delete, line 328 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+static int r_stem_suffix_chain_before_ki(struct SN_env * z) {
+ z->ket = z->c; /* [, line 333 */
+ { int ret = r_mark_ki(z);
+ if (ret == 0) return 0; /* call mark_ki, line 334 */
+ if (ret < 0) return ret;
+ }
+ { int m1 = z->l - z->c; (void)m1; /* or, line 342 */
+ { int ret = r_mark_DA(z);
+ if (ret == 0) goto lab1; /* call mark_DA, line 336 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 336 */
+ { int ret = slice_del(z); /* delete, line 336 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 336 */
+ z->ket = z->c; /* [, line 336 */
+ { int m2 = z->l - z->c; (void)m2; /* or, line 338 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab4; /* call mark_lAr, line 337 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 337 */
+ { int ret = slice_del(z); /* delete, line 337 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 337 */
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab5; } /* call stem_suffix_chain_before_ki, line 337 */
+ if (ret < 0) return ret;
+ }
+ lab5:
+ ;
+ }
+ goto lab3;
+ lab4:
+ z->c = z->l - m2;
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab2; } /* call mark_possessives, line 339 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 339 */
+ { int ret = slice_del(z); /* delete, line 339 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 339 */
+ z->ket = z->c; /* [, line 339 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab6; } /* call mark_lAr, line 339 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 339 */
+ { int ret = slice_del(z); /* delete, line 339 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab6; } /* call stem_suffix_chain_before_ki, line 339 */
+ if (ret < 0) return ret;
+ }
+ lab6:
+ ;
+ }
+ }
+ lab3:
+ lab2:
+ ;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ { int ret = r_mark_nUn(z);
+ if (ret == 0) goto lab7; /* call mark_nUn, line 343 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 343 */
+ { int ret = slice_del(z); /* delete, line 343 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 343 */
+ z->ket = z->c; /* [, line 343 */
+ { int m3 = z->l - z->c; (void)m3; /* or, line 345 */
+ { int ret = r_mark_lArI(z);
+ if (ret == 0) goto lab10; /* call mark_lArI, line 344 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 344 */
+ { int ret = slice_del(z); /* delete, line 344 */
+ if (ret < 0) return ret;
+ }
+ goto lab9;
+ lab10:
+ z->c = z->l - m3;
+ z->ket = z->c; /* [, line 346 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 346 */
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) goto lab13; /* call mark_possessives, line 346 */
+ if (ret < 0) return ret;
+ }
+ goto lab12;
+ lab13:
+ z->c = z->l - m4;
+ { int ret = r_mark_sU(z);
+ if (ret == 0) goto lab11; /* call mark_sU, line 346 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab12:
+ z->bra = z->c; /* ], line 346 */
+ { int ret = slice_del(z); /* delete, line 346 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 346 */
+ z->ket = z->c; /* [, line 346 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab14; } /* call mark_lAr, line 346 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 346 */
+ { int ret = slice_del(z); /* delete, line 346 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab14; } /* call stem_suffix_chain_before_ki, line 346 */
+ if (ret < 0) return ret;
+ }
+ lab14:
+ ;
+ }
+ goto lab9;
+ lab11:
+ z->c = z->l - m3;
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab8; } /* call stem_suffix_chain_before_ki, line 348 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab9:
+ lab8:
+ ;
+ }
+ goto lab0;
+ lab7:
+ z->c = z->l - m1;
+ { int ret = r_mark_ndA(z);
+ if (ret == 0) return 0; /* call mark_ndA, line 351 */
+ if (ret < 0) return ret;
+ }
+ { int m5 = z->l - z->c; (void)m5; /* or, line 353 */
+ { int ret = r_mark_lArI(z);
+ if (ret == 0) goto lab16; /* call mark_lArI, line 352 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 352 */
+ { int ret = slice_del(z); /* delete, line 352 */
+ if (ret < 0) return ret;
+ }
+ goto lab15;
+ lab16:
+ z->c = z->l - m5;
+ { int ret = r_mark_sU(z);
+ if (ret == 0) goto lab17; /* call mark_sU, line 354 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 354 */
+ { int ret = slice_del(z); /* delete, line 354 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 354 */
+ z->ket = z->c; /* [, line 354 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab18; } /* call mark_lAr, line 354 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 354 */
+ { int ret = slice_del(z); /* delete, line 354 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab18; } /* call stem_suffix_chain_before_ki, line 354 */
+ if (ret < 0) return ret;
+ }
+ lab18:
+ ;
+ }
+ goto lab15;
+ lab17:
+ z->c = z->l - m5;
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) return 0; /* call stem_suffix_chain_before_ki, line 356 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab15:
+ ;
+ }
+lab0:
+ return 1;
+}
+
+static int r_stem_noun_suffixes(struct SN_env * z) {
+ { int m1 = z->l - z->c; (void)m1; /* or, line 363 */
+ z->ket = z->c; /* [, line 362 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab1; /* call mark_lAr, line 362 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 362 */
+ { int ret = slice_del(z); /* delete, line 362 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 362 */
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab2; } /* call stem_suffix_chain_before_ki, line 362 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ ;
+ }
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 364 */
+ { int ret = r_mark_ncA(z);
+ if (ret == 0) goto lab3; /* call mark_ncA, line 364 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 364 */
+ { int ret = slice_del(z); /* delete, line 364 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 365 */
+ { int m2 = z->l - z->c; (void)m2; /* or, line 367 */
+ z->ket = z->c; /* [, line 366 */
+ { int ret = r_mark_lArI(z);
+ if (ret == 0) goto lab6; /* call mark_lArI, line 366 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 366 */
+ { int ret = slice_del(z); /* delete, line 366 */
+ if (ret < 0) return ret;
+ }
+ goto lab5;
+ lab6:
+ z->c = z->l - m2;
+ z->ket = z->c; /* [, line 368 */
+ { int m3 = z->l - z->c; (void)m3; /* or, line 368 */
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) goto lab9; /* call mark_possessives, line 368 */
+ if (ret < 0) return ret;
+ }
+ goto lab8;
+ lab9:
+ z->c = z->l - m3;
+ { int ret = r_mark_sU(z);
+ if (ret == 0) goto lab7; /* call mark_sU, line 368 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab8:
+ z->bra = z->c; /* ], line 368 */
+ { int ret = slice_del(z); /* delete, line 368 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 368 */
+ z->ket = z->c; /* [, line 368 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab10; } /* call mark_lAr, line 368 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 368 */
+ { int ret = slice_del(z); /* delete, line 368 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab10; } /* call stem_suffix_chain_before_ki, line 368 */
+ if (ret < 0) return ret;
+ }
+ lab10:
+ ;
+ }
+ goto lab5;
+ lab7:
+ z->c = z->l - m2;
+ z->ket = z->c; /* [, line 370 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab4; } /* call mark_lAr, line 370 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 370 */
+ { int ret = slice_del(z); /* delete, line 370 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab4; } /* call stem_suffix_chain_before_ki, line 370 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab5:
+ lab4:
+ ;
+ }
+ goto lab0;
+ lab3:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 374 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 374 */
+ { int ret = r_mark_ndA(z);
+ if (ret == 0) goto lab13; /* call mark_ndA, line 374 */
+ if (ret < 0) return ret;
+ }
+ goto lab12;
+ lab13:
+ z->c = z->l - m4;
+ { int ret = r_mark_nA(z);
+ if (ret == 0) goto lab11; /* call mark_nA, line 374 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab12:
+ { int m5 = z->l - z->c; (void)m5; /* or, line 377 */
+ { int ret = r_mark_lArI(z);
+ if (ret == 0) goto lab15; /* call mark_lArI, line 376 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 376 */
+ { int ret = slice_del(z); /* delete, line 376 */
+ if (ret < 0) return ret;
+ }
+ goto lab14;
+ lab15:
+ z->c = z->l - m5;
+ { int ret = r_mark_sU(z);
+ if (ret == 0) goto lab16; /* call mark_sU, line 378 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 378 */
+ { int ret = slice_del(z); /* delete, line 378 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 378 */
+ z->ket = z->c; /* [, line 378 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab17; } /* call mark_lAr, line 378 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 378 */
+ { int ret = slice_del(z); /* delete, line 378 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab17; } /* call stem_suffix_chain_before_ki, line 378 */
+ if (ret < 0) return ret;
+ }
+ lab17:
+ ;
+ }
+ goto lab14;
+ lab16:
+ z->c = z->l - m5;
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) goto lab11; /* call stem_suffix_chain_before_ki, line 380 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab14:
+ goto lab0;
+ lab11:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 384 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 384 */
+ { int ret = r_mark_ndAn(z);
+ if (ret == 0) goto lab20; /* call mark_ndAn, line 384 */
+ if (ret < 0) return ret;
+ }
+ goto lab19;
+ lab20:
+ z->c = z->l - m6;
+ { int ret = r_mark_nU(z);
+ if (ret == 0) goto lab18; /* call mark_nU, line 384 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab19:
+ { int m7 = z->l - z->c; (void)m7; /* or, line 384 */
+ { int ret = r_mark_sU(z);
+ if (ret == 0) goto lab22; /* call mark_sU, line 384 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 384 */
+ { int ret = slice_del(z); /* delete, line 384 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 384 */
+ z->ket = z->c; /* [, line 384 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab23; } /* call mark_lAr, line 384 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 384 */
+ { int ret = slice_del(z); /* delete, line 384 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab23; } /* call stem_suffix_chain_before_ki, line 384 */
+ if (ret < 0) return ret;
+ }
+ lab23:
+ ;
+ }
+ goto lab21;
+ lab22:
+ z->c = z->l - m7;
+ { int ret = r_mark_lArI(z);
+ if (ret == 0) goto lab18; /* call mark_lArI, line 384 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab21:
+ goto lab0;
+ lab18:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 386 */
+ { int ret = r_mark_DAn(z);
+ if (ret == 0) goto lab24; /* call mark_DAn, line 386 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 386 */
+ { int ret = slice_del(z); /* delete, line 386 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 386 */
+ z->ket = z->c; /* [, line 386 */
+ { int m8 = z->l - z->c; (void)m8; /* or, line 389 */
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) goto lab27; /* call mark_possessives, line 388 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 388 */
+ { int ret = slice_del(z); /* delete, line 388 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 388 */
+ z->ket = z->c; /* [, line 388 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab28; } /* call mark_lAr, line 388 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 388 */
+ { int ret = slice_del(z); /* delete, line 388 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab28; } /* call stem_suffix_chain_before_ki, line 388 */
+ if (ret < 0) return ret;
+ }
+ lab28:
+ ;
+ }
+ goto lab26;
+ lab27:
+ z->c = z->l - m8;
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab29; /* call mark_lAr, line 390 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 390 */
+ { int ret = slice_del(z); /* delete, line 390 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 390 */
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab30; } /* call stem_suffix_chain_before_ki, line 390 */
+ if (ret < 0) return ret;
+ }
+ lab30:
+ ;
+ }
+ goto lab26;
+ lab29:
+ z->c = z->l - m8;
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab25; } /* call stem_suffix_chain_before_ki, line 392 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab26:
+ lab25:
+ ;
+ }
+ goto lab0;
+ lab24:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 396 */
+ { int m9 = z->l - z->c; (void)m9; /* or, line 396 */
+ { int ret = r_mark_nUn(z);
+ if (ret == 0) goto lab33; /* call mark_nUn, line 396 */
+ if (ret < 0) return ret;
+ }
+ goto lab32;
+ lab33:
+ z->c = z->l - m9;
+ { int ret = r_mark_ylA(z);
+ if (ret == 0) goto lab31; /* call mark_ylA, line 396 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab32:
+ z->bra = z->c; /* ], line 396 */
+ { int ret = slice_del(z); /* delete, line 396 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 397 */
+ { int m10 = z->l - z->c; (void)m10; /* or, line 399 */
+ z->ket = z->c; /* [, line 398 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) goto lab36; /* call mark_lAr, line 398 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 398 */
+ { int ret = slice_del(z); /* delete, line 398 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) goto lab36; /* call stem_suffix_chain_before_ki, line 398 */
+ if (ret < 0) return ret;
+ }
+ goto lab35;
+ lab36:
+ z->c = z->l - m10;
+ z->ket = z->c; /* [, line 400 */
+ { int m11 = z->l - z->c; (void)m11; /* or, line 400 */
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) goto lab39; /* call mark_possessives, line 400 */
+ if (ret < 0) return ret;
+ }
+ goto lab38;
+ lab39:
+ z->c = z->l - m11;
+ { int ret = r_mark_sU(z);
+ if (ret == 0) goto lab37; /* call mark_sU, line 400 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab38:
+ z->bra = z->c; /* ], line 400 */
+ { int ret = slice_del(z); /* delete, line 400 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 400 */
+ z->ket = z->c; /* [, line 400 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab40; } /* call mark_lAr, line 400 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 400 */
+ { int ret = slice_del(z); /* delete, line 400 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab40; } /* call stem_suffix_chain_before_ki, line 400 */
+ if (ret < 0) return ret;
+ }
+ lab40:
+ ;
+ }
+ goto lab35;
+ lab37:
+ z->c = z->l - m10;
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab34; } /* call stem_suffix_chain_before_ki, line 402 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab35:
+ lab34:
+ ;
+ }
+ goto lab0;
+ lab31:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 406 */
+ { int ret = r_mark_lArI(z);
+ if (ret == 0) goto lab41; /* call mark_lArI, line 406 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 406 */
+ { int ret = slice_del(z); /* delete, line 406 */
+ if (ret < 0) return ret;
+ }
+ goto lab0;
+ lab41:
+ z->c = z->l - m1;
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) goto lab42; /* call stem_suffix_chain_before_ki, line 408 */
+ if (ret < 0) return ret;
+ }
+ goto lab0;
+ lab42:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 410 */
+ { int m12 = z->l - z->c; (void)m12; /* or, line 410 */
+ { int ret = r_mark_DA(z);
+ if (ret == 0) goto lab45; /* call mark_DA, line 410 */
+ if (ret < 0) return ret;
+ }
+ goto lab44;
+ lab45:
+ z->c = z->l - m12;
+ { int ret = r_mark_yU(z);
+ if (ret == 0) goto lab46; /* call mark_yU, line 410 */
+ if (ret < 0) return ret;
+ }
+ goto lab44;
+ lab46:
+ z->c = z->l - m12;
+ { int ret = r_mark_yA(z);
+ if (ret == 0) goto lab43; /* call mark_yA, line 410 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab44:
+ z->bra = z->c; /* ], line 410 */
+ { int ret = slice_del(z); /* delete, line 410 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 410 */
+ z->ket = z->c; /* [, line 410 */
+ { int m13 = z->l - z->c; (void)m13; /* or, line 410 */
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) goto lab49; /* call mark_possessives, line 410 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 410 */
+ { int ret = slice_del(z); /* delete, line 410 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 410 */
+ z->ket = z->c; /* [, line 410 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab50; } /* call mark_lAr, line 410 */
+ if (ret < 0) return ret;
+ }
+ lab50:
+ ;
+ }
+ goto lab48;
+ lab49:
+ z->c = z->l - m13;
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab47; } /* call mark_lAr, line 410 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab48:
+ z->bra = z->c; /* ], line 410 */
+ { int ret = slice_del(z); /* delete, line 410 */
+ if (ret < 0) return ret;
+ }
+ z->ket = z->c; /* [, line 410 */
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab47; } /* call stem_suffix_chain_before_ki, line 410 */
+ if (ret < 0) return ret;
+ }
+ lab47:
+ ;
+ }
+ goto lab0;
+ lab43:
+ z->c = z->l - m1;
+ z->ket = z->c; /* [, line 412 */
+ { int m14 = z->l - z->c; (void)m14; /* or, line 412 */
+ { int ret = r_mark_possessives(z);
+ if (ret == 0) goto lab52; /* call mark_possessives, line 412 */
+ if (ret < 0) return ret;
+ }
+ goto lab51;
+ lab52:
+ z->c = z->l - m14;
+ { int ret = r_mark_sU(z);
+ if (ret == 0) return 0; /* call mark_sU, line 412 */
+ if (ret < 0) return ret;
+ }
+ }
+ lab51:
+ z->bra = z->c; /* ], line 412 */
+ { int ret = slice_del(z); /* delete, line 412 */
+ if (ret < 0) return ret;
+ }
+ { int m_keep = z->l - z->c;/* (void) m_keep;*/ /* try, line 412 */
+ z->ket = z->c; /* [, line 412 */
+ { int ret = r_mark_lAr(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab53; } /* call mark_lAr, line 412 */
+ if (ret < 0) return ret;
+ }
+ z->bra = z->c; /* ], line 412 */
+ { int ret = slice_del(z); /* delete, line 412 */
+ if (ret < 0) return ret;
+ }
+ { int ret = r_stem_suffix_chain_before_ki(z);
+ if (ret == 0) { z->c = z->l - m_keep; goto lab53; } /* call stem_suffix_chain_before_ki, line 412 */
+ if (ret < 0) return ret;
+ }
+ lab53:
+ ;
+ }
+ }
+lab0:
+ return 1;
+}
+
+static int r_post_process_last_consonants(struct SN_env * z) {
+ int among_var;
+ z->ket = z->c; /* [, line 416 */
+ among_var = find_among_b(z, a_23, 4); /* substring, line 416 */
+ if (!(among_var)) return 0;
+ z->bra = z->c; /* ], line 416 */
+ switch(among_var) {
+ case 0: return 0;
+ case 1:
+ { int ret = slice_from_s(z, 1, s_16); /* <-, line 417 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 2:
+ { int ret = slice_from_s(z, 2, s_17); /* <-, line 418 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 3:
+ { int ret = slice_from_s(z, 1, s_18); /* <-, line 419 */
+ if (ret < 0) return ret;
+ }
+ break;
+ case 4:
+ { int ret = slice_from_s(z, 1, s_19); /* <-, line 420 */
+ if (ret < 0) return ret;
+ }
+ break;
+ }
+ return 1;
+}
+
+static int r_append_U_to_stems_ending_with_d_or_g(struct SN_env * z) {
+ { int m_test = z->l - z->c; /* test, line 431 */
+ { int m1 = z->l - z->c; (void)m1; /* or, line 431 */
+ if (!(eq_s_b(z, 1, s_20))) goto lab1;
+ goto lab0;
+ lab1:
+ z->c = z->l - m1;
+ if (!(eq_s_b(z, 1, s_21))) return 0;
+ }
+ lab0:
+ z->c = z->l - m_test;
+ }
+ { int m2 = z->l - z->c; (void)m2; /* or, line 433 */
+ { int m_test = z->l - z->c; /* test, line 432 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 1) < 0) goto lab3; /* goto */ /* grouping vowel, line 432 */
+ { int m3 = z->l - z->c; (void)m3; /* or, line 432 */
+ if (!(eq_s_b(z, 1, s_22))) goto lab5;
+ goto lab4;
+ lab5:
+ z->c = z->l - m3;
+ if (!(eq_s_b(z, 2, s_23))) goto lab3;
+ }
+ lab4:
+ z->c = z->l - m_test;
+ }
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 2, s_24); /* <+, line 432 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab3:
+ z->c = z->l - m2;
+ { int m_test = z->l - z->c; /* test, line 434 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 1) < 0) goto lab6; /* goto */ /* grouping vowel, line 434 */
+ { int m4 = z->l - z->c; (void)m4; /* or, line 434 */
+ if (!(eq_s_b(z, 1, s_25))) goto lab8;
+ goto lab7;
+ lab8:
+ z->c = z->l - m4;
+ if (!(eq_s_b(z, 1, s_26))) goto lab6;
+ }
+ lab7:
+ z->c = z->l - m_test;
+ }
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 1, s_27); /* <+, line 434 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab6:
+ z->c = z->l - m2;
+ { int m_test = z->l - z->c; /* test, line 436 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 1) < 0) goto lab9; /* goto */ /* grouping vowel, line 436 */
+ { int m5 = z->l - z->c; (void)m5; /* or, line 436 */
+ if (!(eq_s_b(z, 1, s_28))) goto lab11;
+ goto lab10;
+ lab11:
+ z->c = z->l - m5;
+ if (!(eq_s_b(z, 1, s_29))) goto lab9;
+ }
+ lab10:
+ z->c = z->l - m_test;
+ }
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 1, s_30); /* <+, line 436 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ goto lab2;
+ lab9:
+ z->c = z->l - m2;
+ { int m_test = z->l - z->c; /* test, line 438 */
+ if (out_grouping_b_U(z, g_vowel, 97, 305, 1) < 0) return 0; /* goto */ /* grouping vowel, line 438 */
+ { int m6 = z->l - z->c; (void)m6; /* or, line 438 */
+ if (!(eq_s_b(z, 2, s_31))) goto lab13;
+ goto lab12;
+ lab13:
+ z->c = z->l - m6;
+ if (!(eq_s_b(z, 2, s_32))) return 0;
+ }
+ lab12:
+ z->c = z->l - m_test;
+ }
+ { int c_keep = z->c;
+ int ret = insert_s(z, z->c, z->c, 2, s_33); /* <+, line 438 */
+ z->c = c_keep;
+ if (ret < 0) return ret;
+ }
+ }
+lab2:
+ return 1;
+}
+
+static int r_more_than_one_syllable_word(struct SN_env * z) {
+ { int c_test = z->c; /* test, line 446 */
+ { int i = 2;
+ while(1) { /* atleast, line 446 */
+ int c1 = z->c;
+ { /* gopast */ /* grouping vowel, line 446 */
+ int ret = out_grouping_U(z, g_vowel, 97, 305, 1);
+ if (ret < 0) goto lab0;
+ z->c += ret;
+ }
+ i--;
+ continue;
+ lab0:
+ z->c = c1;
+ break;
+ }
+ if (i > 0) return 0;
+ }
+ z->c = c_test;
+ }
+ return 1;
+}
+
+static int r_is_reserved_word(struct SN_env * z) {
+ { int c1 = z->c; /* or, line 451 */
+ { int c_test = z->c; /* test, line 450 */
+ while(1) { /* gopast, line 450 */
+ if (!(eq_s(z, 2, s_34))) goto lab2;
+ break;
+ lab2:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) goto lab1;
+ z->c = ret; /* gopast, line 450 */
+ }
+ }
+ z->I[0] = 2;
+ if (!(z->I[0] == z->l)) goto lab1;
+ z->c = c_test;
+ }
+ goto lab0;
+ lab1:
+ z->c = c1;
+ { int c_test = z->c; /* test, line 452 */
+ while(1) { /* gopast, line 452 */
+ if (!(eq_s(z, 5, s_35))) goto lab3;
+ break;
+ lab3:
+ { int ret = skip_utf8(z->p, z->c, 0, z->l, 1);
+ if (ret < 0) return 0;
+ z->c = ret; /* gopast, line 452 */
+ }
+ }
+ z->I[0] = 5;
+ if (!(z->I[0] == z->l)) return 0;
+ z->c = c_test;
+ }
+ }
+lab0:
+ return 1;
+}
+
+static int r_postlude(struct SN_env * z) {
+ { int c1 = z->c; /* not, line 456 */
+ { int ret = r_is_reserved_word(z);
+ if (ret == 0) goto lab0; /* call is_reserved_word, line 456 */
+ if (ret < 0) return ret;
+ }
+ return 0;
+ lab0:
+ z->c = c1;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 457 */
+
+ { int m2 = z->l - z->c; (void)m2; /* do, line 458 */
+ { int ret = r_append_U_to_stems_ending_with_d_or_g(z);
+ if (ret == 0) goto lab1; /* call append_U_to_stems_ending_with_d_or_g, line 458 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ { int m3 = z->l - z->c; (void)m3; /* do, line 459 */
+ { int ret = r_post_process_last_consonants(z);
+ if (ret == 0) goto lab2; /* call post_process_last_consonants, line 459 */
+ if (ret < 0) return ret;
+ }
+ lab2:
+ z->c = z->l - m3;
+ }
+ z->c = z->lb;
+ return 1;
+}
+
+extern int turkish_UTF_8_stem(struct SN_env * z) {
+ { int ret = r_more_than_one_syllable_word(z);
+ if (ret == 0) return 0; /* call more_than_one_syllable_word, line 465 */
+ if (ret < 0) return ret;
+ }
+ z->lb = z->c; z->c = z->l; /* backwards, line 467 */
+
+ { int m1 = z->l - z->c; (void)m1; /* do, line 468 */
+ { int ret = r_stem_nominal_verb_suffixes(z);
+ if (ret == 0) goto lab0; /* call stem_nominal_verb_suffixes, line 468 */
+ if (ret < 0) return ret;
+ }
+ lab0:
+ z->c = z->l - m1;
+ }
+ if (!(z->B[0])) return 0; /* Boolean test continue_stemming_noun_suffixes, line 469 */
+ { int m2 = z->l - z->c; (void)m2; /* do, line 470 */
+ { int ret = r_stem_noun_suffixes(z);
+ if (ret == 0) goto lab1; /* call stem_noun_suffixes, line 470 */
+ if (ret < 0) return ret;
+ }
+ lab1:
+ z->c = z->l - m2;
+ }
+ z->c = z->lb;
+ { int ret = r_postlude(z);
+ if (ret == 0) return 0; /* call postlude, line 473 */
+ if (ret < 0) return ret;
+ }
+ return 1;
+}
+
+extern struct SN_env * turkish_UTF_8_create_env(void) { return SN_create_env(0, 1, 1); }
+
+extern void turkish_UTF_8_close_env(struct SN_env * z) { SN_close_env(z, 0); }
+
diff --git a/modules/analysis/snowstem/source/src_c/stem_UTF_8_turkish.h b/modules/analysis/snowstem/source/src_c/stem_UTF_8_turkish.h
new file mode 100644
index 0000000..8173a17
--- /dev/null
+++ b/modules/analysis/snowstem/source/src_c/stem_UTF_8_turkish.h
@@ -0,0 +1,16 @@
+
+/* This file was generated automatically by the Snowball to ANSI C compiler */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern struct SN_env * turkish_UTF_8_create_env(void);
+extern void turkish_UTF_8_close_env(struct SN_env * z);
+
+extern int turkish_UTF_8_stem(struct SN_env * z);
+
+#ifdef __cplusplus
+}
+#endif
+
diff --git a/modules/analysis/snowstem/source/test/README b/modules/analysis/snowstem/source/test/README
new file mode 100644
index 0000000..f15012c
--- /dev/null
+++ b/modules/analysis/snowstem/source/test/README
@@ -0,0 +1,3 @@
+The file 'tests.json' and this file were autogenerated by update_snowstem.pl.
+'tests.json' contains materials from the Snowball project. See the LICENSE
+and NOTICE files for more information.
diff --git a/modules/analysis/snowstem/source/test/tests.json b/modules/analysis/snowstem/source/test/tests.json
new file mode 100644
index 0000000..469b0f3
--- /dev/null
+++ b/modules/analysis/snowstem/source/test/tests.json
@@ -0,0 +1,392 @@
+{
+ "da" : {
+ "stems" : [
+ "a",
+ "blot",
+ "find",
+ "grusdyng",
+ "indsigtsfuld",
+ "lid",
+ "oks",
+ "rør",
+ "stor",
+ "udbred"
+ ],
+ "words" : [
+ "a",
+ "blotter",
+ "find",
+ "grusdyngerne",
+ "indsigtsfulde",
+ "lidet",
+ "okse",
+ "røres",
+ "storheden",
+ "udbredtes"
+ ]
+ },
+ "de" : {
+ "stems" : [
+ "a",
+ "befestigt",
+ "edl",
+ "froscharm",
+ "hau",
+ "kontrakt",
+ "new",
+ "schmachtig",
+ "trink",
+ "vitalitat"
+ ],
+ "words" : [
+ "a",
+ "befestigte",
+ "edlen",
+ "froscharme",
+ "hau",
+ "kontrakt",
+ "new",
+ "schmächtig",
+ "trinken",
+ "vitalitat"
+ ]
+ },
+ "en" : {
+ "stems" : [
+ "'",
+ "borough",
+ "cours",
+ "enlarg",
+ "growl",
+ "labour",
+ "oblivion",
+ "quagmir",
+ "silurus",
+ "tori"
+ ],
+ "words" : [
+ "'",
+ "borough",
+ "courses",
+ "enlarging",
+ "growling",
+ "labourers",
+ "oblivion",
+ "quagmire",
+ "silurus",
+ "tory"
+ ]
+ },
+ "es" : {
+ "stems" : [
+ "a",
+ "autent",
+ "coment",
+ "desenton",
+ "eventual",
+ "implement",
+ "march",
+ "pas",
+ "reconozc",
+ "suced"
+ ],
+ "words" : [
+ "a",
+ "auténtico",
+ "comentó",
+ "desentonados",
+ "eventuales",
+ "implementar",
+ "marcharían",
+ "pasando",
+ "reconozcan",
+ "sucedido"
+ ]
+ },
+ "fi" : {
+ "stems" : [
+ "aa",
+ "allekirjoit",
+ "aset",
+ "bonnier",
+ "eduskuntaryhm",
+ "erikoissairaanhoido",
+ "fält",
+ "harper",
+ "hirsin",
+ "hyökän"
+ ],
+ "words" : [
+ "aa",
+ "allekirjoitti",
+ "aseteta",
+ "bonnierin",
+ "eduskuntaryhmissä",
+ "erikoissairaanhoidossa",
+ "fält",
+ "harper",
+ "hirsinen",
+ "hyökänneen"
+ ]
+ },
+ "fr" : {
+ "stems" : [
+ "a",
+ "bar",
+ "conférent",
+ "disput",
+ "expans",
+ "impuiss",
+ "mérit",
+ "pétrifi",
+ "rel",
+ "souvent"
+ ],
+ "words" : [
+ "a",
+ "bar",
+ "conférences",
+ "dispute",
+ "expansif",
+ "impuissants",
+ "méritait",
+ "pétrifié",
+ "relis",
+ "souvent"
+ ]
+ },
+ "hu" : {
+ "stems" : [
+ "abazin",
+ "bajlódt",
+ "bombázás",
+ "kar",
+ "kirángatt",
+ "kutya",
+ "kötvény",
+ "lovard",
+ "meghívás",
+ "minisztérium"
+ ],
+ "words" : [
+ "abazinok",
+ "bajlódtam",
+ "bombázása",
+ "karddal",
+ "kirángattam",
+ "kutyánk",
+ "kötvényeik",
+ "lovardába",
+ "meghívás",
+ "minisztériumokban"
+ ]
+ },
+ "it" : {
+ "stems" : [
+ "a",
+ "avvi",
+ "compless",
+ "donatell",
+ "got",
+ "letterar",
+ "orribil",
+ "r",
+ "scans",
+ "suggell"
+ ],
+ "words" : [
+ "a",
+ "avviarono",
+ "complessive",
+ "donatella",
+ "gote",
+ "letterario",
+ "orribili",
+ "r",
+ "scansava",
+ "suggella"
+ ]
+ },
+ "nl" : {
+ "stems" : [
+ "a",
+ "betreft",
+ "dommel",
+ "geparkeerd",
+ "inwinn",
+ "meegat",
+ "oorspronk",
+ "rimpel",
+ "telefonisch",
+ "vierspor"
+ ],
+ "words" : [
+ "a",
+ "betreft",
+ "dommel",
+ "geparkeerd",
+ "inwinnen",
+ "meegaat",
+ "oorspronkelijkheid",
+ "rimpelig",
+ "telefonisch",
+ "viersporig"
+ ]
+ },
+ "no" : {
+ "stems" : [
+ "a",
+ "budskap",
+ "fiend",
+ "gradvis",
+ "italiensk",
+ "lovutk",
+ "oppnemn",
+ "risikomoment",
+ "status",
+ "tyv"
+ ],
+ "words" : [
+ "a",
+ "budskapet",
+ "fiender",
+ "gradvis",
+ "italienske",
+ "lovutkast",
+ "oppnemnast",
+ "risikomomentet",
+ "statusen",
+ "tyv"
+ ]
+ },
+ "pt" : {
+ "stems" : [
+ "a",
+ "autor",
+ "coloc",
+ "desmont",
+ "estre",
+ "honr",
+ "macdowell",
+ "pag",
+ "reag",
+ "subsídi"
+ ],
+ "words" : [
+ "a",
+ "autora",
+ "coloco",
+ "desmontaram",
+ "estreou",
+ "honram",
+ "macdowell",
+ "pagando",
+ "reagem",
+ "subsídios"
+ ]
+ },
+ "ro" : {
+ "stems" : [
+ "a",
+ "boaş",
+ "corespondenţ",
+ "etichet",
+ "incertitudin",
+ "lea",
+ "nepot",
+ "povârniş",
+ "sancţiun",
+ "tach"
+ ],
+ "words" : [
+ "a",
+ "boaşele",
+ "corespondenţă",
+ "etichetelor",
+ "incertitudinilor",
+ "lea",
+ "nepotul",
+ "povârnişul",
+ "sancţiuni",
+ "tache"
+ ]
+ },
+ "ru" : {
+ "stems" : [
+ "а",
+ "всем",
+ "жалк",
+ "клад",
+ "накрахмален",
+ "осил",
+ "полезн",
+ "противоположн",
+ "сладчайш",
+ "тщеслав"
+ ],
+ "words" : [
+ "а",
+ "всеми",
+ "жалким",
+ "кладя",
+ "накрахмаленные",
+ "осилила",
+ "полезны",
+ "противоположность",
+ "сладчайший",
+ "тщеславен"
+ ]
+ },
+ "sv" : {
+ "stems" : [
+ "a",
+ "brist",
+ "fot",
+ "gull",
+ "kall",
+ "låt",
+ "otänkbart",
+ "sjuhundr",
+ "sval",
+ "upptänk"
+ ],
+ "words" : [
+ "a",
+ "brister",
+ "fots",
+ "gull",
+ "kalla",
+ "låtelse",
+ "otänkbart",
+ "sjuhundrade",
+ "svalor",
+ "upptänkligt"
+ ]
+ },
+ "tr" : {
+ "stems" : [
+ "a",
+ "başgöstere",
+ "değil",
+ "gazze'ye",
+ "ilginçlik",
+ "kopma",
+ "nmzdek",
+ "selahiyet",
+ "turizm",
+ "yön"
+ ],
+ "words" : [
+ "a",
+ "başgösteren",
+ "değillerdir",
+ "gazze'ye",
+ "ilginçliğinden",
+ "kopmaya",
+ "nmzdeki",
+ "selahiyetleri",
+ "turizmine",
+ "yönler"
+ ]
+ }
+}
diff --git a/modules/analysis/snowstop/devel/update_snowstop.pl b/modules/analysis/snowstop/devel/update_snowstop.pl
new file mode 100644
index 0000000..8014fa0
--- /dev/null
+++ b/modules/analysis/snowstop/devel/update_snowstop.pl
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use File::Spec::Functions qw( catfile catdir );
+use Encode qw( encode );
+use Text::Wrap qw( wrap );
+
+# Don't use tabs. Wrap at 78 columns.
+$Text::Wrap::unexpand = 0;
+$Text::Wrap::columns = 78;
+
+if ( @ARGV != 2 ) {
+ die "Usage: perl update_snowstop.pl SNOWBALL_SVN_CO LUCY_SNOWSTOP_DIR";
+}
+my ( $snow_co_dir, $dest_dir ) = @ARGV;
+
+# Update to a particular rev of the Snowball repository.
+die("Not a directory: '$snow_co_dir'") unless -d $snow_co_dir;
+my $retval = system( "svn", "update", "-r", "541", $snow_co_dir );
+die "svn update failed" if ( $retval >> 8 );
+
+# Open destination C file and print start of file.
+my $outpath = catfile( $dest_dir, 'source', 'snowball_stoplists.c' );
+open( my $out_fh, '>', $outpath ) or die "Can't open '$outpath': $!";
+print $out_fh <<'END_STUFF';
+/* Auto-generated file -- DO NOT EDIT!
+ *
+ * The words in this file are taken from stoplists provided by the Snowball
+ * project.
+ */
+
+#include "Lucy/Analysis/SnowballStopFilter.h"
+
+END_STUFF
+
+my %languages = (
+ da => "danish",
+ de => "german",
+ en => "english",
+ es => "spanish",
+ fi => "finnish",
+ fr => "french",
+ hu => "hungarian",
+ it => "italian",
+ nl => "dutch",
+ no => "norwegian",
+ pt => "portuguese",
+ ru => "russian",
+ sv => "swedish",
+);
+
+for my $iso ( sort keys %languages ) {
+ my $language = $languages{$iso};
+
+ # Grab stoplists from Snowball source files.
+ my $stop_path = "$snow_co_dir/website/algorithms/$language/stop.txt";
+ my $source_enc = $iso eq 'ru' ? 'koi8-r' : 'iso-8859-1';
+ open( my $stopfile_fh, "<:encoding($source_enc)", $stop_path )
+ or die "Couldn't open file '$stop_path': $!";
+ my @words;
+ while ( defined( my $line = <$stopfile_fh> ) ) {
+ $line =~ s/\|.*//g;
+ next unless length($line);
+ push @words, split( /\s+/, $line );
+ }
+
+ # Encode as UTF-8, change all non-ASCII bytes to octal escapes, and format
+ # as C string literals.
+ my @escaped = map { '"' . encode( 'UTF-8', $_ ) . '"' } @words;
+ s/([\x80-\xFF])/octal_escape($1)/ge for @escaped;
+
+ # Wrap text and print to outfile.
+ my $joined = join( ', ', @escaped, 'NULL' );
+ my $wrapped = wrap( ' ', ' ', $joined );
+ print $out_fh <<END_STUFF;
+static const char *words_${iso}[] = {
+$wrapped
+};
+const uint8_t **lucy_SnowStop_snow_${iso} = (const uint8_t**)words_$iso;
+
+END_STUFF
+}
+
+sub octal_escape {
+ my $ord = ord( $_[0] );
+ return sprintf( "\\%.3o", $ord );
+}
+
diff --git a/modules/analysis/snowstop/source/snowball_stoplists.c b/modules/analysis/snowstop/source/snowball_stoplists.c
new file mode 100644
index 0000000..e24cab9
--- /dev/null
+++ b/modules/analysis/snowstop/source/snowball_stoplists.c
@@ -0,0 +1,488 @@
+/* Auto-generated file -- DO NOT EDIT!
+ *
+ * The words in this file are taken from stoplists provided by the Snowball
+ * project.
+ */
+
+#include "Lucy/Analysis/SnowballStopFilter.h"
+
+static const char *words_da[] = {
+ "og", "i", "jeg", "det", "at", "en", "den", "til", "er", "som",
+ "p\303\245", "de", "med", "han", "af", "for", "ikke", "der", "var",
+ "mig", "sig", "men", "et", "har", "om", "vi", "min", "havde", "ham",
+ "hun", "nu", "over", "da", "fra", "du", "ud", "sin", "dem", "os", "op",
+ "man", "hans", "hvor", "eller", "hvad", "skal", "selv", "her", "alle",
+ "vil", "blev", "kunne", "ind", "n\303\245r", "v\303\246re", "dog",
+ "noget", "ville", "jo", "deres", "efter", "ned", "skulle", "denne",
+ "end", "dette", "mit", "ogs\303\245", "under", "have", "dig", "anden",
+ "hende", "mine", "alt", "meget", "sit", "sine", "vor", "mod", "disse",
+ "hvis", "din", "nogle", "hos", "blive", "mange", "ad", "bliver",
+ "hendes", "v\303\246ret", "thi", "jer", "s\303\245dan", NULL
+};
+const uint8_t **lucy_SnowStop_snow_da = (const uint8_t**)words_da;
+
+static const char *words_de[] = {
+ "aber", "alle", "allem", "allen", "aller", "alles", "als", "also", "am",
+ "an", "ander", "andere", "anderem", "anderen", "anderer", "anderes",
+ "anderm", "andern", "anderr", "anders", "auch", "auf", "aus", "bei",
+ "bin", "bis", "bist", "da", "damit", "dann", "der", "den", "des", "dem",
+ "die", "das", "da\303\237", "derselbe", "derselben", "denselben",
+ "desselben", "demselben", "dieselbe", "dieselben", "dasselbe", "dazu",
+ "dein", "deine", "deinem", "deinen", "deiner", "deines", "denn", "derer",
+ "dessen", "dich", "dir", "du", "dies", "diese", "diesem", "diesen",
+ "dieser", "dieses", "doch", "dort", "durch", "ein", "eine", "einem",
+ "einen", "einer", "eines", "einig", "einige", "einigem", "einigen",
+ "einiger", "einiges", "einmal", "er", "ihn", "ihm", "es", "etwas",
+ "euer", "eure", "eurem", "euren", "eurer", "eures", "f\303\274r",
+ "gegen", "gewesen", "hab", "habe", "haben", "hat", "hatte", "hatten",
+ "hier", "hin", "hinter", "ich", "mich", "mir", "ihr", "ihre", "ihrem",
+ "ihren", "ihrer", "ihres", "euch", "im", "in", "indem", "ins", "ist",
+ "jede", "jedem", "jeden", "jeder", "jedes", "jene", "jenem", "jenen",
+ "jener", "jenes", "jetzt", "kann", "kein", "keine", "keinem", "keinen",
+ "keiner", "keines", "k\303\266nnen", "k\303\266nnte", "machen", "man",
+ "manche", "manchem", "manchen", "mancher", "manches", "mein", "meine",
+ "meinem", "meinen", "meiner", "meines", "mit", "muss", "musste", "nach",
+ "nicht", "nichts", "noch", "nun", "nur", "ob", "oder", "ohne", "sehr",
+ "sein", "seine", "seinem", "seinen", "seiner", "seines", "selbst",
+ "sich", "sie", "ihnen", "sind", "so", "solche", "solchem", "solchen",
+ "solcher", "solches", "soll", "sollte", "sondern", "sonst",
+ "\303\274ber", "um", "und", "uns", "unse", "unsem", "unsen", "unser",
+ "unses", "unter", "viel", "vom", "von", "vor", "w\303\244hrend", "war",
+ "waren", "warst", "was", "weg", "weil", "weiter", "welche", "welchem",
+ "welchen", "welcher", "welches", "wenn", "werde", "werden", "wie",
+ "wieder", "will", "wir", "wird", "wirst", "wo", "wollen", "wollte",
+ "w\303\274rde", "w\303\274rden", "zu", "zum", "zur", "zwar", "zwischen",
+ NULL
+};
+const uint8_t **lucy_SnowStop_snow_de = (const uint8_t**)words_de;
+
+static const char *words_en[] = {
+ "i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you",
+ "your", "yours", "yourself", "yourselves", "he", "him", "his", "himself",
+ "she", "her", "hers", "herself", "it", "its", "itself", "they", "them",
+ "their", "theirs", "themselves", "what", "which", "who", "whom", "this",
+ "that", "these", "those", "am", "is", "are", "was", "were", "be", "been",
+ "being", "have", "has", "had", "having", "do", "does", "did", "doing",
+ "would", "should", "could", "ought", "i'm", "you're", "he's", "she's",
+ "it's", "we're", "they're", "i've", "you've", "we've", "they've", "i'd",
+ "you'd", "he'd", "she'd", "we'd", "they'd", "i'll", "you'll", "he'll",
+ "she'll", "we'll", "they'll", "isn't", "aren't", "wasn't", "weren't",
+ "hasn't", "haven't", "hadn't", "doesn't", "don't", "didn't", "won't",
+ "wouldn't", "shan't", "shouldn't", "can't", "cannot", "couldn't",
+ "mustn't", "let's", "that's", "who's", "what's", "here's", "there's",
+ "when's", "where's", "why's", "how's", "a", "an", "the", "and", "but",
+ "if", "or", "because", "as", "until", "while", "of", "at", "by", "for",
+ "with", "about", "against", "between", "into", "through", "during",
+ "before", "after", "above", "below", "to", "from", "up", "down", "in",
+ "out", "on", "off", "over", "under", "again", "further", "then", "once",
+ "here", "there", "when", "where", "why", "how", "all", "any", "both",
+ "each", "few", "more", "most", "other", "some", "such", "no", "nor",
+ "not", "only", "own", "same", "so", "than", "too", "very", NULL
+};
+const uint8_t **lucy_SnowStop_snow_en = (const uint8_t**)words_en;
+
+static const char *words_es[] = {
+ "de", "la", "que", "el", "en", "y", "a", "los", "del", "se", "las",
+ "por", "un", "para", "con", "no", "una", "su", "al", "lo", "como",
+ "m\303\241s", "pero", "sus", "le", "ya", "o", "este", "s\303\255",
+ "porque", "esta", "entre", "cuando", "muy", "sin", "sobre",
+ "tambi\303\251n", "me", "hasta", "hay", "donde", "quien", "desde",
+ "todo", "nos", "durante", "todos", "uno", "les", "ni", "contra", "otros",
+ "ese", "eso", "ante", "ellos", "e", "esto", "m\303\255", "antes",
+ "algunos", "qu\303\251", "unos", "yo", "otro", "otras", "otra",
+ "\303\251l", "tanto", "esa", "estos", "mucho", "quienes", "nada",
+ "muchos", "cual", "poco", "ella", "estar", "estas", "algunas", "algo",
+ "nosotros", "mi", "mis", "t\303\272", "te", "ti", "tu", "tus", "ellas",
+ "nosotras", "vosotros", "vosotras", "os", "m\303\255o", "m\303\255a",
+ "m\303\255os", "m\303\255as", "tuyo", "tuya", "tuyos", "tuyas", "suyo",
+ "suya", "suyos", "suyas", "nuestro", "nuestra", "nuestros", "nuestras",
+ "vuestro", "vuestra", "vuestros", "vuestras", "esos", "esas", "estoy",
+ "est\303\241s", "est\303\241", "estamos", "est\303\241is",
+ "est\303\241n", "est\303\251", "est\303\251s", "estemos",
+ "est\303\251is", "est\303\251n", "estar\303\251", "estar\303\241s",
+ "estar\303\241", "estaremos", "estar\303\251is", "estar\303\241n",
+ "estar\303\255a", "estar\303\255as", "estar\303\255amos",
+ "estar\303\255ais", "estar\303\255an", "estaba", "estabas",
+ "est\303\241bamos", "estabais", "estaban", "estuve", "estuviste",
+ "estuvo", "estuvimos", "estuvisteis", "estuvieron", "estuviera",
+ "estuvieras", "estuvi\303\251ramos", "estuvierais", "estuvieran",
+ "estuviese", "estuvieses", "estuvi\303\251semos", "estuvieseis",
+ "estuviesen", "estando", "estado", "estada", "estados", "estadas",
+ "estad", "he", "has", "ha", "hemos", "hab\303\251is", "han", "haya",
+ "hayas", "hayamos", "hay\303\241is", "hayan", "habr\303\251",
+ "habr\303\241s", "habr\303\241", "habremos", "habr\303\251is",
+ "habr\303\241n", "habr\303\255a", "habr\303\255as", "habr\303\255amos",
+ "habr\303\255ais", "habr\303\255an", "hab\303\255a", "hab\303\255as",
+ "hab\303\255amos", "hab\303\255ais", "hab\303\255an", "hube", "hubiste",
+ "hubo", "hubimos", "hubisteis", "hubieron", "hubiera", "hubieras",
+ "hubi\303\251ramos", "hubierais", "hubieran", "hubiese", "hubieses",
+ "hubi\303\251semos", "hubieseis", "hubiesen", "habiendo", "habido",
+ "habida", "habidos", "habidas", "soy", "eres", "es", "somos", "sois",
+ "son", "sea", "seas", "seamos", "se\303\241is", "sean", "ser\303\251",
+ "ser\303\241s", "ser\303\241", "seremos", "ser\303\251is",
+ "ser\303\241n", "ser\303\255a", "ser\303\255as", "ser\303\255amos",
+ "ser\303\255ais", "ser\303\255an", "era", "eras", "\303\251ramos",
+ "erais", "eran", "fui", "fuiste", "fue", "fuimos", "fuisteis", "fueron",
+ "fuera", "fueras", "fu\303\251ramos", "fuerais", "fueran", "fuese",
+ "fueses", "fu\303\251semos", "fueseis", "fuesen", "siendo", "sido",
+ "tengo", "tienes", "tiene", "tenemos", "ten\303\251is", "tienen",
+ "tenga", "tengas", "tengamos", "teng\303\241is", "tengan",
+ "tendr\303\251", "tendr\303\241s", "tendr\303\241", "tendremos",
+ "tendr\303\251is", "tendr\303\241n", "tendr\303\255a", "tendr\303\255as",
+ "tendr\303\255amos", "tendr\303\255ais", "tendr\303\255an",
+ "ten\303\255a", "ten\303\255as", "ten\303\255amos", "ten\303\255ais",
+ "ten\303\255an", "tuve", "tuviste", "tuvo", "tuvimos", "tuvisteis",
+ "tuvieron", "tuviera", "tuvieras", "tuvi\303\251ramos", "tuvierais",
+ "tuvieran", "tuviese", "tuvieses", "tuvi\303\251semos", "tuvieseis",
+ "tuviesen", "teniendo", "tenido", "tenida", "tenidos", "tenidas",
+ "tened", NULL
+};
+const uint8_t **lucy_SnowStop_snow_es = (const uint8_t**)words_es;
+
+static const char *words_fi[] = {
+ "olla", "olen", "olet", "on", "olemme", "olette", "ovat", "ole", "oli",
+ "olisi", "olisit", "olisin", "olisimme", "olisitte", "olisivat", "olit",
+ "olin", "olimme", "olitte", "olivat", "ollut", "olleet", "en", "et",
+ "ei", "emme", "ette", "eiv\303\244t", "min\303\244", "minun", "minut",
+ "minua", "minussa", "minusta", "minuun", "minulla", "minulta", "minulle",
+ "sin\303\244", "sinun", "sinut", "sinua", "sinussa", "sinusta", "sinuun",
+ "sinulla", "sinulta", "sinulle", "h\303\244n", "h\303\244nen",
+ "h\303\244net", "h\303\244nt\303\244", "h\303\244ness\303\244",
+ "h\303\244nest\303\244", "h\303\244neen", "h\303\244nell\303\244",
+ "h\303\244nelt\303\244", "h\303\244nelle", "me", "meid\303\244n",
+ "meid\303\244t", "meit\303\244", "meiss\303\244", "meist\303\244",
+ "meihin", "meill\303\244", "meilt\303\244", "meille", "te",
+ "teid\303\244n", "teid\303\244t", "teit\303\244", "teiss\303\244",
+ "teist\303\244", "teihin", "teill\303\244", "teilt\303\244", "teille",
+ "he", "heid\303\244n", "heid\303\244t", "heit\303\244", "heiss\303\244",
+ "heist\303\244", "heihin", "heill\303\244", "heilt\303\244", "heille",
+ "t\303\244m\303\244", "t\303\244m\303\244n", "t\303\244t\303\244",
+ "t\303\244ss\303\244", "t\303\244st\303\244", "t\303\244h\303\244n",
+ "tall\303\244", "t\303\244lt\303\244", "t\303\244lle",
+ "t\303\244n\303\244", "t\303\244ksi", "tuo", "tuon", "tuot\303\244",
+ "tuossa", "tuosta", "tuohon", "tuolla", "tuolta", "tuolle", "tuona",
+ "tuoksi", "se", "sen", "sit\303\244", "siin\303\244", "siit\303\244",
+ "siihen", "sill\303\244", "silt\303\244", "sille", "sin\303\244",
+ "siksi", "n\303\244m\303\244", "n\303\244iden", "n\303\244it\303\244",
+ "n\303\244iss\303\244", "n\303\244ist\303\244", "n\303\244ihin",
+ "n\303\244ill\303\244", "n\303\244ilt\303\244", "n\303\244ille",
+ "n\303\244in\303\244", "n\303\244iksi", "nuo", "noiden", "noita",
+ "noissa", "noista", "noihin", "noilla", "noilta", "noille", "noina",
+ "noiksi", "ne", "niiden", "niit\303\244", "niiss\303\244",
+ "niist\303\244", "niihin", "niill\303\244", "niilt\303\244", "niille",
+ "niin\303\244", "niiksi", "kuka", "kenen", "kenet", "ket\303\244",
+ "keness\303\244", "kenest\303\244", "keneen", "kenell\303\244",
+ "kenelt\303\244", "kenelle", "kenen\303\244", "keneksi", "ketk\303\244",
+ "keiden", "ketk\303\244", "keit\303\244", "keiss\303\244",
+ "keist\303\244", "keihin", "keill\303\244", "keilt\303\244", "keille",
+ "kein\303\244", "keiksi", "mik\303\244", "mink\303\244", "mink\303\244",
+ "mit\303\244", "miss\303\244", "mist\303\244", "mihin", "mill\303\244",
+ "milt\303\244", "mille", "min\303\244", "miksi", "mitk\303\244", "joka",
+ "jonka", "jota", "jossa", "josta", "johon", "jolla", "jolta", "jolle",
+ "jona", "joksi", "jotka", "joiden", "joita", "joissa", "joista",
+ "joihin", "joilla", "joilta", "joille", "joina", "joiksi", "ett\303\244",
+ "ja", "jos", "koska", "kuin", "mutta", "niin", "sek\303\244",
+ "sill\303\244", "tai", "vaan", "vai", "vaikka", "kanssa", "mukaan",
+ "noin", "poikki", "yli", "kun", "niin", "nyt", "itse", NULL
+};
+const uint8_t **lucy_SnowStop_snow_fi = (const uint8_t**)words_fi;
+
+static const char *words_fr[] = {
+ "au", "aux", "avec", "ce", "ces", "dans", "de", "des", "du", "elle",
+ "en", "et", "eux", "il", "je", "la", "le", "leur", "lui", "ma", "mais",
+ "me", "m\303\252me", "mes", "moi", "mon", "ne", "nos", "notre", "nous",
+ "on", "ou", "par", "pas", "pour", "qu", "que", "qui", "sa", "se", "ses",
+ "son", "sur", "ta", "te", "tes", "toi", "ton", "tu", "un", "une", "vos",
+ "votre", "vous", "c", "d", "j", "l", "\303\240", "m", "n", "s", "t", "y",
+ "\303\251t\303\251", "\303\251t\303\251e", "\303\251t\303\251es",
+ "\303\251t\303\251s", "\303\251tant", "suis", "es", "est", "sommes",
+ "\303\252tes", "sont", "serai", "seras", "sera", "serons", "serez",
+ "seront", "serais", "serait", "serions", "seriez", "seraient",
+ "\303\251tais", "\303\251tait", "\303\251tions", "\303\251tiez",
+ "\303\251taient", "fus", "fut", "f\303\273mes", "f\303\273tes", "furent",
+ "sois", "soit", "soyons", "soyez", "soient", "fusse", "fusses",
+ "f\303\273t", "fussions", "fussiez", "fussent", "ayant", "eu", "eue",
+ "eues", "eus", "ai", "as", "avons", "avez", "ont", "aurai", "auras",
+ "aura", "aurons", "aurez", "auront", "aurais", "aurait", "aurions",
+ "auriez", "auraient", "avais", "avait", "avions", "aviez", "avaient",
+ "eut", "e\303\273mes", "e\303\273tes", "eurent", "aie", "aies", "ait",
+ "ayons", "ayez", "aient", "eusse", "eusses", "e\303\273t", "eussions",
+ "eussiez", "eussent", "ceci", "cel\303\240", "cet", "cette", "ici",
+ "ils", "les", "leurs", "quel", "quels", "quelle", "quelles", "sans",
+ "soi", NULL
+};
+const uint8_t **lucy_SnowStop_snow_fr = (const uint8_t**)words_fr;
+
+static const char *words_hu[] = {
+ "a", "ahogy", "ahol", "aki", "akik", "akkor", "alatt", "\303\241ltal",
+ "\303\241ltal\303\241ban", "amely", "amelyek", "amelyekben", "amelyeket",
+ "amelyet", "amelynek", "ami", "amit", "amolyan", "am\303\255g", "amikor",
+ "\303\241t", "abban", "ahhoz", "annak", "arra", "arr\303\263l", "az",
+ "azok", "azon", "azt", "azzal", "az\303\251rt", "azt\303\241n",
+ "azut\303\241n", "azonban", "b\303\241r", "be", "bel\303\274l", "benne",
+ "cikk", "cikkek", "cikkeket", "csak", "de", "e", "eddig", "eg\303\251sz",
+ "egy", "egyes", "egyetlen", "egy\303\251b", "egyik", "egyre", "ekkor",
+ "el", "el\303\251g", "ellen", "el\303\265", "el\303\265sz\303\266r",
+ "el\303\265tt", "els\303\265", "\303\251n", "\303\251ppen", "ebben",
+ "ehhez", "emilyen", "ennek", "erre", "ez", "ezt", "ezek", "ezen",
+ "ezzel", "ez\303\251rt", "\303\251s", "fel", "fel\303\251", "hanem",
+ "hiszen", "hogy", "hogyan", "igen", "\303\255gy", "illetve", "ill.",
+ "ill", "ilyen", "ilyenkor", "ison", "ism\303\251t", "itt", "j\303\263",
+ "j\303\263l", "jobban", "kell", "kellett", "kereszt\303\274l",
+ "keress\303\274nk", "ki", "k\303\255v\303\274l", "k\303\266z\303\266tt",
+ "k\303\266z\303\274l", "legal\303\241bb", "lehet", "lehetett", "legyen",
+ "lenne", "lenni", "lesz", "lett", "maga", "mag\303\241t", "majd", "majd",
+ "m\303\241r", "m\303\241s", "m\303\241sik", "meg", "m\303\251g",
+ "mellett", "mert", "mely", "melyek", "mi", "mit", "m\303\255g",
+ "mi\303\251rt", "milyen", "mikor", "minden", "mindent", "mindenki",
+ "mindig", "mint", "mintha", "mivel", "most", "nagy", "nagyobb", "nagyon",
+ "ne", "n\303\251ha", "nekem", "neki", "nem", "n\303\251h\303\241ny",
+ "n\303\251lk\303\274l", "nincs", "olyan", "ott", "\303\266ssze",
+ "\303\265", "\303\265k", "\303\265ket", "pedig", "persze", "r\303\241",
+ "s", "saj\303\241t", "sem", "semmi", "sok", "sokat", "sokkal",
+ "sz\303\241m\303\241ra", "szemben", "szerint", "szinte", "tal\303\241n",
+ "teh\303\241t", "teljes", "tov\303\241bb", "tov\303\241bb\303\241",
+ "t\303\266bb", "\303\272gy", "ugyanis", "\303\272j", "\303\272jabb",
+ "\303\272jra", "ut\303\241n", "ut\303\241na", "utols\303\263", "vagy",
+ "vagyis", "valaki", "valami", "valamint", "val\303\263", "vagyok", "van",
+ "vannak", "volt", "voltam", "voltak", "voltunk", "vissza", "vele",
+ "viszont", "volna", NULL
+};
+const uint8_t **lucy_SnowStop_snow_hu = (const uint8_t**)words_hu;
+
+static const char *words_it[] = {
+ "ad", "al", "allo", "ai", "agli", "all", "agl", "alla", "alle", "con",
+ "col", "coi", "da", "dal", "dallo", "dai", "dagli", "dall", "dagl",
+ "dalla", "dalle", "di", "del", "dello", "dei", "degli", "dell", "degl",
+ "della", "delle", "in", "nel", "nello", "nei", "negli", "nell", "negl",
+ "nella", "nelle", "su", "sul", "sullo", "sui", "sugli", "sull", "sugl",
+ "sulla", "sulle", "per", "tra", "contro", "io", "tu", "lui", "lei",
+ "noi", "voi", "loro", "mio", "mia", "miei", "mie", "tuo", "tua", "tuoi",
+ "tue", "suo", "sua", "suoi", "sue", "nostro", "nostra", "nostri",
+ "nostre", "vostro", "vostra", "vostri", "vostre", "mi", "ti", "ci", "vi",
+ "lo", "la", "li", "le", "gli", "ne", "il", "un", "uno", "una", "ma",
+ "ed", "se", "perch\303\251", "anche", "come", "dov", "dove", "che",
+ "chi", "cui", "non", "pi\303\271", "quale", "quanto", "quanti", "quanta",
+ "quante", "quello", "quelli", "quella", "quelle", "questo", "questi",
+ "questa", "queste", "si", "tutto", "tutti", "a", "c", "e", "i", "l", "o",
+ "ho", "hai", "ha", "abbiamo", "avete", "hanno", "abbia", "abbiate",
+ "abbiano", "avr\303\262", "avrai", "avr\303\240", "avremo", "avrete",
+ "avranno", "avrei", "avresti", "avrebbe", "avremmo", "avreste",
+ "avrebbero", "avevo", "avevi", "aveva", "avevamo", "avevate", "avevano",
+ "ebbi", "avesti", "ebbe", "avemmo", "aveste", "ebbero", "avessi",
+ "avesse", "avessimo", "avessero", "avendo", "avuto", "avuta", "avuti",
+ "avute", "sono", "sei", "\303\250", "siamo", "siete", "sia", "siate",
+ "siano", "sar\303\262", "sarai", "sar\303\240", "saremo", "sarete",
+ "saranno", "sarei", "saresti", "sarebbe", "saremmo", "sareste",
+ "sarebbero", "ero", "eri", "era", "eravamo", "eravate", "erano", "fui",
+ "fosti", "fu", "fummo", "foste", "furono", "fossi", "fosse", "fossimo",
+ "fossero", "essendo", "faccio", "fai", "facciamo", "fanno", "faccia",
+ "facciate", "facciano", "far\303\262", "farai", "far\303\240", "faremo",
+ "farete", "faranno", "farei", "faresti", "farebbe", "faremmo", "fareste",
+ "farebbero", "facevo", "facevi", "faceva", "facevamo", "facevate",
+ "facevano", "feci", "facesti", "fece", "facemmo", "faceste", "fecero",
+ "facessi", "facesse", "facessimo", "facessero", "facendo", "sto", "stai",
+ "sta", "stiamo", "stanno", "stia", "stiate", "stiano", "star\303\262",
+ "starai", "star\303\240", "staremo", "starete", "staranno", "starei",
+ "staresti", "starebbe", "staremmo", "stareste", "starebbero", "stavo",
+ "stavi", "stava", "stavamo", "stavate", "stavano", "stetti", "stesti",
+ "stette", "stemmo", "steste", "stettero", "stessi", "stesse", "stessimo",
+ "stessero", "stando", NULL
+};
+const uint8_t **lucy_SnowStop_snow_it = (const uint8_t**)words_it;
+
+static const char *words_nl[] = {
+ "de", "en", "van", "ik", "te", "dat", "die", "in", "een", "hij", "het",
+ "niet", "zijn", "is", "was", "op", "aan", "met", "als", "voor", "had",
+ "er", "maar", "om", "hem", "dan", "zou", "of", "wat", "mijn", "men",
+ "dit", "zo", "door", "over", "ze", "zich", "bij", "ook", "tot", "je",
+ "mij", "uit", "der", "daar", "haar", "naar", "heb", "hoe", "heeft",
+ "hebben", "deze", "u", "want", "nog", "zal", "me", "zij", "nu", "ge",
+ "geen", "omdat", "iets", "worden", "toch", "al", "waren", "veel", "meer",
+ "doen", "toen", "moet", "ben", "zonder", "kan", "hun", "dus", "alles",
+ "onder", "ja", "eens", "hier", "wie", "werd", "altijd", "doch", "wordt",
+ "wezen", "kunnen", "ons", "zelf", "tegen", "na", "reeds", "wil", "kon",
+ "niets", "uw", "iemand", "geweest", "andere", NULL
+};
+const uint8_t **lucy_SnowStop_snow_nl = (const uint8_t**)words_nl;
+
+static const char *words_no[] = {
+ "og", "i", "jeg", "det", "at", "en", "et", "den", "til", "er", "som",
+ "p\303\245", "de", "med", "han", "av", "ikke", "ikkje", "der",
+ "s\303\245", "var", "meg", "seg", "men", "ett", "har", "om", "vi", "min",
+ "mitt", "ha", "hadde", "hun", "n\303\245", "over", "da", "ved", "fra",
+ "du", "ut", "sin", "dem", "oss", "opp", "man", "kan", "hans", "hvor",
+ "eller", "hva", "skal", "selv", "sj\303\270l", "her", "alle", "vil",
+ "bli", "ble", "blei", "blitt", "kunne", "inn", "n\303\245r",
+ "v\303\246re", "kom", "noen", "noe", "ville", "dere", "som", "deres",
+ "kun", "ja", "etter", "ned", "skulle", "denne", "for", "deg", "si",
+ "sine", "sitt", "mot", "\303\245", "meget", "hvorfor", "dette", "disse",
+ "uten", "hvordan", "ingen", "din", "ditt", "blir", "samme", "hvilken",
+ "hvilke", "s\303\245nn", "inni", "mellom", "v\303\245r", "hver", "hvem",
+ "vors", "hvis", "b\303\245de", "bare", "enn", "fordi", "f\303\270r",
+ "mange", "ogs\303\245", "slik", "v\303\246rt", "v\303\246re",
+ "b\303\245e", "begge", "siden", "dykk", "dykkar", "dei", "deira",
+ "deires", "deim", "di", "d\303\245", "eg", "ein", "eit", "eitt", "elles",
+ "honom", "hj\303\245", "ho", "hoe", "henne", "hennar", "hennes", "hoss",
+ "hossen", "ikkje", "ingi", "inkje", "korleis", "korso", "kva", "kvar",
+ "kvarhelst", "kven", "kvi", "kvifor", "me", "medan", "mi", "mine",
+ "mykje", "no", "nokon", "noka", "nokor", "noko", "nokre", "si", "sia",
+ "sidan", "so", "somt", "somme", "um", "upp", "vere", "vore", "verte",
+ "vort", "varte", "vart", NULL
+};
+const uint8_t **lucy_SnowStop_snow_no = (const uint8_t**)words_no;
+
+static const char *words_pt[] = {
+ "de", "a", "o", "que", "e", "do", "da", "em", "um", "para", "com",
+ "n\303\243o", "uma", "os", "no", "se", "na", "por", "mais", "as", "dos",
+ "como", "mas", "ao", "ele", "das", "\303\240", "seu", "sua", "ou",
+ "quando", "muito", "nos", "j\303\241", "eu", "tamb\303\251m",
+ "s\303\263", "pelo", "pela", "at\303\251", "isso", "ela", "entre",
+ "depois", "sem", "mesmo", "aos", "seus", "quem", "nas", "me", "esse",
+ "eles", "voc\303\252", "essa", "num", "nem", "suas", "meu", "\303\240s",
+ "minha", "numa", "pelos", "elas", "qual", "n\303\263s", "lhe", "deles",
+ "essas", "esses", "pelas", "este", "dele", "tu", "te", "voc\303\252s",
+ "vos", "lhes", "meus", "minhas", "teu", "tua", "teus", "tuas", "nosso",
+ "nossa", "nossos", "nossas", "dela", "delas", "esta", "estes", "estas",
+ "aquele", "aquela", "aqueles", "aquelas", "isto", "aquilo", "estou",
+ "est\303\241", "estamos", "est\303\243o", "estive", "esteve",
+ "estivemos", "estiveram", "estava", "est\303\241vamos", "estavam",
+ "estivera", "estiv\303\251ramos", "esteja", "estejamos", "estejam",
+ "estivesse", "estiv\303\251ssemos", "estivessem", "estiver",
+ "estivermos", "estiverem", "hei", "h\303\241", "havemos", "h\303\243o",
+ "houve", "houvemos", "houveram", "houvera", "houv\303\251ramos", "haja",
+ "hajamos", "hajam", "houvesse", "houv\303\251ssemos", "houvessem",
+ "houver", "houvermos", "houverem", "houverei", "houver\303\241",
+ "houveremos", "houver\303\243o", "houveria", "houver\303\255amos",
+ "houveriam", "sou", "somos", "s\303\243o", "era", "\303\251ramos",
+ "eram", "fui", "foi", "fomos", "foram", "fora", "f\303\264ramos", "seja",
+ "sejamos", "sejam", "fosse", "f\303\264ssemos", "fossem", "for",
+ "formos", "forem", "serei", "ser\303\241", "seremos", "ser\303\243o",
+ "seria", "ser\303\255amos", "seriam", "tenho", "tem", "temos",
+ "t\303\251m", "tinha", "t\303\255nhamos", "tinham", "tive", "teve",
+ "tivemos", "tiveram", "tivera", "tiv\303\251ramos", "tenha", "tenhamos",
+ "tenham", "tivesse", "tiv\303\251ssemos", "tivessem", "tiver",
+ "tivermos", "tiverem", "terei", "ter\303\241", "teremos", "ter\303\243o",
+ "teria", "ter\303\255amos", "teriam", NULL
+};
+const uint8_t **lucy_SnowStop_snow_pt = (const uint8_t**)words_pt;
+
+static const char *words_ru[] = {
+ "\320\270", "\320\262", "\320\262\320\276", "\320\275\320\265",
+ "\321\207\321\202\320\276", "\320\276\320\275", "\320\275\320\260",
+ "\321\217", "\321\201", "\321\201\320\276", "\320\272\320\260\320\272",
+ "\320\260", "\321\202\320\276", "\320\262\321\201\320\265",
+ "\320\276\320\275\320\260", "\321\202\320\260\320\272",
+ "\320\265\320\263\320\276", "\320\275\320\276", "\320\264\320\260",
+ "\321\202\321\213", "\320\272", "\321\203", "\320\266\320\265",
+ "\320\262\321\213", "\320\267\320\260", "\320\261\321\213",
+ "\320\277\320\276", "\321\202\320\276\320\273\321\214\320\272\320\276",
+ "\320\265\320\265", "\320\274\320\275\320\265",
+ "\320\261\321\213\320\273\320\276", "\320\262\320\276\321\202",
+ "\320\276\321\202", "\320\274\320\265\320\275\321\217",
+ "\320\265\321\211\320\265", "\320\275\320\265\321\202", "\320\276",
+ "\320\270\320\267", "\320\265\320\274\321\203",
+ "\321\202\320\265\320\277\320\265\321\200\321\214",
+ "\320\272\320\276\320\263\320\264\320\260",
+ "\320\264\320\260\320\266\320\265", "\320\275\321\203",
+ "\320\262\320\264\321\200\321\203\320\263", "\320\273\320\270",
+ "\320\265\321\201\320\273\320\270", "\321\203\320\266\320\265",
+ "\320\270\320\273\320\270", "\320\275\320\270",
+ "\320\261\321\213\321\202\321\214", "\320\261\321\213\320\273",
+ "\320\275\320\265\320\263\320\276", "\320\264\320\276",
+ "\320\262\320\260\321\201",
+ "\320\275\320\270\320\261\321\203\320\264\321\214",
+ "\320\276\320\277\321\217\321\202\321\214", "\321\203\320\266",
+ "\320\262\320\260\320\274",
+ "\321\201\320\272\320\260\320\267\320\260\320\273",
+ "\320\262\320\265\320\264\321\214", "\321\202\320\260\320\274",
+ "\320\277\320\276\321\202\320\276\320\274",
+ "\321\201\320\265\320\261\321\217",
+ "\320\275\320\270\321\207\320\265\320\263\320\276", "\320\265\320\271",
+ "\320\274\320\276\320\266\320\265\321\202", "\320\276\320\275\320\270",
+ "\321\202\321\203\321\202", "\320\263\320\264\320\265",
+ "\320\265\321\201\321\202\321\214", "\320\275\320\260\320\264\320\276",
+ "\320\275\320\265\320\271", "\320\264\320\273\321\217",
+ "\320\274\321\213", "\321\202\320\265\320\261\321\217",
+ "\320\270\321\205", "\321\207\320\265\320\274",
+ "\320\261\321\213\320\273\320\260", "\321\201\320\260\320\274",
+ "\321\207\321\202\320\276\320\261", "\320\261\320\265\320\267",
+ "\320\261\321\203\320\264\321\202\320\276",
+ "\321\207\320\265\320\273\320\276\320\262\320\265\320\272",
+ "\321\207\320\265\320\263\320\276", "\321\200\320\260\320\267",
+ "\321\202\320\276\320\266\320\265", "\321\201\320\265\320\261\320\265",
+ "\320\277\320\276\320\264", "\320\266\320\270\320\267\320\275\321\214",
+ "\320\261\321\203\320\264\320\265\321\202", "\320\266",
+ "\321\202\320\276\320\263\320\264\320\260", "\320\272\321\202\320\276",
+ "\321\215\321\202\320\276\321\202",
+ "\320\263\320\276\320\262\320\276\321\200\320\270\320\273",
+ "\321\202\320\276\320\263\320\276",
+ "\320\277\320\276\321\202\320\276\320\274\321\203",
+ "\321\215\321\202\320\276\320\263\320\276",
+ "\320\272\320\260\320\272\320\276\320\271",
+ "\321\201\320\276\320\262\321\201\320\265\320\274",
+ "\320\275\320\270\320\274", "\320\267\320\264\320\265\321\201\321\214",
+ "\321\215\321\202\320\276\320\274", "\320\276\320\264\320\270\320\275",
+ "\320\277\320\276\321\207\321\202\320\270", "\320\274\320\276\320\271",
+ "\321\202\320\265\320\274", "\321\207\321\202\320\276\320\261\321\213",
+ "\320\275\320\265\320\265",
+ "\320\272\320\260\320\266\320\265\321\202\321\201\321\217",
+ "\321\201\320\265\320\271\321\207\320\260\321\201",
+ "\320\261\321\213\320\273\320\270", "\320\272\321\203\320\264\320\260",
+ "\320\267\320\260\321\207\320\265\320\274",
+ "\321\201\320\272\320\260\320\267\320\260\321\202\321\214",
+ "\320\262\321\201\320\265\321\205",
+ "\320\275\320\270\320\272\320\276\320\263\320\264\320\260",
+ "\321\201\320\265\320\263\320\276\320\264\320\275\321\217",
+ "\320\274\320\276\320\266\320\275\320\276", "\320\277\321\200\320\270",
+ "\320\275\320\260\320\272\320\276\320\275\320\265\321\206",
+ "\320\264\320\262\320\260", "\320\276\320\261",
+ "\320\264\321\200\321\203\320\263\320\276\320\271",
+ "\321\205\320\276\321\202\321\214",
+ "\320\277\320\276\321\201\320\273\320\265", "\320\275\320\260\320\264",
+ "\320\261\320\276\320\273\321\214\321\210\320\265",
+ "\321\202\320\276\321\202", "\321\207\320\265\321\200\320\265\320\267",
+ "\321\215\321\202\320\270", "\320\275\320\260\321\201",
+ "\320\277\321\200\320\276", "\320\262\321\201\320\265\320\263\320\276",
+ "\320\275\320\270\321\205", "\320\272\320\260\320\272\320\260\321\217",
+ "\320\274\320\275\320\276\320\263\320\276",
+ "\321\200\320\260\320\267\320\262\320\265",
+ "\321\201\320\272\320\260\320\267\320\260\320\273\320\260",
+ "\321\202\321\200\320\270", "\321\215\321\202\321\203",
+ "\320\274\320\276\321\217",
+ "\320\262\320\277\321\200\320\276\321\207\320\265\320\274",
+ "\321\205\320\276\321\200\320\276\321\210\320\276",
+ "\321\201\320\262\320\276\321\216", "\321\215\321\202\320\276\320\271",
+ "\320\277\320\265\321\200\320\265\320\264",
+ "\320\270\320\275\320\276\320\263\320\264\320\260",
+ "\320\273\321\203\321\207\321\210\320\265",
+ "\321\207\321\203\321\202\321\214", "\321\202\320\276\320\274",
+ "\320\275\320\265\320\273\321\214\320\267\321\217",
+ "\321\202\320\260\320\272\320\276\320\271", "\320\270\320\274",
+ "\320\261\320\276\320\273\320\265\320\265",
+ "\320\262\321\201\320\265\320\263\320\264\320\260",
+ "\320\272\320\276\320\275\320\265\321\207\320\275\320\276",
+ "\320\262\321\201\321\216", "\320\274\320\265\320\266\320\264\321\203",
+ NULL
+};
+const uint8_t **lucy_SnowStop_snow_ru = (const uint8_t**)words_ru;
+
+static const char *words_sv[] = {
+ "och", "det", "att", "i", "en", "jag", "hon", "som", "han", "p\303\245",
+ "den", "med", "var", "sig", "f\303\266r", "s\303\245", "till",
+ "\303\244r", "men", "ett", "om", "hade", "de", "av", "icke", "mig", "du",
+ "henne", "d\303\245", "sin", "nu", "har", "inte", "hans", "honom",
+ "skulle", "hennes", "d\303\244r", "min", "man", "ej", "vid", "kunde",
+ "n\303\245got", "fr\303\245n", "ut", "n\303\244r", "efter", "upp", "vi",
+ "dem", "vara", "vad", "\303\266ver", "\303\244n", "dig", "kan", "sina",
+ "h\303\244r", "ha", "mot", "alla", "under", "n\303\245gon", "eller",
+ "allt", "mycket", "sedan", "ju", "denna", "sj\303\244lv", "detta",
+ "\303\245t", "utan", "varit", "hur", "ingen", "mitt", "ni", "bli",
+ "blev", "oss", "din", "dessa", "n\303\245gra", "deras", "blir", "mina",
+ "samma", "vilken", "er", "s\303\245dan", "v\303\245r", "blivit", "dess",
+ "inom", "mellan", "s\303\245dant", "varf\303\266r", "varje", "vilka",
+ "ditt", "vem", "vilket", "sitta", "s\303\245dana", "vart", "dina",
+ "vars", "v\303\245rt", "v\303\245ra", "ert", "era", "vilkas", NULL
+};
+const uint8_t **lucy_SnowStop_snow_sv = (const uint8_t**)words_sv;
+
diff --git a/perl/Build.PL b/perl/Build.PL
new file mode 100644
index 0000000..a37b793
--- /dev/null
+++ b/perl/Build.PL
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use 5.008003;
+use strict;
+use warnings;
+use lib 'buildlib';
+use Lucy::Build;
+
+my $builder = Lucy::Build->new(
+ module_name => 'Lucy',
+ license => 'apache',
+ dist_author =>
+ 'The Apache Lucy Project <lucy-dev at incubator dot apache dot org>',
+ dist_version => '0.1.0',
+ requires => {
+ 'JSON::XS' => 1.53,
+ 'perl' => '5.8.3',
+ },
+ build_requires => {
+ 'Parse::RecDescent' => 1.94,
+ 'Module::Build' => 0.280801,
+ 'ExtUtils::CBuilder' => 0.21,
+ 'ExtUtils::ParseXS' => 2.18,
+ 'Devel::PPPort' => 3.13,
+ },
+ meta_merge => { keywords => [qw( search lucy lucene )], },
+ meta_add => {
+ resources => {
+ homepage => 'http://incubator.apache.org/lucy',
+ repository => 'http://svn.apache.org/repos/asf/incubator/lucy',
+ },
+ },
+ add_to_cleanup => [
+ qw(
+ Lucy-*
+ MANIFEST.bak
+ perltidy.ERR
+ *.pdb
+ *.manifest
+ ),
+ ],
+);
+
+$builder->create_build_script();
+
+__END__
diff --git a/perl/INSTALL b/perl/INSTALL
new file mode 100644
index 0000000..8eb001e
--- /dev/null
+++ b/perl/INSTALL
@@ -0,0 +1,14 @@
+Installing Apache Lucy with Perl bindings
+=========================================
+
+To install Apache Lucy as a suite of Perl modules, run the following commands:
+
+ perl Build.PL
+ perl Build
+ perl Build test
+ perl Build install
+
+Module::Build is required. Since Perl 5.10.0, Module::Build has been bundled
+with the Perl core, but on older systems it may be necessary to install it
+from CPAN first.
+
diff --git a/perl/MANIFEST b/perl/MANIFEST
new file mode 100644
index 0000000..6dd50fb
--- /dev/null
+++ b/perl/MANIFEST
@@ -0,0 +1,410 @@
+Build.PL
+buildlib/Lucy/Build.pm
+buildlib/Lucy/Redacted.pm
+buildlib/Lucy/Test/TestUtils.pm
+buildlib/Lucy/Test/USConSchema.pm
+INSTALL
+lib/Lucy.pm
+lib/Lucy.pod
+lib/Lucy/Analysis/Analyzer.pm
+lib/Lucy/Analysis/CaseFolder.pm
+lib/Lucy/Analysis/Inversion.pm
+lib/Lucy/Analysis/PolyAnalyzer.pm
+lib/Lucy/Analysis/RegexTokenizer.pm
+lib/Lucy/Analysis/SnowballStemmer.pm
+lib/Lucy/Analysis/SnowballStopFilter.pm
+lib/Lucy/Analysis/Token.pm
+lib/Lucy/Docs/Cookbook.pod
+lib/Lucy/Docs/Cookbook/CustomQuery.pod
+lib/Lucy/Docs/Cookbook/CustomQueryParser.pod
+lib/Lucy/Docs/Cookbook/FastUpdates.pod
+lib/Lucy/Docs/DevGuide.pm
+lib/Lucy/Docs/DocIDs.pod
+lib/Lucy/Docs/FileFormat.pod
+lib/Lucy/Docs/FileLocking.pm
+lib/Lucy/Docs/IRTheory.pod
+lib/Lucy/Docs/Tutorial.pod
+lib/Lucy/Docs/Tutorial/Analysis.pod
+lib/Lucy/Docs/Tutorial/BeyondSimple.pod
+lib/Lucy/Docs/Tutorial/FieldType.pod
+lib/Lucy/Docs/Tutorial/Highlighter.pod
+lib/Lucy/Docs/Tutorial/QueryObjects.pod
+lib/Lucy/Docs/Tutorial/Simple.pod
+lib/Lucy/Document/Doc.pm
+lib/Lucy/Document/HitDoc.pm
+lib/Lucy/Highlight/HeatMap.pm
+lib/Lucy/Highlight/Highlighter.pm
+lib/Lucy/Index/BackgroundMerger.pm
+lib/Lucy/Index/DataReader.pm
+lib/Lucy/Index/DataWriter.pm
+lib/Lucy/Index/DeletionsReader.pm
+lib/Lucy/Index/DeletionsWriter.pm
+lib/Lucy/Index/DocReader.pm
+lib/Lucy/Index/DocVector.pm
+lib/Lucy/Index/DocWriter.pm
+lib/Lucy/Index/FilePurger.pm
+lib/Lucy/Index/HighlightReader.pm
+lib/Lucy/Index/HighlightWriter.pm
+lib/Lucy/Index/Indexer.pm
+lib/Lucy/Index/IndexManager.pm
+lib/Lucy/Index/IndexReader.pm
+lib/Lucy/Index/Inverter.pm
+lib/Lucy/Index/Lexicon.pm
+lib/Lucy/Index/LexiconReader.pm
+lib/Lucy/Index/LexiconWriter.pm
+lib/Lucy/Index/PolyLexicon.pm
+lib/Lucy/Index/PolyReader.pm
+lib/Lucy/Index/Posting.pm
+lib/Lucy/Index/Posting/MatchPosting.pm
+lib/Lucy/Index/Posting/RichPosting.pm
+lib/Lucy/Index/Posting/ScorePosting.pm
+lib/Lucy/Index/PostingList.pm
+lib/Lucy/Index/PostingListReader.pm
+lib/Lucy/Index/PostingListWriter.pm
+lib/Lucy/Index/SegLexicon.pm
+lib/Lucy/Index/Segment.pm
+lib/Lucy/Index/SegPostingList.pm
+lib/Lucy/Index/SegReader.pm
+lib/Lucy/Index/SegWriter.pm
+lib/Lucy/Index/Similarity.pm
+lib/Lucy/Index/Snapshot.pm
+lib/Lucy/Index/SortCache.pm
+lib/Lucy/Index/SortReader.pm
+lib/Lucy/Index/SortWriter.pm
+lib/Lucy/Index/TermInfo.pm
+lib/Lucy/Index/TermVector.pm
+lib/Lucy/Object/BitVector.pm
+lib/Lucy/Object/ByteBuf.pm
+lib/Lucy/Object/CharBuf.pm
+lib/Lucy/Object/Err.pm
+lib/Lucy/Object/Hash.pm
+lib/Lucy/Object/Host.pm
+lib/Lucy/Object/I32Array.pm
+lib/Lucy/Object/LockFreeRegistry.pm
+lib/Lucy/Object/Num.pm
+lib/Lucy/Object/Obj.pm
+lib/Lucy/Object/VArray.pm
+lib/Lucy/Object/VTable.pm
+lib/Lucy/Plan/Architecture.pm
+lib/Lucy/Plan/BlobType.pm
+lib/Lucy/Plan/FieldType.pm
+lib/Lucy/Plan/Float32Type.pm
+lib/Lucy/Plan/Float64Type.pm
+lib/Lucy/Plan/FullTextType.pm
+lib/Lucy/Plan/Int32Type.pm
+lib/Lucy/Plan/Int64Type.pm
+lib/Lucy/Plan/Schema.pm
+lib/Lucy/Plan/StringType.pm
+lib/Lucy/Search/ANDMatcher.pm
+lib/Lucy/Search/ANDQuery.pm
+lib/Lucy/Search/BitVecMatcher.pm
+lib/Lucy/Search/Collector.pm
+lib/Lucy/Search/Collector/BitCollector.pm
+lib/Lucy/Search/Collector/SortCollector.pm
+lib/Lucy/Search/Compiler.pm
+lib/Lucy/Search/HitQueue.pm
+lib/Lucy/Search/Hits.pm
+lib/Lucy/Search/IndexSearcher.pm
+lib/Lucy/Search/LeafQuery.pm
+lib/Lucy/Search/MatchAllQuery.pm
+lib/Lucy/Search/MatchDoc.pm
+lib/Lucy/Search/Matcher.pm
+lib/Lucy/Search/NoMatchQuery.pm
+lib/Lucy/Search/NOTMatcher.pm
+lib/Lucy/Search/NOTQuery.pm
+lib/Lucy/Search/ORQuery.pm
+lib/Lucy/Search/ORScorer.pm
+lib/Lucy/Search/PhraseQuery.pm
+lib/Lucy/Search/PolyCompiler.pm
+lib/Lucy/Search/PolyQuery.pm
+lib/Lucy/Search/PolySearcher.pm
+lib/Lucy/Search/Query.pm
+lib/Lucy/Search/QueryParser.pm
+lib/Lucy/Search/RangeQuery.pm
+lib/Lucy/Search/RequiredOptionalMatcher.pm
+lib/Lucy/Search/RequiredOptionalQuery.pm
+lib/Lucy/Search/Searcher.pm
+lib/Lucy/Search/SortRule.pm
+lib/Lucy/Search/SortSpec.pm
+lib/Lucy/Search/Span.pm
+lib/Lucy/Search/TermQuery.pm
+lib/Lucy/Search/TopDocs.pm
+lib/Lucy/Simple.pm
+lib/Lucy/Store/FileHandle.pm
+lib/Lucy/Store/Folder.pm
+lib/Lucy/Store/FSFileHandle.pm
+lib/Lucy/Store/FSFolder.pm
+lib/Lucy/Store/InStream.pm
+lib/Lucy/Store/Lock.pm
+lib/Lucy/Store/LockErr.pm
+lib/Lucy/Store/LockFactory.pm
+lib/Lucy/Store/OutStream.pm
+lib/Lucy/Store/RAMFile.pm
+lib/Lucy/Store/RAMFileHandle.pm
+lib/Lucy/Store/RAMFolder.pm
+lib/Lucy/Test.pm
+lib/Lucy/Test/Util/BBSortEx.pm
+lib/Lucy/Util/Debug.pm
+lib/Lucy/Util/IndexFileNames.pm
+lib/Lucy/Util/Json.pm
+lib/Lucy/Util/MemoryPool.pm
+lib/Lucy/Util/PriorityQueue.pm
+lib/Lucy/Util/SortExternal.pm
+lib/Lucy/Util/Stepper.pm
+lib/Lucy/Util/StringHelper.pm
+lib/LucyX/Index/ByteBufDocReader.pm
+lib/LucyX/Index/ByteBufDocWriter.pm
+lib/LucyX/Index/LongFieldSim.pm
+lib/LucyX/Index/ZlibDocReader.pm
+lib/LucyX/Index/ZlibDocWriter.pm
+lib/LucyX/Remote/SearchClient.pm
+lib/LucyX/Remote/SearchServer.pm
+lib/LucyX/Search/Filter.pm
+lib/LucyX/Search/MockMatcher.pm
+lib/LucyX/Search/ProximityQuery.pm
+MANIFEST This list of files
+sample/FlatQueryParser.pm
+sample/indexer.pl
+sample/PrefixQuery.pm
+sample/README.txt
+sample/search.cgi
+sample/us_constitution/amend1.txt
+sample/us_constitution/amend10.txt
+sample/us_constitution/amend11.txt
+sample/us_constitution/amend12.txt
+sample/us_constitution/amend13.txt
+sample/us_constitution/amend14.txt
+sample/us_constitution/amend15.txt
+sample/us_constitution/amend16.txt
+sample/us_constitution/amend17.txt
+sample/us_constitution/amend18.txt
+sample/us_constitution/amend19.txt
+sample/us_constitution/amend2.txt
+sample/us_constitution/amend20.txt
+sample/us_constitution/amend21.txt
+sample/us_constitution/amend22.txt
+sample/us_constitution/amend23.txt
+sample/us_constitution/amend24.txt
+sample/us_constitution/amend25.txt
+sample/us_constitution/amend26.txt
+sample/us_constitution/amend27.txt
+sample/us_constitution/amend3.txt
+sample/us_constitution/amend4.txt
+sample/us_constitution/amend5.txt
+sample/us_constitution/amend6.txt
+sample/us_constitution/amend7.txt
+sample/us_constitution/amend8.txt
+sample/us_constitution/amend9.txt
+sample/us_constitution/art1sec1.txt
+sample/us_constitution/art1sec10.txt
+sample/us_constitution/art1sec2.txt
+sample/us_constitution/art1sec3.txt
+sample/us_constitution/art1sec4.txt
+sample/us_constitution/art1sec5.txt
+sample/us_constitution/art1sec6.txt
+sample/us_constitution/art1sec7.txt
+sample/us_constitution/art1sec8.txt
+sample/us_constitution/art1sec9.txt
+sample/us_constitution/art2sec1.txt
+sample/us_constitution/art2sec2.txt
+sample/us_constitution/art2sec3.txt
+sample/us_constitution/art2sec4.txt
+sample/us_constitution/art3sec1.txt
+sample/us_constitution/art3sec2.txt
+sample/us_constitution/art3sec3.txt
+sample/us_constitution/art4sec1.txt
+sample/us_constitution/art4sec2.txt
+sample/us_constitution/art4sec3.txt
+sample/us_constitution/art4sec4.txt
+sample/us_constitution/art5.txt
+sample/us_constitution/art6.txt
+sample/us_constitution/art7.txt
+sample/us_constitution/index.html
+sample/us_constitution/preamble.txt
+sample/us_constitution/uscon.css
+t/001-build_indexes.t
+t/002-lucy.t
+t/015-sort_external.t
+t/018-host.t
+t/021-vtable.t
+t/023-stepper.t
+t/025-debug.t
+t/026-serialization.t
+t/028-sortexrun.t
+t/050-ramfile.t
+t/051-fsfile.t
+t/102-strings_io.t
+t/105-folder.t
+t/106-locking.t
+t/109-read_locking.t
+t/110-shared_lock.t
+t/111-index_manager.t
+t/150-polyanalyzer.t
+t/151-analyzer.t
+t/152-inversion.t
+t/153-case_folder.t
+t/154-regex_tokenizer.t
+t/155-snowball_stop_filter.t
+t/156-snowball_stemmer.t
+t/200-doc.t
+t/201-hit_doc.t
+t/204-doc_reader.t
+t/205-seg_reader.t
+t/207-seg_lexicon.t
+t/208-terminfo.t
+t/209-seg_lexicon_heavy.t
+t/210-deldocs.t
+t/211-seg_posting_list.t
+t/213-segment_merging.t
+t/214-spec_field.t
+t/215-term_vectors.t
+t/216-schema.t
+t/217-poly_lexicon.t
+t/218-del_merging.t
+t/219-byte_buf_doc.t
+t/220-zlib_doc.t
+t/221-sort_writer.t
+t/224-lex_reader.t
+t/233-background_merger.t
+t/302-many_fields.t
+t/303-highlighter.t
+t/304-verify_utf8.t
+t/305-indexer.t
+t/306-dynamic_schema.t
+t/308-simple.t
+t/309-span.t
+t/310-heat_map.t
+t/311-hl_selection.t
+t/400-match_posting.t
+t/501-termquery.t
+t/502-phrasequery.t
+t/504-similarity.t
+t/505-hit_queue.t
+t/506-collector.t
+t/507-filter.t
+t/508-hits.t
+t/509-poly_searcher.t
+t/510-remote_search.t
+t/511-sort_spec.t
+t/513-matcher.t
+t/514-and_matcher.t
+t/515-range_query.t
+t/518-or_scorer.t
+t/519-req_opt_matcher.t
+t/520-match_doc.t
+t/523-and_query.t
+t/524-poly_query.t
+t/525-match_all_query.t
+t/526-not_query.t
+t/527-req_opt_query.t
+t/528-leaf_query.t
+t/529-no_match_query.t
+t/532-sort_collector.t
+t/601-queryparser.t
+t/602-boosts.t
+t/603-query_boosts.t
+t/604-simple_search.t
+t/605-store_pos_boost.t
+t/607-queryparser_multi_field.t
+t/610-queryparser_logic.t
+t/611-queryparser_syntax.t
+t/613-proximityquery.t
+t/701-uscon.t
+t/999-remove_indexes.t
+t/binding/016-varray.t
+t/binding/017-hash.t
+t/binding/019-obj.t
+t/binding/022-bytebuf.t
+t/binding/029-charbuf.t
+t/binding/034-err.t
+t/binding/038-lock_free_registry.t
+t/binding/101-simple_io.t
+t/binding/206-snapshot.t
+t/binding/506-collector.t
+t/binding/702-sample.t
+t/binding/800-stack.t
+t/binding/801-pod_checker.t
+t/charmonizer/001-integers.t
+t/charmonizer/002-func_macro.t
+t/charmonizer/003-headers.t
+t/charmonizer/004-large_files.t
+t/charmonizer/005-unused_vars.t
+t/charmonizer/006-variadic_macros.t
+t/charmonizer/007-dirmanip.t
+t/core/012-priority_queue.t
+t/core/013-bit_vector.t
+t/core/016-varray.t
+t/core/017-hash.t
+t/core/019-obj.t
+t/core/022-bytebuf.t
+t/core/024-memory_pool.t
+t/core/029-charbuf.t
+t/core/030-number_utils.t
+t/core/031-num.t
+t/core/032-string_helper.t
+t/core/033-index_file_names.t
+t/core/035-json.t
+t/core/036-i32_array.t
+t/core/037-atomic.t
+t/core/038-lock_free_registry.t
+t/core/039-memory.t
+t/core/050-ram_file_handle.t
+t/core/051-fs_file_handle.t
+t/core/052-instream.t
+t/core/053-file_handle.t
+t/core/054-io_primitives.t
+t/core/055-io_chunks.t
+t/core/061-ram_dir_handle.t
+t/core/062-fs_dir_handle.t
+t/core/103-fs_folder.t
+t/core/104-ram_folder.t
+t/core/105-folder.t
+t/core/111-index_manager.t
+t/core/112-cf_writer.t
+t/core/113-cf_reader.t
+t/core/150-analyzer.t
+t/core/150-polyanalyzer.t
+t/core/153-case_folder.t
+t/core/154-regex_tokenizer.t
+t/core/155-snowball_stop_filter.t
+t/core/156-snowball_stemmer.t
+t/core/206-snapshot.t
+t/core/216-schema.t
+t/core/220-doc_writer.t
+t/core/221-highlight_writer.t
+t/core/222-posting_list_writer.t
+t/core/223-seg_writer.t
+t/core/225-polyreader.t
+t/core/230-full_text_type.t
+t/core/231-blob_type.t
+t/core/232-numeric_type.t
+t/core/234-field_type.t
+t/core/301-segment.t
+t/core/501-termquery.t
+t/core/502-phrasequery.t
+t/core/515-range_query.t
+t/core/523-and_query.t
+t/core/525-match_all_query.t
+t/core/526-not_query.t
+t/core/527-req_opt_query.t
+t/core/528-leaf_query.t
+t/core/529-no_match_query.t
+t/core/530-series_matcher.t
+t/core/531-or_query.t
+xs/Lucy/Analysis/CaseFolder.c
+xs/Lucy/Analysis/RegexTokenizer.c
+xs/Lucy/Document/Doc.c
+xs/Lucy/Index/DocReader.c
+xs/Lucy/Index/Inverter.c
+xs/Lucy/Index/PolyReader.c
+xs/Lucy/Index/SegReader.c
+xs/Lucy/Object/Err.c
+xs/Lucy/Object/Host.c
+xs/Lucy/Object/LockFreeRegistry.c
+xs/Lucy/Object/Obj.c
+xs/Lucy/Object/VTable.c
+xs/Lucy/Store/FSFolder.c
+xs/Lucy/Util/Json.c
+xs/Lucy/Util/StringHelper.c
+xs/XSBind.c
+xs/XSBind.h
diff --git a/perl/MANIFEST.SKIP b/perl/MANIFEST.SKIP
new file mode 100644
index 0000000..bdb87b3
--- /dev/null
+++ b/perl/MANIFEST.SKIP
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cvs files and directories
+\bCVS\b
+,v$
+
+# all object files
+\.o
+
+# Module::Build generated files and dirs.
+^Build$
+^blib/
+^_build
+^MYMETA.yml$
+
+# Module::Build generated files and dirs within clownfish.
+^clownfish/Build$
+^clownfish/blib
+^clownfish/_build
+^clownfish/MYMETA.yml$
+^clownfish/include/ppport.h
+^clownfish/lib/Clownfish.c$
+
+# autogenerated by custom Build.PL
+Lucy\.xs$
+^typemap$
+
+# Makemaker generated files and dirs.
+^MANIFEST\.
+^Makefile$
+^Makefile\.old$
+^MakeMaker-\d
+pm_to_blib
+
+# hidden files
+^\.
+/\.
+
+# Apple window status files
+\.DS_Store
+
+# vim swap files
+\.swp$
+
+# log files
+\.log$
+
+# test indexes which may be lying around
+test_index/
+
+# benchmarking corpora
+lucy_index\b
+lucene_index\b
+extracted_corpus\b
+
+# various detritus
+^helper
+^_Inline
+\.gz$
+\.ERR$
+
+^MYMETA.yml$
diff --git a/perl/buildlib/Lucy/Build.pm b/perl/buildlib/Lucy/Build.pm
new file mode 100644
index 0000000..8bc1503
--- /dev/null
+++ b/perl/buildlib/Lucy/Build.pm
@@ -0,0 +1,771 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib '../clownfish/blib/arch';
+use lib '../clownfish/blib/lib';
+use lib 'clownfish/blib/arch';
+use lib 'clownfish/blib/lib';
+
+package Lucy::Build::CBuilder;
+BEGIN { our @ISA = "ExtUtils::CBuilder"; }
+use Config;
+
+my %cc;
+
+sub new {
+ my ( $class, %args ) = @_;
+ require ExtUtils::CBuilder;
+ if ( $ENV{LUCY_VALGRIND} ) {
+ $args{config} ||= {};
+ $args{config}{optimize} ||= $Config{optimize};
+ $args{config}{optimize} =~ s/\-O\d+/-O1/g;
+ }
+ my $self = $class->SUPER::new(%args);
+ $cc{"$self"} = $args{'config'}->{'cc'};
+ return $self;
+}
+
+sub get_cc { $cc{"$_[0]"} }
+
+sub DESTROY {
+ my $self = shift;
+ delete $cc{"$self"};
+}
+
+# This method isn't implemented by CBuilder for Windows, so we issue a basic
+# link command that works on at least one system and hope for the best.
+sub link_executable {
+ my ( $self, %args ) = @_;
+ if ( $self->get_cc eq 'cl' ) {
+ my ( $objects, $exe_file ) = @args{qw( objects exe_file )};
+ $self->do_system("link /out:$exe_file @$objects");
+ return $exe_file;
+ }
+ else {
+ return $self->SUPER::link_executable(%args);
+ }
+}
+
+package Lucy::Build;
+use base qw( Module::Build );
+
+use File::Spec::Functions
+ qw( catdir catfile curdir splitpath updir no_upwards );
+use File::Path qw( mkpath rmtree );
+use File::Copy qw( copy move );
+use File::Find qw( find );
+use Module::Build::ModuleInfo;
+use Config;
+use Env qw( @PATH );
+use Fcntl;
+use Carp;
+use Cwd qw( getcwd );
+
+BEGIN { unshift @PATH, curdir() }
+
+sub extra_ccflags {
+ my $self = shift;
+ my $extra_ccflags = defined $ENV{CFLAGS} ? "$ENV{CFLAGS} " : "";
+ my $gcc_version
+ = $ENV{REAL_GCC_VERSION}
+ || $self->config('gccversion')
+ || undef;
+ if ( defined $gcc_version ) {
+ $gcc_version =~ /^(\d+(\.\d+))/
+ or die "Invalid GCC version: $gcc_version";
+ $gcc_version = $1;
+ }
+
+ if ( defined $ENV{LUCY_DEBUG} ) {
+ if ( defined $gcc_version ) {
+ $extra_ccflags .= "-DLUCY_DEBUG ";
+ $extra_ccflags
+ .= "-DPERL_GCC_PEDANTIC -std=gnu99 -pedantic -Wall ";
+ $extra_ccflags .= "-Wextra " if $gcc_version >= 3.4; # correct
+ $extra_ccflags .= "-Wno-variadic-macros "
+ if $gcc_version > 3.4; # at least not on gcc 3.4
+ }
+ }
+
+ if ( $ENV{LUCY_VALGRIND} and defined $gcc_version ) {
+ $extra_ccflags .= "-fno-inline-functions ";
+ }
+
+ # Compile as C++ under MSVC.
+ if ( $self->config('cc') eq 'cl' ) {
+ $extra_ccflags .= '/TP ';
+ }
+
+ if ( defined $gcc_version ) {
+ # Tell GCC explicitly to run with maximum options.
+ if ( $extra_ccflags !~ m/-std=/ ) {
+ $extra_ccflags .= "-std=gnu99 ";
+ }
+ if ( $extra_ccflags !~ m/-D_GNU_SOURCE/ ) {
+ $extra_ccflags .= "-D_GNU_SOURCE ";
+ }
+ }
+
+ return $extra_ccflags;
+}
+
+=for Rationale
+
+When the distribution tarball for the Perl binding of Lucy is built, core/,
+charmonizer/, and any other needed files/directories are copied into the
+perl/ directory within the main Lucy directory. Then the distro is built from
+the contents of the perl/ directory, leaving out all the files in ruby/, etc.
+However, during development, the files are accessed from their original
+locations.
+
+=cut
+
+my $is_distro_not_devel = -e 'core';
+my $base_dir = $is_distro_not_devel ? curdir() : updir();
+
+my $CHARMONIZE_EXE_PATH = 'charmonize' . $Config{_exe};
+my $CHARMONIZER_ORIG_DIR = catdir( $base_dir, 'charmonizer' );
+my $CHARMONIZER_SRC_DIR = catdir( $CHARMONIZER_ORIG_DIR, 'src' );
+my $SNOWSTEM_SRC_DIR
+ = catdir( $base_dir, qw( modules analysis snowstem source ) );
+my $SNOWSTEM_INC_DIR = catdir( $SNOWSTEM_SRC_DIR, 'include' );
+my $SNOWSTOP_SRC_DIR
+ = catdir( $base_dir, qw( modules analysis snowstop source ) );
+my $CORE_SOURCE_DIR = catdir( $base_dir, 'core' );
+my $CLOWNFISH_DIR = catdir( $base_dir, 'clownfish' );
+my $CLOWNFISH_BUILD = catfile( $CLOWNFISH_DIR, 'Build' );
+my $AUTOGEN_DIR = 'autogen';
+my $XS_SOURCE_DIR = 'xs';
+my $LIB_DIR = 'lib';
+my $XS_FILEPATH = catfile( $LIB_DIR, "Lucy.xs" );
+my $AUTOBIND_PM_PATH = catfile( $LIB_DIR, 'Lucy', 'Autobinding.pm' );
+
+sub new { shift->SUPER::new( recursive_test_files => 1, @_ ) }
+
+# Build the charmonize executable.
+sub ACTION_charmonizer {
+ my $self = shift;
+
+ # Gather .c and .h Charmonizer files.
+ my $charm_source_files
+ = $self->rscan_dir( $CHARMONIZER_SRC_DIR, qr/Charmonizer.+\.[ch]$/ );
+ my $charmonize_c = catfile( $CHARMONIZER_ORIG_DIR, 'charmonize.c' );
+ my @all_source = ( $charmonize_c, @$charm_source_files );
+
+ # Don't compile if we're up to date.
+ return if $self->up_to_date( \@all_source, $CHARMONIZE_EXE_PATH );
+
+ print "Building $CHARMONIZE_EXE_PATH...\n\n";
+
+ my $cbuilder
+ = Lucy::Build::CBuilder->new( config => { cc => $self->config('cc') },
+ );
+
+ my @o_files;
+ for (@all_source) {
+ next unless /\.c$/;
+ next if m#Charmonizer/Test#;
+ my $o_file = $cbuilder->object_file($_);
+ $self->add_to_cleanup($o_file);
+ push @o_files, $o_file;
+
+ next if $self->up_to_date( $_, $o_file );
+
+ $cbuilder->compile(
+ source => $_,
+ include_dirs => [$CHARMONIZER_SRC_DIR],
+ extra_compiler_flags => $self->extra_ccflags,
+ );
+ }
+
+ $self->add_to_cleanup($CHARMONIZE_EXE_PATH);
+ my $exe_path = $cbuilder->link_executable(
+ objects => \@o_files,
+ exe_file => $CHARMONIZE_EXE_PATH,
+ );
+}
+
+# Run the charmonizer executable, creating the charmony.h file.
+sub ACTION_charmony {
+ my $self = shift;
+ my $charmony_path = 'charmony.h';
+
+ $self->dispatch('charmonizer');
+
+ return if $self->up_to_date( $CHARMONIZE_EXE_PATH, $charmony_path );
+ print "\nWriting $charmony_path...\n\n";
+
+ # Clean up after Charmonizer if it doesn't succeed on its own.
+ $self->add_to_cleanup("_charm*");
+ $self->add_to_cleanup("*.pdb");
+ $self->add_to_cleanup($charmony_path);
+
+ # Prepare arguments to charmonize.
+ my $cc = $self->config('cc');
+ my $flags = $self->config('ccflags') . ' ' . $self->extra_ccflags;
+ my $verbosity = $ENV{DEBUG_CHARM} ? 2 : 1;
+ $flags =~ s/"/\\"/g;
+
+ if ( $ENV{CHARM_VALGRIND} ) {
+ system( "valgrind --leak-check=yes ./$CHARMONIZE_EXE_PATH $cc "
+ . "\"$flags\" $verbosity" )
+ and die "Failed to write charmony.h";
+ }
+ else {
+ system("./$CHARMONIZE_EXE_PATH \"$cc\" \"$flags\" $verbosity")
+ and die "Failed to write charmony.h: $!";
+ }
+}
+
+sub _compile_clownfish {
+ my $self = shift;
+
+ require Clownfish::Hierarchy;
+ require Clownfish::Binding::Perl;
+ require Clownfish::Binding::Perl::Class;
+
+ # Compile Clownfish.
+ my $hierarchy = Clownfish::Hierarchy->new(
+ source => $CORE_SOURCE_DIR,
+ dest => $AUTOGEN_DIR,
+ );
+ $hierarchy->build;
+
+ # Process all __BINDING__ blocks.
+ my $pm_filepaths = $self->rscan_dir( $LIB_DIR, qr/\.pm$/ );
+ my @pm_filepaths_with_xs;
+ for my $pm_filepath (@$pm_filepaths) {
+ open( my $pm_fh, '<', $pm_filepath )
+ or die "Can't open '$pm_filepath': $!";
+ my $pm_content = do { local $/; <$pm_fh> };
+ my ($autobind_frag)
+ = $pm_content =~ /^__BINDING__\s*(.*?)(?:^__\w+__|\Z)/sm;
+ if ($autobind_frag) {
+ push @pm_filepaths_with_xs, $pm_filepath;
+ eval $autobind_frag;
+ confess("Invalid __BINDING__ from $pm_filepath: $@") if $@;
+ }
+ }
+
+ my $binding = Clownfish::Binding::Perl->new(
+ parcel => 'Lucy',
+ hierarchy => $hierarchy,
+ lib_dir => $LIB_DIR,
+ boot_class => 'Lucy',
+ header => $self->autogen_header,
+ footer => '',
+ );
+
+ return ( $hierarchy, $binding, \@pm_filepaths_with_xs );
+}
+
+sub ACTION_pod {
+ my $self = shift;
+ $self->dispatch("build_clownfish");
+ $self->_write_pod(@_);
+}
+
+sub _write_pod {
+ my ( $self, $binding ) = @_;
+ if ( !$binding ) {
+ ( undef, $binding ) = $self->_compile_clownfish;
+ }
+ my $pod_files = $binding->prepare_pod( lib_dir => $LIB_DIR );
+ print "Writing POD...\n";
+ while ( my ( $filepath, $pod ) = each %$pod_files ) {
+ $self->add_to_cleanup($filepath);
+ unlink $filepath;
+ sysopen( my $pod_fh, $filepath, O_CREAT | O_EXCL | O_WRONLY )
+ or confess("Can't open '$filepath': $!");
+ print $pod_fh $pod;
+ }
+}
+
+sub ACTION_build_clownfish {
+ my $self = shift;
+ my $old_dir = getcwd();
+ chdir($CLOWNFISH_DIR);
+ if ( !-f 'Build' ) {
+ print "\nBuilding Clownfish compiler... \n";
+ system("$^X Build.PL");
+ system("$^X Build code");
+ print "\nFinished building Clownfish compiler.\n\n";
+ }
+ chdir($old_dir);
+}
+
+sub ACTION_clownfish {
+ my $self = shift;
+
+ $self->dispatch('charmony');
+ $self->dispatch('build_clownfish');
+
+ # Create destination dir, copy xs helper files.
+ if ( !-d $AUTOGEN_DIR ) {
+ mkdir $AUTOGEN_DIR or die "Can't mkdir '$AUTOGEN_DIR': $!";
+ }
+ $self->add_to_cleanup($AUTOGEN_DIR);
+
+ my $pm_filepaths = $self->rscan_dir( $LIB_DIR, qr/\.pm$/ );
+ my $cfh_filepaths = $self->rscan_dir( $CORE_SOURCE_DIR, qr/\.cfh$/ );
+
+ # Don't bother parsing Clownfish files if everything's up to date.
+ return
+ if $self->up_to_date(
+ [ @$cfh_filepaths, @$pm_filepaths ],
+ [ $XS_FILEPATH, $AUTOGEN_DIR, ]
+ );
+
+ # Write out all autogenerated files.
+ print "Parsing Clownfish files...\n";
+ my ( $hierarchy, $perl_binding, $pm_filepaths_with_xs )
+ = $self->_compile_clownfish;
+ require Clownfish::Binding::Core;
+ my $core_binding = Clownfish::Binding::Core->new(
+ hierarchy => $hierarchy,
+ dest => $AUTOGEN_DIR,
+ header => $self->autogen_header,
+ footer => '',
+ );
+ print "Writing Clownfish autogenerated files...\n";
+ my $modified = $core_binding->write_all_modified;
+ if ($modified) {
+ unlink('typemap');
+ print "Writing typemap...\n";
+ $self->add_to_cleanup('typemap');
+ $perl_binding->write_xs_typemap;
+ }
+
+ # Rewrite XS if either any .cfh files or relevant .pm files were modified.
+ $modified ||=
+ $self->up_to_date( \@$pm_filepaths_with_xs, $XS_FILEPATH )
+ ? 0
+ : 1;
+
+ if ($modified) {
+ $self->add_to_cleanup($XS_FILEPATH);
+ $self->add_to_cleanup($AUTOBIND_PM_PATH);
+ $perl_binding->write_boot;
+ $perl_binding->write_bindings;
+ $self->_write_pod($perl_binding);
+ }
+
+ # Touch autogenerated files in case the modifications were inconsequential
+ # and didn't trigger a rewrite, so that we won't have to check them again
+ # next pass.
+ if (!$self->up_to_date(
+ [ @$cfh_filepaths, @$pm_filepaths_with_xs ], $XS_FILEPATH
+ )
+ )
+ {
+ utime( time, time, $XS_FILEPATH ); # touch
+ }
+ if (!$self->up_to_date(
+ [ @$cfh_filepaths, @$pm_filepaths_with_xs ], $AUTOGEN_DIR
+ )
+ )
+ {
+ utime( time, time, $AUTOGEN_DIR ); # touch
+ }
+}
+
+# Write ppport.h, which supplies some XS routines not found in older Perls and
+# allows us to use more up-to-date XS API while still supporting Perls back to
+# 5.8.3.
+#
+# The Devel::PPPort docs recommend that we distribute ppport.h rather than
+# require Devel::PPPort itself, but ppport.h isn't compatible with the Apache
+# license.
+sub ACTION_ppport {
+ my $self = shift;
+ if ( !-e 'ppport.h' ) {
+ require Devel::PPPort;
+ $self->add_to_cleanup('ppport.h');
+ Devel::PPPort::WriteFile();
+ }
+}
+
+sub ACTION_suppressions {
+ my $self = shift;
+ my $LOCAL_SUPP = 'local.supp';
+ return
+ if $self->up_to_date( '../devel/bin/valgrind_triggers.pl',
+ $LOCAL_SUPP );
+
+ # Generate suppressions.
+ print "Writing $LOCAL_SUPP...\n";
+ $self->add_to_cleanup($LOCAL_SUPP);
+ my $command
+ = "yes | "
+ . $self->_valgrind_base_command
+ . "--gen-suppressions=yes "
+ . $self->perl
+ . " ../devel/bin/valgrind_triggers.pl 2>&1";
+ my $suppressions = `$command`;
+ $suppressions =~ s/^==.*?\n//mg;
+ my $rule_number = 1;
+ while ( $suppressions =~ /<insert.a.*?>/ ) {
+ $suppressions =~ s/^\s*<insert.a.*?>/{\n <core_perl_$rule_number>/m;
+ $rule_number++;
+ }
+
+ # Change e.g. fun:_vgrZU_libcZdsoZa_calloc to fun:calloc
+ $suppressions =~ s/fun:\w+_((m|c|re)alloc)/fun:$1/g;
+
+ # Write local suppressions file.
+ open( my $supp_fh, '>', $LOCAL_SUPP )
+ or confess("Can't open '$LOCAL_SUPP': $!");
+ print $supp_fh $suppressions;
+}
+
+sub _valgrind_base_command {
+ return
+ "PERL_DESTRUCT_LEVEL=2 LUCY_VALGRIND=1 valgrind "
+ . "--leak-check=yes "
+ . "--show-reachable=yes "
+ . "--num-callers=10 "
+ . "--suppressions=../devel/conf/lucyperl.supp ";
+}
+
+sub ACTION_test_valgrind {
+ my $self = shift;
+ die "Must be run under a perl that was compiled with -DDEBUGGING"
+ unless $self->config('ccflags') =~ /-D?DEBUGGING\b/;
+ $self->dispatch('code');
+ $self->dispatch('suppressions');
+
+ # Unbuffer STDOUT, grab test file names and suppressions files.
+ $|++;
+ my $t_files = $self->find_test_files; # not public M::B API, may fail
+ my $valgrind_command = $self->_valgrind_base_command;
+ $valgrind_command .= "--suppressions=local.supp ";
+
+ if ( my $local_supp = $self->args('suppressions') ) {
+ for my $supp ( split( ',', $local_supp ) ) {
+ $valgrind_command .= "--suppressions=$supp ";
+ }
+ }
+
+ # Iterate over test files.
+ my @failed;
+ for my $t_file (@$t_files) {
+
+ # Run test file under Valgrind.
+ print "Testing $t_file...";
+ die "Can't find '$t_file'" unless -f $t_file;
+ my $command = "$valgrind_command $^X -Mblib $t_file 2>&1";
+ my $output = "\n" . ( scalar localtime(time) ) . "\n$command\n";
+ $output .= `$command`;
+
+ # Screen-scrape Valgrind output, looking for errors and leaks.
+ if ( $?
+ or $output =~ /ERROR SUMMARY:\s+[^0\s]/
+ or $output =~ /definitely lost:\s+[^0\s]/
+ or $output =~ /possibly lost:\s+[^0\s]/
+ or $output =~ /still reachable:\s+[^0\s]/ )
+ {
+ print " failed.\n";
+ push @failed, $t_file;
+ print "$output\n";
+ }
+ else {
+ print " succeeded.\n";
+ }
+ }
+
+ # If there are failed tests, print a summary list.
+ if (@failed) {
+ print "\nFailed "
+ . scalar @failed . "/"
+ . scalar @$t_files
+ . " test files:\n "
+ . join( "\n ", @failed ) . "\n";
+ exit(1);
+ }
+}
+
+sub ACTION_compile_custom_xs {
+ my $self = shift;
+
+ $self->dispatch('ppport');
+
+ require ExtUtils::ParseXS;
+
+ my $cbuilder
+ = Lucy::Build::CBuilder->new( config => { cc => $self->config('cc') },
+ );
+ my $archdir = catdir( $self->blib, 'arch', 'auto', 'Lucy', );
+ mkpath( $archdir, 0, 0777 ) unless -d $archdir;
+ my @include_dirs = (
+ curdir(), $CORE_SOURCE_DIR, $AUTOGEN_DIR, $XS_SOURCE_DIR,
+ $CHARMONIZER_SRC_DIR, $SNOWSTEM_INC_DIR
+ );
+ my @objects;
+
+ # Compile C source files.
+ my $c_files = [];
+ push @$c_files, @{ $self->rscan_dir( $CORE_SOURCE_DIR, qr/\.c$/ ) };
+ push @$c_files, @{ $self->rscan_dir( $XS_SOURCE_DIR, qr/\.c$/ ) };
+ push @$c_files, @{ $self->rscan_dir( $CHARMONIZER_SRC_DIR, qr/\.c$/ ) };
+ push @$c_files, @{ $self->rscan_dir( $AUTOGEN_DIR, qr/\.c$/ ) };
+ push @$c_files, @{ $self->rscan_dir( $SNOWSTEM_SRC_DIR, qr/\.c$/ ) };
+ push @$c_files, @{ $self->rscan_dir( $SNOWSTOP_SRC_DIR, qr/\.c$/ ) };
+ for my $c_file (@$c_files) {
+ my $o_file = $c_file;
+ my $ccs_file = $c_file;
+ $o_file =~ s/\.c/$Config{_o}/;
+ $ccs_file =~ s/\.c/.ccs/;
+ push @objects, $o_file;
+ next if $self->up_to_date( $c_file, $o_file );
+ $self->add_to_cleanup($o_file);
+ $self->add_to_cleanup($ccs_file);
+ $cbuilder->compile(
+ source => $c_file,
+ extra_compiler_flags => $self->extra_ccflags,
+ include_dirs => \@include_dirs,
+ object_file => $o_file,
+ );
+ }
+
+ # .xs => .c
+ my $perl_binding_c_file = catfile( $LIB_DIR, 'Lucy.c' );
+ $self->add_to_cleanup($perl_binding_c_file);
+ if ( !$self->up_to_date( $XS_FILEPATH, $perl_binding_c_file ) ) {
+ ExtUtils::ParseXS::process_file(
+ filename => $XS_FILEPATH,
+ prototypes => 0,
+ output => $perl_binding_c_file,
+ );
+ }
+
+ # .c => .o
+ my $lucy_pm_file = catfile( $LIB_DIR, 'Lucy.pm' );
+ my $info = Module::Build::ModuleInfo->new_from_file($lucy_pm_file);
+ my $version = $info->version;
+ my $perl_binding_o_file = catfile( $LIB_DIR, "Lucy$Config{_o}" );
+ unshift @objects, $perl_binding_o_file;
+ $self->add_to_cleanup($perl_binding_o_file);
+ if ( !$self->up_to_date( $perl_binding_c_file, $perl_binding_o_file ) ) {
+ $cbuilder->compile(
+ source => $perl_binding_c_file,
+ extra_compiler_flags => $self->extra_ccflags,
+ include_dirs => \@include_dirs,
+ object_file => $perl_binding_o_file,
+ # 'defines' is an undocumented parameter to compile(), so we
+ # should officially roll our own variant and generate compiler
+ # flags. However, that involves writing a bunch of
+ # platform-dependent code, so we'll just take the chance that this
+ # will break.
+ defines => {
+ VERSION => qq|"$version"|,
+ XS_VERSION => qq|"$version"|,
+ },
+ );
+ }
+
+ # Create .bs bootstrap file, needed by Dynaloader.
+ my $bs_file = catfile( $archdir, "Lucy.bs" );
+ $self->add_to_cleanup($bs_file);
+ if ( !$self->up_to_date( $perl_binding_o_file, $bs_file ) ) {
+ require ExtUtils::Mkbootstrap;
+ ExtUtils::Mkbootstrap::Mkbootstrap($bs_file);
+ if ( !-f $bs_file ) {
+ # Create file in case Mkbootstrap didn't do anything.
+ open( my $fh, '>', $bs_file )
+ or confess "Can't open $bs_file: $!";
+ }
+ utime( (time) x 2, $bs_file ); # touch
+ }
+
+ # Clean up after CBuilder under MSVC.
+ $self->add_to_cleanup('compilet*');
+ $self->add_to_cleanup('*.ccs');
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy.ccs' ) );
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy.def' ) );
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy_def.old' ) );
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy.exp' ) );
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy.lib' ) );
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy.lds' ) );
+ $self->add_to_cleanup( catfile( 'lib', 'Lucy.base' ) );
+
+ # .o => .(a|bundle)
+ my $lib_file = catfile( $archdir, "Lucy.$Config{dlext}" );
+ if ( !$self->up_to_date( [ @objects, $AUTOGEN_DIR ], $lib_file ) ) {
+ # TODO: use Charmonizer to determine whether pthreads are userland.
+ my $link_flags = $Config{osname} =~ /openbsd/i ? '-pthread ' : '';
+ $cbuilder->link(
+ module_name => 'Lucy',
+ objects => \@objects,
+ lib_file => $lib_file,
+ extra_linker_flags => $link_flags,
+ );
+ }
+}
+
+sub ACTION_code {
+ my $self = shift;
+
+ $self->dispatch('clownfish');
+ $self->dispatch('compile_custom_xs');
+
+ $self->SUPER::ACTION_code;
+}
+
+sub autogen_header {
+ my $self = shift;
+ return <<"END_AUTOGEN";
+/***********************************************
+
+ !!!! DO NOT EDIT !!!!
+
+ This file was auto-generated by Build.PL.
+
+ ***********************************************/
+
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+END_AUTOGEN
+}
+
+sub ACTION_dist {
+ my $self = shift;
+
+ $self->dispatch('pod');
+
+ # We build our Perl release tarball from $REPOS_ROOT/perl, rather than
+ # from the top-level.
+ #
+ # Because some items we need are outside this directory, we need to copy a
+ # bunch of stuff. After the tarball is packaged up, we delete the copied
+ # directories.
+ my @items_to_copy = qw(
+ core
+ modules
+ charmonizer
+ devel
+ clownfish
+ CHANGES
+ LICENSE
+ NOTICE
+ README
+ );
+ print "Copying files...\n";
+ for my $item (@items_to_copy) {
+ confess("'$item' already exists") if -e $item;
+ system("cp -R ../$item $item");
+ }
+
+ $self->dispatch('manifest');
+ my $no_index = $self->_gen_pause_exclusion_list;
+ $self->meta_add( { no_index => $no_index } );
+ $self->SUPER::ACTION_dist;
+
+ # Clean up.
+ print "Removing copied files...\n";
+ rmtree($_) for @items_to_copy;
+ unlink("META.yml");
+ move("MANIFEST.bak", "MANIFEST") or die "move() failed: $!";
+}
+
+# Generate a list of files for PAUSE, search.cpan.org, etc to ignore.
+sub _gen_pause_exclusion_list {
+ my $self = shift;
+
+ # Only exclude files that are actually on-board.
+ open( my $man_fh, '<', 'MANIFEST' ) or die "Can't open MANIFEST: $!";
+ my @manifest_entries = <$man_fh>;
+ chomp @manifest_entries;
+
+ my @excluded_files;
+ for my $entry (@manifest_entries) {
+ # Allow README and Changes.
+ next if $entry =~ m#^(README|Changes)#;
+
+ # Allow public modules.
+ if ( $entry =~ m#^(perl/)?lib\b.+\.(pm|pod)$# ) {
+ open( my $fh, '<', $entry ) or die "Can't open '$entry': $!";
+ my $content = do { local $/; <$fh> };
+ next if $content =~ /=head1\s*NAME/;
+ }
+
+ # Disallow everything else.
+ push @excluded_files, $entry;
+ }
+
+ # Exclude redacted modules.
+ if ( eval { require "buildlib/Lucy/Redacted.pm" } ) {
+ my @redacted = map {
+ my @parts = split( /\W+/, $_ );
+ catfile( $LIB_DIR, @parts ) . '.pm'
+ } Lucy::Redacted->redacted, Lucy::Redacted->hidden;
+ push @excluded_files, @redacted;
+ }
+
+ my %uniquifier;
+ @excluded_files = sort grep { !$uniquifier{$_}++ } @excluded_files;
+ return { file => \@excluded_files };
+}
+
+sub ACTION_semiclean {
+ my $self = shift;
+ print "Cleaning up most build files.\n";
+ my @candidates
+ = grep { $_ !~ /(charmonizer|^_charm|charmony|charmonize|snowstem)/ }
+ $self->cleanup;
+ for my $path ( map { glob($_) } @candidates ) {
+ next unless -e $path;
+ rmtree($path);
+ confess("Failed to remove '$path'") if -e $path;
+ }
+}
+
+sub ACTION_clean {
+ my $self = shift;
+ if ( -e $CLOWNFISH_BUILD ) {
+ system("$^X $CLOWNFISH_BUILD clean")
+ and die "Clownfish clean failed";
+ }
+ $self->SUPER::ACTION_clean;
+}
+
+sub ACTION_realclean {
+ my $self = shift;
+ if ( -e $CLOWNFISH_BUILD ) {
+ system("$^X $CLOWNFISH_BUILD realclean")
+ and die "Clownfish realclean failed";
+ }
+ $self->SUPER::ACTION_realclean;
+}
+
+1;
+
+__END__
diff --git a/perl/buildlib/Lucy/Redacted.pm b/perl/buildlib/Lucy/Redacted.pm
new file mode 100644
index 0000000..ef93be2
--- /dev/null
+++ b/perl/buildlib/Lucy/Redacted.pm
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Lucy::Redacted;
+use Exporter;
+BEGIN {
+ our @ISA = qw( Exporter );
+ our @EXPORT_OK = qw( list );
+}
+
+# Return a partial list of Lucy classes which were once public but are
+# now either deprecated, removed, or moved.
+
+sub redacted {
+ return qw(
+ Lucy::Analysis::LCNormalizer
+ Lucy::Analysis::Token
+ Lucy::Analysis::TokenBatch
+ Lucy::Index::Term
+ Lucy::InvIndex
+ Lucy::InvIndexer
+ Lucy::QueryParser::QueryParser
+ Lucy::Search::BooleanQuery
+ Lucy::Search::QueryFilter
+ Lucy::Search::SearchServer
+ Lucy::Search::SearchClient
+ );
+}
+
+# Hide additional stuff from PAUSE and search.cpan.org.
+sub hidden {
+ return qw(
+ Lucy::Analysis::Inversion
+ Lucy::Object::Num
+ Lucy::Plan::Int32Type
+ Lucy::Plan::Int64Type
+ Lucy::Plan::Float32Type
+ Lucy::Plan::Float64Type
+ Lucy::Redacted
+ Lucy::Test::Object::TestCharBuf
+ Lucy::Test::TestUtils
+ Lucy::Test::USConSchema
+ Lucy::Util::BitVector
+ );
+}
+
+1;
diff --git a/perl/buildlib/Lucy/Test/TestUtils.pm b/perl/buildlib/Lucy/Test/TestUtils.pm
new file mode 100644
index 0000000..c817a04
--- /dev/null
+++ b/perl/buildlib/Lucy/Test/TestUtils.pm
@@ -0,0 +1,248 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Lucy::Test::TestUtils;
+use base qw( Exporter );
+
+our @EXPORT_OK = qw(
+ working_dir
+ create_working_dir
+ remove_working_dir
+ create_index
+ create_uscon_index
+ test_index_loc
+ persistent_test_index_loc
+ init_test_index_loc
+ get_uscon_docs
+ utf8_test_strings
+ test_analyzer
+ doc_ids_from_td_coll
+ modulo_set
+);
+
+use Lucy;
+use Lucy::Test;
+
+use lib 'sample';
+use Lucy::Test::USConSchema;
+
+use File::Spec::Functions qw( catdir catfile curdir );
+use Encode qw( _utf8_off );
+use File::Path qw( rmtree );
+use Carp;
+
+my $working_dir = catfile( curdir(), 'lucy_test' );
+
+# Return a directory within the system's temp directory where we will put all
+# testing scratch files.
+sub working_dir {$working_dir}
+
+sub create_working_dir {
+ mkdir( $working_dir, 0700 ) or die "Can't mkdir '$working_dir': $!";
+}
+
+# Verify that this user owns the working dir, then zap it. Returns true upon
+# success.
+sub remove_working_dir {
+ return unless -d $working_dir;
+ rmtree $working_dir;
+ return 1;
+}
+
+# Return a location for a test index to be used by a single test file. If
+# the test file crashes it cannot clean up after itself, so we put the cleanup
+# routine in a single test file to be run at or near the end of the test
+# suite.
+sub test_index_loc {
+ return catdir( $working_dir, 'test_index' );
+}
+
+# Return a location for a test index intended to be shared by multiple test
+# files. It will be cleaned as above.
+sub persistent_test_index_loc {
+ return catdir( $working_dir, 'persistent_test_index' );
+}
+
+# Destroy anything left over in the test_index location, then create the
+# directory. Finally, return the path.
+sub init_test_index_loc {
+ my $dir = test_index_loc();
+ rmtree $dir;
+ die "Can't clean up '$dir'" if -e $dir;
+ mkdir $dir or die "Can't mkdir '$dir': $!";
+ return $dir;
+}
+
+# Build a RAM index, using the supplied array of strings as source material.
+# The index will have a single field: "content".
+sub create_index {
+ my $folder = Lucy::Store::RAMFolder->new;
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => Lucy::Test::TestSchema->new,
+ );
+ $indexer->add_doc( { content => $_ } ) for @_;
+ $indexer->commit;
+ return $folder;
+}
+
+# Slurp us constitition docs and build hashrefs.
+sub get_uscon_docs {
+
+ my $uscon_dir = catdir( 'sample', 'us_constitution' );
+ opendir( my $uscon_dh, $uscon_dir )
+ or die "couldn't opendir '$uscon_dir': $!";
+ my @filenames = grep {/\.txt$/} sort readdir $uscon_dh;
+ closedir $uscon_dh or die "couldn't closedir '$uscon_dir': $!";
+
+ my %docs;
+
+ for my $filename (@filenames) {
+ my $filepath = catfile( $uscon_dir, $filename );
+ open( my $fh, '<', $filepath )
+ or die "couldn't open file '$filepath': $!";
+ my $content = do { local $/; <$fh> };
+ $content =~ /\A(.+?)^\s+(.*)/ms
+ or die "Can't extract title/bodytext from '$filepath'";
+ my $title = $1;
+ my $bodytext = $2;
+ $bodytext =~ s/\s+/ /sg;
+ my $category
+ = $filename =~ /art/ ? 'article'
+ : $filename =~ /amend/ ? 'amendment'
+ : $filename =~ /preamble/ ? 'preamble'
+ : confess "Can't derive category for $filename";
+
+ $docs{$filename} = {
+ title => $title,
+ bodytext => $bodytext,
+ url => "/us_constitution/$filename",
+ category => $category,
+ };
+ }
+
+ return \%docs;
+}
+
+sub create_uscon_index {
+ my $folder
+ = Lucy::Store::FSFolder->new( path => persistent_test_index_loc() );
+ my $schema = Lucy::Test::USConSchema->new;
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ truncate => 1,
+ create => 1,
+ );
+
+ $indexer->add_doc( { content => "zz$_" } ) for ( 0 .. 10000 );
+ $indexer->commit;
+ undef $indexer;
+
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ my $source_docs = get_uscon_docs();
+ $indexer->add_doc( { content => $_->{bodytext} } )
+ for values %$source_docs;
+ $indexer->commit;
+ undef $indexer;
+
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ my @chars = ( 'a' .. 'z' );
+ for ( 0 .. 1000 ) {
+ my $content = '';
+ for my $num_words ( 1 .. int( rand(20) ) ) {
+ for ( 1 .. ( int( rand(10) ) + 10 ) ) {
+ $content .= @chars[ rand(@chars) ];
+ }
+ $content .= ' ';
+ }
+ $indexer->add_doc( { content => $content } );
+ }
+ $indexer->optimize;
+ $indexer->commit;
+}
+
+# Return 3 strings useful for verifying UTF-8 integrity.
+sub utf8_test_strings {
+ my $smiley = "\x{263a}";
+ my $not_a_smiley = $smiley;
+ _utf8_off($not_a_smiley);
+ my $frowny = $not_a_smiley;
+ utf8::upgrade($frowny);
+ return ( $smiley, $not_a_smiley, $frowny );
+}
+
+# Verify an Analyzer's transform, transform_text, and split methods.
+sub test_analyzer {
+ my ( $analyzer, $source, $expected, $message ) = @_;
+
+ my $inversion = Lucy::Analysis::Inversion->new( text => $source );
+ $inversion = $analyzer->transform($inversion);
+ my @got;
+ while ( my $token = $inversion->next ) {
+ push @got, $token->get_text;
+ }
+ Test::More::is_deeply( \@got, $expected, "analyze: $message" );
+
+ $inversion = $analyzer->transform_text($source);
+ @got = ();
+ while ( my $token = $inversion->next ) {
+ push @got, $token->get_text;
+ }
+ Test::More::is_deeply( \@got, $expected, "transform_text: $message" );
+
+ @got = @{ $analyzer->split($source) };
+ Test::More::is_deeply( \@got, $expected, "split: $message" );
+}
+
+# Extract all doc nums from a SortCollector. Return two sorted array refs:
+# by_score and by_id.
+sub doc_ids_from_td_coll {
+ my $collector = shift;
+ my @by_score;
+ my $match_docs = $collector->pop_match_docs;
+ my @by_score_then_id = map { $_->get_doc_id }
+ sort {
+ $b->get_score <=> $a->get_score
+ || $a->get_doc_id <=> $b->get_doc_id
+ } @$match_docs;
+ my @by_id = sort { $a <=> $b } @by_score_then_id;
+ return ( \@by_score_then_id, \@by_id );
+}
+
+# Use a modulus to generate a set of numbers.
+sub modulo_set {
+ my ( $interval, $max ) = @_;
+ my @out;
+ for ( my $doc = $interval; $doc < $max; $doc += $interval ) {
+ push @out, $doc;
+ }
+ return \@out;
+}
+
+1;
+
+__END__
+
+
diff --git a/perl/buildlib/Lucy/Test/USConSchema.pm b/perl/buildlib/Lucy/Test/USConSchema.pm
new file mode 100644
index 0000000..cebbaec
--- /dev/null
+++ b/perl/buildlib/Lucy/Test/USConSchema.pm
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Lucy::Test::USConSchema;
+use base 'Lucy::Plan::Schema';
+use Lucy::Analysis::PolyAnalyzer;
+use Lucy::Plan::FullTextType;
+use Lucy::Plan::StringType;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $analyzer = Lucy::Analysis::PolyAnalyzer->new( language => 'en' );
+ my $title_type = Lucy::Plan::FullTextType->new( analyzer => $analyzer, );
+ my $content_type = Lucy::Plan::FullTextType->new(
+ analyzer => $analyzer,
+ highlightable => 1,
+ );
+ my $url_type = Lucy::Plan::StringType->new( indexed => 0, );
+ my $cat_type = Lucy::Plan::StringType->new;
+ $self->spec_field( name => 'title', type => $title_type );
+ $self->spec_field( name => 'content', type => $content_type );
+ $self->spec_field( name => 'url', type => $url_type );
+ $self->spec_field( name => 'category', type => $cat_type );
+ return $self;
+}
+
+1;
diff --git a/perl/lib/Lucy.pm b/perl/lib/Lucy.pm
new file mode 100644
index 0000000..bb81414
--- /dev/null
+++ b/perl/lib/Lucy.pm
@@ -0,0 +1,651 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Lucy;
+
+use 5.008003;
+use Exporter;
+
+our $VERSION = '0.001000';
+$VERSION = eval $VERSION;
+
+use XSLoader;
+# This loads a large number of disparate subs.
+BEGIN { XSLoader::load( 'Lucy', '0.001000' ) }
+
+BEGIN {
+ push our @ISA, 'Exporter';
+ our @EXPORT_OK = qw( to_clownfish to_perl kdump );
+}
+
+use Lucy::Autobinding;
+
+sub kdump {
+ require Data::Dumper;
+ my $kdumper = Data::Dumper->new( [@_] );
+ $kdumper->Sortkeys( sub { return [ sort keys %{ $_[0] } ] } );
+ $kdumper->Indent(1);
+ warn $kdumper->Dump;
+}
+
+sub error {$Lucy::Object::Err::error}
+
+{
+ package Lucy::Util::IndexFileNames;
+ BEGIN {
+ push our @ISA, 'Exporter';
+ our @EXPORT_OK = qw(
+ extract_gen
+ latest_snapshot
+ );
+ }
+}
+
+{
+ package Lucy::Util::StringHelper;
+ BEGIN {
+ push our @ISA, 'Exporter';
+ our @EXPORT_OK = qw(
+ utf8_flag_on
+ utf8_flag_off
+ to_base36
+ from_base36
+ utf8ify
+ utf8_valid
+ cat_bytes
+ );
+ }
+}
+
+{
+ package Lucy::Analysis::Inversion;
+
+ our %new_PARAMS = (
+ # params
+ text => undef
+ );
+}
+
+{
+ package Lucy::Analysis::Token;
+
+ our %new_PARAMS = (
+ text => undef,
+ start_offset => undef,
+ end_offset => undef,
+ pos_inc => 1,
+ boost => 1.0,
+ );
+}
+
+{
+ package Lucy::Analysis::RegexTokenizer;
+
+ sub compile_token_re { return qr/$_[1]/ }
+
+ sub new {
+ my ( $either, %args ) = @_;
+ my $token_re = delete $args{token_re};
+ $args{pattern} = "$token_re" if $token_re;
+ return $either->_new(%args);
+ }
+}
+
+{
+ package Lucy::Document::Doc;
+ use Storable qw( nfreeze thaw );
+ use bytes;
+ no bytes;
+
+ our %new_PARAMS = (
+ fields => undef,
+ doc_id => 0,
+ );
+
+ use overload
+ fallback => 1,
+ '%{}' => \&get_fields;
+
+ sub serialize_fields {
+ my ( $self, $outstream ) = @_;
+ my $buf = nfreeze( $self->get_fields );
+ $outstream->write_c32( bytes::length($buf) );
+ $outstream->print($buf);
+ }
+
+ sub deserialize_fields {
+ my ( $self, $instream ) = @_;
+ my $len = $instream->read_c32;
+ my $buf;
+ $instream->read( $buf, $len );
+ $self->set_fields( thaw($buf) );
+ }
+}
+
+{
+ package Lucy::Document::HitDoc;
+
+ our %new_PARAMS = (
+ fields => undef,
+ score => 0,
+ doc_id => 0,
+ );
+}
+
+{
+ package Lucy::Object::I32Array;
+ our %new_PARAMS = ( ints => undef );
+}
+
+{
+ package Lucy::Object::LockFreeRegistry;
+ sub DESTROY { } # leak all
+}
+
+{
+ package Lucy::Object::Obj;
+ use Lucy qw( to_clownfish to_perl );
+ sub load { return $_[0]->_load( to_clownfish( $_[1] ) ) }
+}
+
+{
+ package Lucy::Object::VTable;
+
+ sub find_parent_class {
+ my ( undef, $package ) = @_;
+ no strict 'refs';
+ for my $parent ( @{"$package\::ISA"} ) {
+ return $parent if $parent->isa('Lucy::Object::Obj');
+ }
+ return;
+ }
+
+ sub novel_host_methods {
+ my ( undef, $package ) = @_;
+ no strict 'refs';
+ my $stash = \%{"$package\::"};
+ my $methods
+ = Lucy::Object::VArray->new( capacity => scalar keys %$stash );
+ while ( my ( $symbol, $glob ) = each %$stash ) {
+ next if ref $glob;
+ next unless *$glob{CODE};
+ $methods->push( Lucy::Object::CharBuf->new($symbol) );
+ }
+ return $methods;
+ }
+
+ sub _register {
+ my ( undef, %args ) = @_;
+ my $singleton_class = $args{singleton}->get_name;
+ my $parent_class = $args{parent}->get_name;
+ if ( !$singleton_class->isa($parent_class) ) {
+ no strict 'refs';
+ push @{"$singleton_class\::ISA"}, $parent_class;
+ }
+ }
+}
+
+{
+ package Lucy::Index::Indexer;
+
+ sub new {
+ my ( $either, %args ) = @_;
+ my $flags = 0;
+ $flags |= CREATE if delete $args{'create'};
+ $flags |= TRUNCATE if delete $args{'truncate'};
+ return $either->_new( %args, flags => $flags );
+ }
+
+ our %add_doc_PARAMS = ( doc => undef, boost => 1.0 );
+}
+
+{
+ package Lucy::Index::IndexReader;
+ use Carp;
+
+ sub new {
+ confess(
+ "IndexReader is an abstract class; use open() instead of new()");
+ }
+ sub lexicon {
+ my $self = shift;
+ my $lex_reader = $self->fetch("Lucy::Index::LexiconReader");
+ return $lex_reader->lexicon(@_) if $lex_reader;
+ return;
+ }
+ sub posting_list {
+ my $self = shift;
+ my $plist_reader = $self->fetch("Lucy::Index::PostingListReader");
+ return $plist_reader->posting_list(@_) if $plist_reader;
+ return;
+ }
+ sub offsets { shift->_offsets->to_arrayref }
+}
+
+{
+ package Lucy::Index::PolyReader;
+ use Lucy qw( to_clownfish );
+
+ sub try_read_snapshot {
+ my ( undef, %args ) = @_;
+ my ( $snapshot, $folder, $path ) = @args{qw( snapshot folder path )};
+ eval { $snapshot->read_file( folder => $folder, path => $path ); };
+ if ($@) { return Lucy::Object::CharBuf->new($@) }
+ else { return undef }
+ }
+
+ sub try_open_segreaders {
+ my ( $self, $segments ) = @_;
+ my $schema = $self->get_schema;
+ my $folder = $self->get_folder;
+ my $snapshot = $self->get_snapshot;
+ my $seg_readers
+ = Lucy::Object::VArray->new( capacity => scalar @$segments );
+ my $segs = to_clownfish($segments); # FIXME: Don't convert twice.
+ eval {
+ # Create a SegReader for each segment in the index.
+ my $num_segs = scalar @$segments;
+ for ( my $seg_tick = 0; $seg_tick < $num_segs; $seg_tick++ ) {
+ my $seg_reader = Lucy::Index::SegReader->new(
+ schema => $schema,
+ folder => $folder,
+ segments => $segs,
+ seg_tick => $seg_tick,
+ snapshot => $snapshot,
+ );
+ $seg_readers->push($seg_reader);
+ }
+ };
+ if ($@) {
+ return Lucy::Object::CharBuf->new($@);
+ }
+ return $seg_readers;
+ }
+}
+
+{
+ package Lucy::Index::Segment;
+ use Lucy qw( to_clownfish );
+ sub store_metadata {
+ my ( $self, %args ) = @_;
+ $self->_store_metadata( %args,
+ metadata => to_clownfish( $args{metadata} ) );
+ }
+}
+
+{
+ package Lucy::Index::SegReader;
+
+ sub try_init_components {
+ my $self = shift;
+ my $arch = $self->get_schema->get_architecture;
+ eval { $arch->init_seg_reader($self); };
+ if ($@) { return Lucy::Object::CharBuf->new($@); }
+ return;
+ }
+}
+
+{
+ package Lucy::Index::SortCache;
+ our %value_PARAMS = ( ord => undef, );
+}
+
+{
+ package Lucy::Search::Compiler;
+ use Carp;
+ use Scalar::Util qw( blessed );
+
+ sub new {
+ my ( $either, %args ) = @_;
+ if ( !defined $args{boost} ) {
+ confess("'parent' is not a Query")
+ unless ( blessed( $args{parent} )
+ and $args{parent}->isa("Lucy::Search::Query") );
+ $args{boost} = $args{parent}->get_boost;
+ }
+ return $either->do_new(%args);
+ }
+}
+
+{
+ package Lucy::Search::Query;
+
+ sub make_compiler {
+ my ( $self, %args ) = @_;
+ $args{boost} = $self->get_boost unless defined $args{boost};
+ return $self->_make_compiler(%args);
+ }
+}
+
+{
+ package Lucy::Search::SortRule;
+
+ my %types = (
+ field => FIELD(),
+ score => SCORE(),
+ doc_id => DOC_ID(),
+ );
+
+ sub new {
+ my ( $either, %args ) = @_;
+ my $type = delete $args{type} || 'field';
+ confess("Invalid type: '$type'") unless defined $types{$type};
+ return $either->_new( %args, type => $types{$type} );
+ }
+}
+
+{
+ package Lucy::Object::BitVector;
+ sub to_arrayref { shift->to_array->to_arrayref }
+}
+
+{
+ package Lucy::Object::ByteBuf;
+ {
+ # Override autogenerated deserialize binding.
+ no warnings 'redefine';
+ sub deserialize { shift->_deserialize(@_) }
+ }
+}
+
+{
+ package Lucy::Object::ViewByteBuf;
+ use Carp;
+ sub new { confess "ViewByteBuf objects can only be created from C." }
+}
+
+{
+ package Lucy::Object::CharBuf;
+
+ {
+ # Defeat obscure bugs in the XS auto-generation by redefining clone()
+ # and deserialize(). (Because of how the typemap works for CharBuf*,
+ # the auto-generated methods return UTF-8 Perl scalars rather than
+ # actual CharBuf objects.)
+ no warnings 'redefine';
+ sub clone { shift->_clone(@_) }
+ sub deserialize { shift->_deserialize(@_) }
+ }
+}
+
+{
+ package Lucy::Object::ViewCharBuf;
+ use Carp;
+ sub new { confess "ViewCharBuf has no public constructor." }
+}
+
+{
+ package Lucy::Object::ZombieCharBuf;
+ use Carp;
+ sub new { confess "ZombieCharBuf objects can only be created from C." }
+ sub DESTROY { }
+}
+
+{
+ package Lucy::Object::Err;
+ sub do_to_string { shift->to_string }
+ use Scalar::Util qw( blessed );
+ use Carp qw( confess longmess );
+ use overload
+ '""' => \&do_to_string,
+ fallback => 1;
+
+ sub new {
+ my ( $either, $message ) = @_;
+ my ( undef, $file, $line ) = caller;
+ $message .= ", $file line $line\n";
+ return $either->_new( mess => Lucy::Object::CharBuf->new($message) );
+ }
+
+ sub do_throw {
+ my $err = shift;
+ my $longmess = longmess();
+ $longmess =~ s/^\s*/\t/;
+ $err->cat_mess($longmess);
+ die $err;
+ }
+
+ our $error;
+ sub set_error {
+ my $val = $_[1];
+ if ( defined $val ) {
+ confess("Not a Lucy::Object::Err")
+ unless ( blessed($val)
+ && $val->isa("Lucy::Object::Err") );
+ }
+ $error = $val;
+ }
+ sub get_error {$error}
+}
+
+{
+ package Lucy::Object::Hash;
+ no warnings 'redefine';
+ sub deserialize { shift->_deserialize(@_) }
+}
+
+{
+ package Lucy::Object::VArray;
+ no warnings 'redefine';
+ sub clone { CORE::shift->_clone }
+ sub deserialize { CORE::shift->_deserialize(@_) }
+}
+
+{
+ package Lucy::Store::FileHandle;
+ BEGIN {
+ push our @ISA, 'Exporter';
+ our @EXPORT_OK = qw( build_fh_flags );
+ }
+
+ sub build_fh_flags {
+ my $args = shift;
+ my $flags = 0;
+ $flags |= FH_CREATE if delete $args->{create};
+ $flags |= FH_READ_ONLY if delete $args->{read_only};
+ $flags |= FH_WRITE_ONLY if delete $args->{write_only};
+ $flags |= FH_EXCLUSIVE if delete $args->{exclusive};
+ return $flags;
+ }
+
+ sub open {
+ my ( $either, %args ) = @_;
+ $args{flags} ||= 0;
+ $args{flags} |= build_fh_flags( \%args );
+ return $either->_open(%args);
+ }
+}
+
+{
+ package Lucy::Store::FSFileHandle;
+
+ sub open {
+ my ( $either, %args ) = @_;
+ $args{flags} ||= 0;
+ $args{flags} |= Lucy::Store::FileHandle::build_fh_flags( \%args );
+ return $either->_open(%args);
+ }
+}
+
+{
+ package Lucy::Store::FSFolder;
+ use File::Spec::Functions qw( rel2abs );
+ sub absolutify { return rel2abs( $_[1] ) }
+}
+
+{
+ package Lucy::Store::RAMFileHandle;
+
+ sub open {
+ my ( $either, %args ) = @_;
+ $args{flags} ||= 0;
+ $args{flags} |= Lucy::Store::FileHandle::build_fh_flags( \%args );
+ return $either->_open(%args);
+ }
+}
+
+{
+ package Lucy::Util::Debug;
+ BEGIN {
+ push our @ISA, 'Exporter';
+ our @EXPORT_OK = qw(
+ DEBUG
+ DEBUG_PRINT
+ DEBUG_ENABLED
+ ASSERT
+ set_env_cache
+ num_allocated
+ num_freed
+ num_globals
+ );
+ }
+}
+
+{
+ package Lucy::Util::Json;
+ use Scalar::Util qw( blessed );
+ use Lucy qw( to_clownfish );
+ use Lucy::Util::StringHelper qw( utf8_valid utf8_flag_on );
+ use JSON::XS qw();
+
+ my $json_encoder = JSON::XS->new->pretty(1)->canonical(1);
+
+ sub slurp_json {
+ my ( undef, %args ) = @_;
+ my $result;
+ my $instream = $args{folder}->open_in( $args{path} )
+ or return;
+ my $len = $instream->length;
+ my $json;
+ $instream->read( $json, $len );
+ if ( utf8_valid($json) ) {
+ utf8_flag_on($json);
+ $result = eval { to_clownfish( $json_encoder->decode($json) ) };
+ }
+ else {
+ $@ = "Invalid UTF-8";
+ }
+ if ( $@ or !$result ) {
+ Lucy::Object::Err->set_error(
+ Lucy::Object::Err->new( $@ || "Failed to decode JSON" ) );
+ return;
+ }
+ return $result;
+ }
+
+ sub spew_json {
+ my ( undef, %args ) = @_;
+ my $json = eval { $json_encoder->encode( $args{'dump'} ) };
+ if ( !defined $json ) {
+ Lucy::Object::Err->set_error( Lucy::Object::Err->new($@) );
+ return 0;
+ }
+ my $outstream = $args{folder}->open_out( $args{path} );
+ return 0 unless $outstream;
+ eval {
+ $outstream->print($json);
+ $outstream->close;
+ };
+ if ($@) {
+ my $error;
+ if ( blessed($@) && $@->isa("Lucy::Object::Err") ) {
+ $error = $@;
+ }
+ else {
+ $error = Lucy::Object::Err->new($@);
+ }
+ Lucy::Object::Err->set_error($error);
+ return 0;
+ }
+ return 1;
+ }
+
+ sub to_json {
+ my ( undef, $dump ) = @_;
+ return $json_encoder->encode($dump);
+ }
+
+ sub from_json {
+ return to_clownfish( $json_encoder->decode( $_[1] ) );
+ }
+
+ sub set_tolerant { $json_encoder->allow_nonref( $_[1] ) }
+}
+
+{
+ package Lucy::Object::Host;
+ BEGIN {
+ if ( !__PACKAGE__->isa('Lucy::Object::Obj') ) {
+ push our @ISA, 'Lucy::Object::Obj';
+ }
+ }
+}
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy
+
+BOOT:
+ lucy_Lucy_bootstrap();
+
+IV
+_dummy_function()
+CODE:
+ RETVAL = 1;
+OUTPUT:
+ RETVAL
+
+SV*
+to_clownfish(sv)
+ SV *sv;
+CODE:
+{
+ lucy_Obj *obj = XSBind_perl_to_cfish(sv);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(obj);
+}
+OUTPUT: RETVAL
+
+SV*
+to_perl(sv)
+ SV *sv;
+CODE:
+{
+ if (sv_isobject(sv) && sv_derived_from(sv, "Lucy::Object::Obj")) {
+ IV tmp = SvIV(SvRV(sv));
+ lucy_Obj* obj = INT2PTR(lucy_Obj*, tmp);
+ RETVAL = XSBind_cfish_to_perl(obj);
+ }
+ else {
+ RETVAL = newSVsv(sv);
+ }
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy",
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/Lucy.pod b/perl/lib/Lucy.pod
new file mode 100644
index 0000000..3be5468
--- /dev/null
+++ b/perl/lib/Lucy.pod
@@ -0,0 +1,245 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy - Apache Lucy search engine library.
+
+=head1 VERSION
+
+0.1.0
+
+=head1 SYNOPSIS
+
+First, plan out your index structure, create the index, and add documents:
+
+ # indexer.pl
+
+ use Lucy::Index::Indexer;
+ use Lucy::Plan::Schema;
+ use Lucy::Analysis::PolyAnalyzer;
+ use Lucy::Plan::FullTextType;
+
+ # Create a Schema which defines index fields.
+ my $schema = Lucy::Plan::Schema->new;
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+ );
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ );
+ $schema->spec_field( name => 'title', type => $type );
+ $schema->spec_field( name => 'content', type => $type );
+
+ # Create the index and add documents.
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => '/path/to/index',
+ create => 1,
+ );
+ while ( my ( $title, $content ) = each %source_docs ) {
+ $indexer->add_doc({
+ title => $title,
+ content => $content,
+ });
+ }
+ $indexer->commit;
+
+Then, search the index:
+
+ # search.pl
+
+ use Lucy::Search::IndexSearcher;
+
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => '/path/to/index'
+ );
+ my $hits = $searcher->hits( query => "foo bar" );
+ while ( my $hit = $hits->next ) {
+ print "$hit->{title}\n";
+ }
+
+=head1 DESCRIPTION
+
+Apache Lucy is a high-performance, modular search engine library.
+
+=head2 Features
+
+=over
+
+=item *
+
+Extremely fast. A single machine can handle millions of documents.
+
+=item *
+
+Scalable to multiple machines.
+
+=item *
+
+Incremental indexing (addition/deletion of documents to/from an existing
+index).
+
+=item *
+
+Configurable near-real-time index updates.
+
+=item *
+
+Unicode support.
+
+=item *
+
+Support for boolean operators AND, OR, and AND NOT; parenthetical groupings;
+prepended +plus and -minus.
+
+=item *
+
+Algorithmic selection of relevant excerpts and highlighting of search terms
+within excerpts.
+
+=item *
+
+Highly customizable query and indexing APIs.
+
+=item *
+
+Customizable sorting.
+
+=item *
+
+Phrase matching.
+
+=item *
+
+Stemming.
+
+=item *
+
+Stoplists.
+
+=back
+
+=head2 Getting Started
+
+L<Lucy::Simple> provides a stripped down API which may suffice for many
+tasks.
+
+L<Lucy::Docs::Tutorial> demonstrates how to build a basic CGI search
+application.
+
+The tutorial spends most of its time on these five classes:
+
+=over
+
+=item *
+
+L<Lucy::Plan::Schema> - Plan out your index.
+
+=item *
+
+L<Lucy::Plan::FieldType> - Define index fields.
+
+=item *
+
+L<Lucy::Index::Indexer> - Manipulate index content.
+
+=item *
+
+L<Lucy::Search::IndexSearcher> - Search an index.
+
+=item *
+
+L<Lucy::Analysis::PolyAnalyzer> - A one-size-fits-all parser/tokenizer.
+
+=back
+
+=head2 Delving Deeper
+
+L<Lucy::Docs::Cookbook> augments the tutorial with more advanced
+recipes.
+
+For creating complex queries, see L<Lucy::Search::Query> and its
+subclasses L<TermQuery|Lucy::Search::TermQuery>,
+L<PhraseQuery|Lucy::Search::PhraseQuery>,
+L<ANDQuery|Lucy::Search::ANDQuery>,
+L<ORQuery|Lucy::Search::ORQuery>,
+L<NOTQuery|Lucy::Search::NOTQuery>,
+L<RequiredOptionalQuery|Lucy::Search::RequiredOptionalQuery>,
+L<MatchAllQuery|Lucy::Search::MatchAllQuery>, and
+L<NoMatchQuery|Lucy::Search::NoMatchQuery>, plus
+L<Lucy::Search::QueryParser>.
+
+For distributed searching, see L<LucyX::Remote::SearchServer>,
+L<LucyX::Remote::SearchClient>, and L<Lucy::Search::PolySearcher>.
+
+=head2 Backwards Compatibility Policy
+
+Lucy will spin off stable forks into new namespaces periodically. The first
+will be named "Lucy1". Users who require strong backwards compatibility
+should use a stable fork.
+
+The main namespace, "Lucy", is an API-unstable development branch (as hinted
+at by its 0.x.x version number). Superficial interface changes happen
+frequently. Hard file format compatibility breaks which require reindexing
+are rare, as we generally try to provide continuity across multiple releases,
+but we reserve the right to make such changes.
+
+=head1 CLASS METHODS
+
+The Lucy module itself does not have a large interface, providing only a
+single public class method.
+
+=head2 error
+
+ my $instream = $folder->open_in( file => 'foo' ) or die Lucy->error;
+
+Access a shared variable which is set by some routines on failure. It will
+always be either a L<Lucy::Object::Err> object or undef.
+
+=head1 SUPPORT
+
+The Apache Lucy homepage, where you'll find links to our mailing lists and so
+on, is L<http://incubator.apache.org/lucy>. Please direct support questions
+to the Lucy users mailing list.
+
+=head1 BUGS
+
+Not thread-safe.
+
+Some exceptions leak memory.
+
+If you find a bug, please inquire on the Lucy users mailing list about it,
+then report it on the Lucy issue tracker once it has been confirmed:
+L<https://issues.apache.org/jira/browse/LUCY>.
+
+=head1 DISCLAIMER
+
+Apache Lucy is an effort undergoing incubation at The Apache Software
+Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of
+all newly accepted projects until a further review indicates that the
+infrastructure, communications, and decision making process have stabilized in
+a manner consistent with other successful ASF projects. While incubation
+status is not necessarily a reflection of the completeness or stability of the
+code, it does indicate that the project has yet to be fully endorsed by the
+ASF.
+
+=head1 COPYRIGHT
+
+Apache Lucy is distributed under the Apache License, Version 2.0, as
+described in the file C<LICENSE> included with the distribution.
+
+=cut
+
diff --git a/perl/lib/Lucy/Analysis/Analyzer.pm b/perl/lib/Lucy/Analysis/Analyzer.pm
new file mode 100644
index 0000000..a4eebf7
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/Analyzer.pm
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::Analyzer;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::Analyzer",
+ bind_methods => [qw( Transform Transform_Text Split )],
+ bind_constructors => ["new"],
+ make_pod => { synopsis => " # Abstract base class.\n", }
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/CaseFolder.pm b/perl/lib/Lucy/Analysis/CaseFolder.pm
new file mode 100644
index 0000000..4b70d94
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/CaseFolder.pm
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::CaseFolder;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $case_folder = Lucy::Analysis::CaseFolder->new;
+
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $tokenizer, $stemmer ],
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $case_folder = Lucy::Analysis::CaseFolder->new;
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::CaseFolder",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/Inversion.pm b/perl/lib/Lucy/Analysis/Inversion.pm
new file mode 100644
index 0000000..8dd23d2
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/Inversion.pm
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::Inversion;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs = <<'END_XS';
+MODULE = Lucy PACKAGE = Lucy::Analysis::Inversion
+
+SV*
+new(...)
+CODE:
+{
+ lucy_Token *starter_token = NULL;
+ // parse params, only if there's more than one arg
+ if (items > 1) {
+ SV *text_sv = NULL;
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Analysis::Inversion::new_PARAMS",
+ ALLOT_SV(&text_sv, "text", 4, false),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+ if (XSBind_sv_defined(text_sv)) {
+ STRLEN len;
+ char *text = SvPVutf8(text_sv, len);
+ starter_token = lucy_Token_new(text, len, 0, len, 1.0, 1);
+ }
+ }
+
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_Inversion_new(starter_token));
+ LUCY_DECREF(starter_token);
+}
+OUTPUT: RETVAL
+END_XS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::Inversion",
+ bind_methods => [qw( Append Reset Invert Next )],
+ xs_code => $xs,
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/PolyAnalyzer.pm b/perl/lib/Lucy/Analysis/PolyAnalyzer.pm
new file mode 100644
index 0000000..726a99d
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/PolyAnalyzer.pm
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::PolyAnalyzer;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $schema = Lucy::Plan::Schema->new;
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+ );
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ );
+ $schema->spec_field( name => 'title', type => $type );
+ $schema->spec_field( name => 'content', type => $type );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $analyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'es',
+ );
+
+ # or...
+
+ my $case_folder = Lucy::Analysis::CaseFolder->new;
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'en' );
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $whitespace_tokenizer, $stemmer, ], );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::PolyAnalyzer",
+ bind_constructors => ["new"],
+ bind_methods => [qw( Get_Analyzers )],
+ make_pod => {
+ methods => [qw( get_analyzers )],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/RegexTokenizer.pm b/perl/lib/Lucy/Analysis/RegexTokenizer.pm
new file mode 100644
index 0000000..32bca4f
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/RegexTokenizer.pm
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::RegexTokenizer;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $whitespace_tokenizer
+ = Lucy::Analysis::RegexTokenizer->new( pattern => '\S+' );
+
+ # or...
+ my $word_char_tokenizer
+ = Lucy::Analysis::RegexTokenizer->new( pattern => '\w+' );
+
+ # or...
+ my $apostrophising_tokenizer = Lucy::Analysis::RegexTokenizer->new;
+
+ # Then... once you have a tokenizer, put it into a PolyAnalyzer:
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $word_char_tokenizer, $stemmer ], );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $word_char_tokenizer = Lucy::Analysis::RegexTokenizer->new(
+ pattern => '\w+', # required
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::RegexTokenizer",
+ bind_constructors => ["_new"],
+ make_pod => {
+ constructor => { sample => $constructor },
+ synopsis => $synopsis,
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/SnowballStemmer.pm b/perl/lib/Lucy/Analysis/SnowballStemmer.pm
new file mode 100644
index 0000000..c1ab1bd
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/SnowballStemmer.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::SnowballStemmer;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'es' );
+
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $tokenizer, $stemmer ],
+ );
+
+This class is a wrapper around the Snowball stemming library, so it supports
+the same languages.
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'es' );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::SnowballStemmer",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor }
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/SnowballStopFilter.pm b/perl/lib/Lucy/Analysis/SnowballStopFilter.pm
new file mode 100644
index 0000000..a40b18f
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/SnowballStopFilter.pm
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::SnowballStopFilter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
+ language => 'fr',
+ );
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $tokenizer, $stopfilter, $stemmer ],
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
+ language => 'de',
+ );
+
+ # or...
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
+ stoplist => \%stoplist,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::SnowballStopFilter",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor }
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Analysis/Token.pm b/perl/lib/Lucy/Analysis/Token.pm
new file mode 100644
index 0000000..e662448
--- /dev/null
+++ b/perl/lib/Lucy/Analysis/Token.pm
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Analysis::Token;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs = <<'END_XS';
+MODULE = Lucy PACKAGE = Lucy::Analysis::Token
+
+SV*
+new(either_sv, ...)
+ SV *either_sv;
+CODE:
+{
+ SV *text_sv = NULL;
+ uint32_t start_off = 0;
+ uint32_t end_off = 0;
+ int32_t pos_inc = 1;
+ float boost = 1.0f;
+
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Analysis::Token::new_PARAMS",
+ ALLOT_SV(&text_sv, "text", 4, true),
+ ALLOT_U32(&start_off, "start_offset", 12, true),
+ ALLOT_U32(&end_off, "end_offset", 10, true),
+ ALLOT_I32(&pos_inc, "pos_inc", 7, false),
+ ALLOT_F32(&boost, "boost", 5, false),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+
+ STRLEN len;
+ char *text = SvPVutf8(text_sv, len);
+ lucy_Token *self = (lucy_Token*)XSBind_new_blank_obj(either_sv);
+ lucy_Token_init(self, text, len, start_off, end_off, boost,
+ pos_inc);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+
+SV*
+get_text(self)
+ lucy_Token *self;
+CODE:
+ RETVAL = newSVpvn(Lucy_Token_Get_Text(self), Lucy_Token_Get_Len(self));
+ SvUTF8_on(RETVAL);
+OUTPUT: RETVAL
+
+void
+set_text(self, sv)
+ lucy_Token *self;
+ SV *sv;
+PPCODE:
+{
+ STRLEN len;
+ char *ptr = SvPVutf8(sv, len);
+ Lucy_Token_Set_Text(self, ptr, len);
+}
+END_XS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Analysis::Token",
+ bind_methods => [
+ qw(
+ Get_Start_Offset
+ Get_End_Offset
+ Get_Boost
+ Get_Pos_Inc
+ )
+ ],
+ xs_code => $xs,
+);
+
diff --git a/perl/lib/Lucy/Docs/Cookbook.pod b/perl/lib/Lucy/Docs/Cookbook.pod
new file mode 100644
index 0000000..6726db9
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Cookbook.pod
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Cookbook - Apache Lucy recipes.
+
+=head1 DESCRIPTION
+
+The Cookbook provides thematic documentation covering some of Apache Lucy's
+more sophisticated features. For a step-by-step introduction to Lucy,
+see L<Lucy::Docs::Tutorial>.
+
+=head2 Chapters
+
+=over
+
+=item *
+
+L<Lucy::Docs::Cookbook::FastUpdates> - While index updates are fast on
+average, worst-case update performance may be significantly slower. To make
+index updates consistently quick, we must manually intervene to control the
+process of index segment consolidation.
+
+=item *
+
+L<Lucy::Docs::Cookbook::CustomQuery> - Explore Lucy's support for
+custom query types by creating a "PrefixQuery" class to handle trailing
+wildcards.
+
+=item *
+
+L<Lucy::Docs::Cookbook::CustomQueryParser> - Define your own custom
+search query syntax using Lucy::Search::QueryParser and
+L<Parse::RecDescent>.
+
+=back
+
+=head2 Materials
+
+Some of the recipes in the Cookbook reference the completed
+L<Tutorial|Lucy::Docs::Tutorial> application. These materials can be
+found in the C<sample> directory at the root of the Lucy distribution:
+
+ sample/indexer.pl # indexing app
+ sample/search.cgi # search app
+ sample/us_constitution # corpus
+
+
diff --git a/perl/lib/Lucy/Docs/Cookbook/CustomQuery.pod b/perl/lib/Lucy/Docs/Cookbook/CustomQuery.pod
new file mode 100644
index 0000000..3baac9c
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Cookbook/CustomQuery.pod
@@ -0,0 +1,314 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Cookbook::CustomQuery - Sample subclass of Query.
+
+=head1 ABSTRACT
+
+Explore Apache Lucy's support for custom query types by creating a
+"PrefixQuery" class to handle trailing wildcards.
+
+ my $prefix_query = PrefixQuery->new(
+ field => 'content',
+ query_string => 'foo*',
+ );
+ my $hits = $searcher->hits( query => $prefix_query );
+ ...
+
+=head1 Query, Compiler, and Matcher
+
+To add support for a new query type, we need three classes: a Query, a
+Compiler, and a Matcher.
+
+=over
+
+=item *
+
+PrefixQuery - a subclass of L<Lucy::Search::Query>, and the only class
+that client code will deal with directly.
+
+=item *
+
+PrefixCompiler - a subclass of L<Lucy::Search::Compiler>, whose primary
+role is to compile a PrefixQuery to a PrefixMatcher.
+
+=item *
+
+PrefixMatcher - a subclass of L<Lucy::Search::Matcher>, which does the
+heavy lifting: it applies the query to individual documents and assigns a
+score to each match.
+
+=back
+
+The PrefixQuery class on its own isn't enough because a Query object's role is
+limited to expressing an abstract specification for the search. A Query is
+basically nothing but metadata; execution is left to the Query's companion
+Compiler and Matcher.
+
+Here's a simplified sketch illustrating how a Searcher's hits() method ties
+together the three classes.
+
+ sub hits {
+ my ( $self, $query ) = @_;
+ my $compiler = $query->make_compiler( searcher => $self );
+ my $matcher = $compiler->make_matcher(
+ reader => $self->get_reader,
+ need_score => 1,
+ );
+ my @hits = $matcher->capture_hits;
+ return \@hits;
+ }
+
+=head2 PrefixQuery
+
+Our PrefixQuery class will have two attributes: a query string and a field
+name.
+
+ package PrefixQuery;
+ use base qw( Lucy::Search::Query );
+ use Carp;
+ use Scalar::Util qw( blessed );
+
+ # Inside-out member vars and hand-rolled accessors.
+ my %query_string;
+ my %field;
+ sub get_query_string { my $self = shift; return $query_string{$$self} }
+ sub get_field { my $self = shift; return $field{$$self} }
+
+PrefixQuery's constructor collects and validates the attributes.
+
+ sub new {
+ my ( $class, %args ) = @_;
+ my $query_string = delete $args{query_string};
+ my $field = delete $args{field};
+ my $self = $class->SUPER::new(%args);
+ confess("'query_string' param is required")
+ unless defined $query_string;
+ confess("Invalid query_string: '$query_string'")
+ unless $query_string =~ /\*\s*$/;
+ confess("'field' param is required")
+ unless defined $field;
+ $query_string{$$self} = $query_string;
+ $field{$$self} = $field;
+ return $self;
+ }
+
+Since this is an inside-out class, we'll need a destructor:
+
+ sub DESTROY {
+ my $self = shift;
+ delete $query_string{$$self};
+ delete $field{$$self};
+ $self->SUPER::DESTROY;
+ }
+
+The equals() method determines whether two Queries are logically equivalent:
+
+ sub equals {
+ my ( $self, $other ) = @_;
+ return 0 unless blessed($other);
+ return 0 unless $other->isa("PrefixQuery");
+ return 0 unless $field{$$self} eq $field{$$other};
+ return 0 unless $query_string{$$self} eq $query_string{$$other};
+ return 1;
+ }
+
+The last thing we'll need is a make_compiler() factory method which kicks out
+a subclass of L<Compiler|Lucy::Search::Compiler>.
+
+ sub make_compiler {
+ my $self = shift;
+ return PrefixCompiler->new( @_, parent => $self );
+ }
+
+=head2 PrefixCompiler
+
+PrefixQuery's make_compiler() method will be called internally at search-time
+by objects which subclass L<Lucy::Search::Searcher> -- such as
+L<IndexSearchers|Lucy::Search::IndexSearcher>.
+
+A Searcher is associated with a particular collection of documents. These
+documents may all reside in one index, as with IndexSearcher, or they may be
+spread out across multiple indexes on one or more machines, as with
+L<Lucy::Search::PolySearcher>.
+
+Searcher objects have access to certain statistical information about the
+collections they represent; for instance, a Searcher can tell you how many
+documents are in the collection...
+
+ my $maximum_number_of_docs_in_collection = $searcher->doc_max;
+
+... or how many documents a specific term appears in:
+
+ my $term_appears_in_this_many_docs = $searcher->doc_freq(
+ field => 'content',
+ term => 'foo',
+ );
+
+Such information can be used by sophisticated Compiler implementations to
+assign more or less heft to individual queries or sub-queries. However, we're
+not going to bother with weighting for this demo; we'll just assign a fixed
+score of 1.0 to each matching document.
+
+We don't need to write a constructor, as it will suffice to inherit new() from
+Lucy::Search::Compiler. The only method we need to implement for
+PrefixCompiler is make_matcher().
+
+ package PrefixCompiler;
+ use base qw( Lucy::Search::Compiler );
+
+ sub make_matcher {
+ my ( $self, %args ) = @_;
+ my $seg_reader = $args{reader};
+
+ # Retrieve low-level components LexiconReader and PostingListReader.
+ my $lex_reader
+ = $seg_reader->obtain("Lucy::Index::LexiconReader");
+ my $plist_reader
+ = $seg_reader->obtain("Lucy::Index::PostingListReader");
+
+ # Acquire a Lexicon and seek it to our query string.
+ my $substring = $self->get_parent->get_query_string;
+ $substring =~ s/\*.\s*$//;
+ my $field = $self->get_parent->get_field;
+ my $lexicon = $lex_reader->lexicon( field => $field );
+ return unless $lexicon;
+ $lexicon->seek($substring);
+
+ # Accumulate PostingLists for each matching term.
+ my @posting_lists;
+ while ( defined( my $term = $lexicon->get_term ) ) {
+ last unless $term =~ /^\Q$substring/;
+ my $posting_list = $plist_reader->posting_list(
+ field => $field,
+ term => $term,
+ );
+ if ($posting_list) {
+ push @posting_lists, $posting_list;
+ }
+ last unless $lexicon->next;
+ }
+ return unless @posting_lists;
+
+ return PrefixMatcher->new( posting_lists => \@posting_lists );
+ }
+
+PrefixCompiler gets access to a L<SegReader|Lucy::Index::SegReader>
+object when make_matcher() gets called. From the SegReader and its
+sub-components L<LexiconReader|Lucy::Index::LexiconReader> and
+L<PostingListReader|Lucy::Index::PostingListReader>, we acquire a
+L<Lexicon|Lucy::Index::Lexicon>, scan through the Lexicon's unique
+terms, and acquire a L<PostingList|Lucy::Index::PostingList> for each
+term that matches our prefix.
+
+Each of these PostingList objects represents a set of documents which match
+the query.
+
+=head2 PrefixMatcher
+
+The Matcher subclass is the most involved.
+
+ package PrefixMatcher;
+ use base qw( Lucy::Search::Matcher );
+
+ # Inside-out member vars.
+ my %doc_ids;
+ my %tick;
+
+ sub new {
+ my ( $class, %args ) = @_;
+ my $posting_lists = delete $args{posting_lists};
+ my $self = $class->SUPER::new(%args);
+
+ # Cheesy but simple way of interleaving PostingList doc sets.
+ my %all_doc_ids;
+ for my $posting_list (@$posting_lists) {
+ while ( my $doc_id = $posting_list->next ) {
+ $all_doc_ids{$doc_id} = undef;
+ }
+ }
+ my @doc_ids = sort { $a <=> $b } keys %all_doc_ids;
+ $doc_ids{$$self} = \@doc_ids;
+
+ # Track our position within the array of doc ids.
+ $tick{$$self} = -1;
+
+ return $self;
+ }
+
+ sub DESTROY {
+ my $self = shift;
+ delete $doc_ids{$$self};
+ delete $tick{$$self};
+ $self->SUPER::DESTROY;
+ }
+
+The doc ids must be in order, or some will be ignored; hence the C<sort>
+above.
+
+In addition to the constructor and destructor, there are three methods that
+must be overridden.
+
+next() advances the Matcher to the next valid matching doc.
+
+ sub next {
+ my $self = shift;
+ my $doc_ids = $doc_ids{$$self};
+ my $tick = ++$tick{$$self};
+ return 0 if $tick >= scalar @$doc_ids;
+ return $doc_ids->[$tick];
+ }
+
+get_doc_id() returns the current document id, or 0 if the Matcher is
+exhausted. (L<Document numbers|Lucy::Docs::DocIDs> start at 1, so 0 is
+a sentinel.)
+
+ sub get_doc_id {
+ my $self = shift;
+ my $tick = $tick{$$self};
+ my $doc_ids = $doc_ids{$$self};
+ return $tick < scalar @$doc_ids ? $doc_ids->[$tick] : 0;
+ }
+
+score() conveys the relevance score of the current match. We'll just return a
+fixed score of 1.0:
+
+ sub score { 1.0 }
+
+=head1 Usage
+
+To get a basic feel for PrefixQuery, insert the FlatQueryParser module
+described in L<Lucy::Docs::Cookbook::CustomQueryParser> (which supports
+PrefixQuery) into the search.cgi sample app.
+
+ my $parser = FlatQueryParser->new( schema => $searcher->get_schema );
+ my $query = $parser->parse($q);
+
+If you're planning on using PrefixQuery in earnest, though, you may want to
+change up analyzers to avoid stemming, because stemming -- another approach to
+prefix conflation -- is not perfectly compatible with prefix searches.
+
+ # Polyanalyzer with no SnowballStemmer.
+ my $analyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [
+ Lucy::Analysis::RegexTokenizer->new,
+ Lucy::Analysis::CaseFolder->new,
+ ],
+ );
+
+=cut
+
diff --git a/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod b/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod
new file mode 100644
index 0000000..d59bc7b
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Cookbook/CustomQueryParser.pod
@@ -0,0 +1,236 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Cookbook::CustomQueryParser - Sample subclass of QueryParser.
+
+=head1 ABSTRACT
+
+Implement a custom search query language using a subclass of
+L<Lucy::Search::QueryParser>.
+
+=head1 The language
+
+At first, our query language will support only simple term queries and phrases
+delimited by double quotes. For simplicity's sake, it will not support
+parenthetical groupings, boolean operators, or prepended plus/minus. The
+results for all subqueries will be unioned together -- i.e. joined using an OR
+-- which is usually the best approach for small-to-medium-sized document
+collections.
+
+Later, we'll add support for trailing wildcards.
+
+=head1 Single-field parser
+
+Our initial parser implentation will generate queries against a single fixed
+field, "content", and it will analyze text using a fixed choice of English
+PolyAnalyzer. We won't subclass Lucy::Search::QueryParser just yet.
+
+ package FlatQueryParser;
+ use Lucy::Search::TermQuery;
+ use Lucy::Search::PhraseQuery;
+ use Lucy::Search::ORQuery;
+ use Carp;
+
+ sub new {
+ my $analyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+ );
+ return bless {
+ field => 'content',
+ analyzer => $analyzer,
+ }, __PACKAGE__;
+ }
+
+Some private helper subs for creating TermQuery and PhraseQuery objects will
+help keep the size of our main parse() subroutine down:
+
+ sub _make_term_query {
+ my ( $self, $term ) = @_;
+ return Lucy::Search::TermQuery->new(
+ field => $self->{field},
+ term => $term,
+ );
+ }
+
+ sub _make_phrase_query {
+ my ( $self, $terms ) = @_;
+ return Lucy::Search::PhraseQuery->new(
+ field => $self->{field},
+ terms => $terms,
+ );
+ }
+
+Our private _tokenize() method treats double-quote delimited material as a
+single token and splits on whitespace everywhere else.
+
+ sub _tokenize {
+ my ( $self, $query_string ) = @_;
+ my @tokens;
+ while ( length $query_string ) {
+ if ( $query_string =~ s/^\s+// ) {
+ next; # skip whitespace
+ }
+ elsif ( $query_string =~ s/^("[^"]*(?:"|$))// ) {
+ push @tokens, $1; # double-quoted phrase
+ }
+ else {
+ $query_string =~ s/(\S+)//;
+ push @tokens, $1; # single word
+ }
+ }
+ return \@tokens;
+ }
+
+The main parsing routine creates an array of tokens by calling _tokenize(),
+runs the tokens through through the PolyAnalyzer, creates TermQuery or
+PhraseQuery objects according to how many tokens emerge from the
+PolyAnalyzer's split() method, and adds each of the sub-queries to the primary
+ORQuery.
+
+ sub parse {
+ my ( $self, $query_string ) = @_;
+ my $tokens = $self->_tokenize($query_string);
+ my $analyzer = $self->{analyzer};
+ my $or_query = Lucy::Search::ORQuery->new;
+
+ for my $token (@$tokens) {
+ if ( $token =~ s/^"// ) {
+ $token =~ s/"$//;
+ my $terms = $analyzer->split($token);
+ my $query = $self->_make_phrase_query($terms);
+ $or_query->add_child($phrase_query);
+ }
+ else {
+ my $terms = $analyzer->split($token);
+ if ( @$terms == 1 ) {
+ my $query = $self->_make_term_query( $terms->[0] );
+ $or_query->add_child($query);
+ }
+ elsif ( @$terms > 1 ) {
+ my $query = $self->_make_phrase_query($terms);
+ $or_query->add_child($query);
+ }
+ }
+ }
+
+ return $or_query;
+ }
+
+=head1 Multi-field parser
+
+Most often, the end user will want their search query to match not only a
+single 'content' field, but also 'title' and so on. To make that happen, we
+have to turn queries such as this...
+
+ foo AND NOT bar
+
+... into the logical equivalent of this:
+
+ (title:foo OR content:foo) AND NOT (title:bar OR content:bar)
+
+Rather than continue with our own from-scratch parser class and write the
+routines to accomplish that expansion, we're now going to subclass Lucy::Search::QueryParser
+and take advantage of some of its existing methods.
+
+Our first parser implementation had the "content" field name and the choice of
+English PolyAnalyzer hard-coded for simplicity, but we don't need to do that
+once we subclass Lucy::Search::QueryParser. QueryParser's constructor --
+which we will inherit, allowing us to eliminate our own constructor --
+requires a Schema which conveys field
+and Analyzer information, so we can just defer to that.
+
+ package FlatQueryParser;
+ use base qw( Lucy::Search::QueryParser );
+ use Lucy::Search::TermQuery;
+ use Lucy::Search::PhraseQuery;
+ use Lucy::Search::ORQuery;
+ use PrefixQuery;
+ use Carp;
+
+ # Inherit new()
+
+We're also going to jettison our _make_term_query() and _make_phrase_query()
+helper subs and chop our parse() subroutine way down. Our revised parse()
+routine will generate Lucy::Search::LeafQuery objects instead of TermQueries
+and PhraseQueries:
+
+ sub parse {
+ my ( $self, $query_string ) = @_;
+ my $tokens = $self->_tokenize($query_string);
+ my $or_query = Lucy::Search::ORQuery->new;
+ for my $token (@$tokens) {
+ my $leaf_query = Lucy::Search::LeafQuery->new( text => $token );
+ $or_query->add_child($leaf_query);
+ }
+ return $self->expand($or_query);
+ }
+
+The magic happens in QueryParser's expand() method, which walks the ORQuery
+object we supply to it looking for LeafQuery objects, and calls expand_leaf()
+for each one it finds. expand_leaf() performs field-specific analysis,
+decides whether each query should be a TermQuery or a PhraseQuery, and if
+multiple fields are required, creates an ORQuery which mults out e.g. C<foo>
+into C<(title:foo OR content:foo)>.
+
+=head1 Extending the query language
+
+To add support for trailing wildcards to our query language, we need to
+override expand_leaf() to accommodate PrefixQuery, while deferring to the
+parent class implementation on TermQuery and PhraseQuery.
+
+ sub expand_leaf {
+ my ( $self, $leaf_query ) = @_;
+ my $text = $leaf_query->get_text;
+ if ( $text =~ /\*$/ ) {
+ my $or_query = Lucy::Search::ORQuery->new;
+ for my $field ( @{ $self->get_fields } ) {
+ my $prefix_query = PrefixQuery->new(
+ field => $field,
+ query_string => $text,
+ );
+ $or_query->add_child($prefix_query);
+ }
+ return $or_query;
+ }
+ else {
+ return $self->SUPER::expand_leaf($leaf_query);
+ }
+ }
+
+Ordinarily, those asterisks would have been stripped when running tokens
+through the PolyAnalyzer -- query strings containing "foo*" would produce
+TermQueries for the term "foo". Our override intercepts tokens with trailing
+asterisks and processes them as PrefixQueries before C<SUPER::expand_leaf> can
+discard them, so that a search for "foo*" can match "food", "foosball", and so
+on.
+
+=head1 Usage
+
+Insert our custom parser into the search.cgi sample app to get a feel for how
+it behaves:
+
+ my $parser = FlatQueryParser->new( schema => $searcher->get_schema );
+ my $query = $parser->parse( decode( 'UTF-8', $cgi->param('q') || '' ) );
+ my $hits = $searcher->hits(
+ query => $query,
+ offset => $offset,
+ num_wanted => $page_size,
+ );
+ ...
+
+=cut
+
diff --git a/perl/lib/Lucy/Docs/Cookbook/FastUpdates.pod b/perl/lib/Lucy/Docs/Cookbook/FastUpdates.pod
new file mode 100644
index 0000000..2ebebb6
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Cookbook/FastUpdates.pod
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Cookbook::FastUpdates - Near real-time index updates.
+
+=head1 ABSTRACT
+
+While index updates are fast on average, worst-case update performance may be
+significantly slower. To make index updates consistently quick, we must
+manually intervene to control the process of index segment consolidation.
+
+=head1 The problem
+
+Ordinarily, modifying an index is cheap. New data is added to new segments,
+and the time to write a new segment scales more or less linearly with the
+number of documents added during the indexing session.
+
+Deletions are also cheap most of the time, because we don't remove documents
+immediately but instead mark them as deleted, and adding the deletion mark is
+cheap.
+
+However, as new segments are added and the deletion rate for existing segments
+increases, search-time performance slowly begins to degrade. At some point,
+it becomes necessary to consolidate existing segments, rewriting their data
+into a new segment.
+
+If the recycled segments are small, the time it takes to rewrite them may not
+be significant. Every once in a while, though, a large amount of data must be
+rewritten.
+
+=head1 Procrastinating and playing catch-up
+
+The simplest way to force fast index updates is to avoid rewriting anything.
+
+Indexer relies upon L<IndexManager|Lucy::Index::IndexManager>'s
+recycle() method to tell it which segments should be consolidated. If we
+subclass IndexManager and override recycle() so that it always returns an
+empty array, we get consistently quick performance:
+
+ package NoMergeManager;
+ use base qw( Lucy::Index::IndexManager );
+ sub recycle { [] }
+
+ package main;
+ my $indexer = Lucy::Index::Indexer->new(
+ index => '/path/to/index',
+ manager => NoMergeManager->new,
+ );
+ ...
+ $indexer->commit;
+
+However, we can't procrastinate forever. Eventually, we'll have to run an
+ordinary, uncontrolled indexing session, potentially triggering a large
+rewrite of lots of small and/or degraded segments:
+
+ my $indexer = Lucy::Index::Indexer->new(
+ index => '/path/to/index',
+ # manager => NoMergeManager->new,
+ );
+ ...
+ $indexer->commit;
+
+=head1 Acceptable worst-case update time, slower degradation
+
+Never merging anything at all in the main indexing process is probably
+overkill. Small segments are relatively cheap to merge; we just need to guard
+against the big rewrites.
+
+Setting a ceiling on the number of documents in the segments to be recycled
+allows us to avoid a mass proliferation of tiny, single-document segments,
+while still offering decent worst-case update speed:
+
+ package LightMergeManager;
+ use base qw( Lucy::Index::IndexManager );
+
+ sub recycle {
+ my $self = shift;
+ my $seg_readers = $self->SUPER::recycle(@_);
+ @$seg_readers = grep { $_->doc_max < 10 } @$seg_readers;
+ return $seg_readers;
+ }
+
+However, we still have to consolidate every once in a while, and while that
+happens content updates will be locked out.
+
+=head1 Background merging
+
+If it's not acceptable to lock out updates while the index consolidation
+process runs, the alternative is to move the consolidation process out of
+band, using Lucy::Index::BackgroundMerger.
+
+It's never safe to have more than one Indexer attempting to modify the content
+of an index at the same time, but a BackgroundMerger and an Indexer can
+operate simultaneously:
+
+ # Indexing process.
+ use Scalar::Util qw( blessed );
+ my $retries = 0;
+ while (1) {
+ eval {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => '/path/to/index',
+ manager => LightMergeManager->new,
+ );
+ $indexer->add_doc($doc);
+ $indexer->commit;
+ };
+ last unless $@;
+ if ( blessed($@) and $@->isa("Lucy::Store::LockErr") ) {
+ # Catch LockErr.
+ warn "Couldn't get lock ($retries retries)";
+ $retries++;
+ }
+ else {
+ die "Write failed: $@";
+ }
+ }
+
+ # Background merge process.
+ my $manager = Lucy::Index::IndexManager->new;
+ $index_manager->set_write_lock_timeout(60_000);
+ my $bg_merger = Lucy::Index::BackgroundMerger->new(
+ index => '/path/to/index',
+ manager => $manager,
+ );
+ $bg_merger->commit;
+
+The exception handling code becomes useful once you have more than one index
+modification process happening simultaneously. By default, Indexer tries
+several times to acquire a write lock over the span of one second, then holds
+it until commit() completes. BackgroundMerger handles most of its work
+without the write lock, but it does need it briefly once at the beginning and
+once again near the end. Under normal loads, the internal retry logic will
+resolve conflicts, but if it's not acceptable to miss an insert, you probably
+want to catch LockErr exceptions thrown by Indexer. In contrast, a LockErr
+from BackgroundMerger probably just needs to be logged.
+
+=cut
+
diff --git a/perl/lib/Lucy/Docs/DevGuide.pm b/perl/lib/Lucy/Docs/DevGuide.pm
new file mode 100644
index 0000000..1ae1724
--- /dev/null
+++ b/perl/lib/Lucy/Docs/DevGuide.pm
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Docs::DevGuide",
+ make_pod => {},
+);
+
+
diff --git a/perl/lib/Lucy/Docs/DocIDs.pod b/perl/lib/Lucy/Docs/DocIDs.pod
new file mode 100644
index 0000000..4210f3d
--- /dev/null
+++ b/perl/lib/Lucy/Docs/DocIDs.pod
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::DocIDs - Characteristics of Apache Lucy document ids.
+
+=head1 DESCRIPTION
+
+=head2 Document ids are signed 32-bit integers
+
+Document ids in Apache Lucy start at 1. Because 0 is never a valid doc id, we
+can use it as a sentinel value:
+
+ while ( my $doc_id = $posting_list->next ) {
+ ...
+ }
+
+=head2 Document ids are ephemeral
+
+The document ids used by Lucy are associated with a single index
+snapshot. The moment an index is updated, the mapping of document ids to
+documents is subject to change.
+
+Since IndexReader objects represent a point-in-time view of an index, document
+ids are guaranteed to remain static for the life of the reader. However,
+because they are not permanent, Lucy document ids cannot be used as
+foreign keys to locate records in external data sources. If you truly need a
+primary key field, you must define it and populate it yourself.
+
+Furthermore, the order of document ids does not tell you anything about the
+sequence in which documents were added to the index.
+
+=cut
+
diff --git a/perl/lib/Lucy/Docs/FileFormat.pod b/perl/lib/Lucy/Docs/FileFormat.pod
new file mode 100644
index 0000000..2859442
--- /dev/null
+++ b/perl/lib/Lucy/Docs/FileFormat.pod
@@ -0,0 +1,239 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::FileFormat - Overview of index file format.
+
+=head1 OVERVIEW
+
+It is not necessary to understand the current implementation details of the
+index file format in order to use Apache Lucy effectively, but it may be
+helpful if you are interested in tweaking for high performance, exotic usage,
+or debugging and development.
+
+On a file system, an index is a directory. The files inside have a
+hierarchical relationship: an index is made up of "segments", each of which is
+an independent inverted index with its own subdirectory; each segment is made
+up of several component parts.
+
+ [index]--|
+ |--snapshot_XXX.json
+ |--schema_XXX.json
+ |--write.lock
+ |
+ |--seg_1--|
+ | |--segmeta.json
+ | |--cfmeta.json
+ | |--cf.dat-------|
+ | |--[lexicon]
+ | |--[postings]
+ | |--[documents]
+ | |--[highlight]
+ | |--[deletions]
+ |
+ |--seg_2--|
+ | |--segmeta.json
+ | |--cfmeta.json
+ | |--cf.dat-------|
+ | |--[lexicon]
+ | |--[postings]
+ | |--[documents]
+ | |--[highlight]
+ | |--[deletions]
+ |
+ |--[...]--|
+
+=head1 Write-once philosophy
+
+All segment directory names consist of the string "seg_" followed by a number
+in base 36: seg_1, seg_5m, seg_p9s2 and so on, with higher numbers indicating
+more recent segments. Once a segment is finished and committed, its name is
+never re-used and its files are never modified.
+
+Old segments become obsolete and can be removed when their data has been
+consolidated into new segments during the process of segment merging and
+optimization. A fully-optimized index has only one segment.
+
+=head1 Top-level entries
+
+There are a handful of "top-level" files and directories which belong to the
+entire index rather than to a particular segment.
+
+=head2 snapshot_XXX.json
+
+A "snapshot" file, e.g. C<snapshot_m7p.json>, is list of index files and
+directories. Because index files, once written, are never modified, the list
+of entries in a snapshot defines a point-in-time view of the data in an index.
+
+Like segment directories, snapshot files also utilize the
+unique-base-36-number naming convention; the higher the number, the more
+recent the file. The appearance of a new snapshot file within the index
+directory constitutes an index update. While a new segment is being written
+new files may be added to the index directory, but until a new snapshot file
+gets written, a Searcher opening the index for reading won't know about them.
+
+=head2 schema_XXX.json
+
+The schema file is a Schema object describing the index's format, serialized
+as JSON. It, too, is versioned, and a given snapshot file will reference one
+and only one schema file.
+
+=head2 locks
+
+By default, only one indexing process may safely modify the index at any given
+time. Processes reserve an index by laying claim to the C<write.lock> file
+within the C<locks/> directory. A smattering of other lock files may be used
+from time to time, as well.
+
+=head1 A segment's component parts
+
+By default, each segment has up to five logical components: lexicon, postings,
+document storage, highlight data, and deletions. Binary data from these
+components gets stored in virtual files within the "cf.dat" compound file;
+metadata is stored in a shared "segmeta.json" file.
+
+=head2 segmeta.json
+
+The segmeta.json file is a central repository for segment metadata. In
+addition to information such as document counts and field numbers, it also
+warehouses arbitrary metadata on behalf of individual index components.
+
+=head2 Lexicon
+
+Each indexed field gets its own lexicon in each segment. The exact files
+involved depend on the field's type, but generally speaking there will be two
+parts. First, there's a primary C<lexicon-XXX.dat> file which houses a
+complete term list associating terms with corpus frequency statistics,
+postings file locations, etc. Second, one or more "lexicon index" files may
+be present which contain periodic samples from the primary lexicon file to
+facilitate fast lookups.
+
+=head2 Postings
+
+"Posting" is a technical term from the field of
+L<information retrieval|Lucy::Docs::IRTheory>, defined as a single
+instance of a one term indexing one document. If you are looking at the index
+in the back of a book, and you see that "freedom" is referenced on pages 8,
+86, and 240, that would be three postings, which taken together form a
+"posting list". The same terminology applies to an index in electronic form.
+
+Each segment has one postings file per indexed field. When a search is
+performed for a single term, first that term is looked up in the lexicon. If
+the term exists in the segment, the record in the lexicon will contain
+information about which postings file to look at and where to look.
+
+The first thing any posting record tells you is a document id. By iterating
+over all the postings associated with a term, you can find all the documents
+that match that term, a process which is analogous to looking up page numbers
+in a book's index. However, each posting record typically contains other
+information in addition to document id, e.g. the positions at which the term
+occurs within the field.
+
+=head2 Documents
+
+The document storage section is a simple database, organized into two files:
+
+=over
+
+=item *
+
+B<documents.dat> - Serialized documents.
+
+=item *
+
+B<documents.ix> - Document storage index, a solid array of 64-bit integers
+where each integer location corresponds to a document id, and the value at
+that location points at a file position in the documents.dat file.
+
+=back
+
+=head2 Highlight data
+
+The files which store data used for excerpting and highlighting are organized
+similarly to the files used to store documents.
+
+=over
+
+=item *
+
+B<highlight.dat> - Chunks of serialized highlight data, one per doc id.
+
+=item *
+
+B<highlight.ix> - Highlight data index -- as with the C<documents.ix> file, a
+solid array of 64-bit file pointers.
+
+=back
+
+=head2 Deletions
+
+When a document is "deleted" from a segment, it is not actually purged right
+away; it is merely marked as "deleted" via a deletions file. Deletions files
+contains bit vectors with one bit for each document in the segment; if bit
+#254 is set then document 254 is deleted, and if that document turns up in a
+search it will be masked out.
+
+It is only when a segment's contents are rewritten to a new segment during the
+segment-merging process that deleted documents truly go away.
+
+=head1 Compound Files
+
+If you peer inside an index directory, you won't actually find any files named
+"documents.dat", "highlight.ix", etc. unless there is an indexing process
+underway. What you will find instead is one "cf.dat" and one "cfmeta.json"
+file per segment.
+
+To minimize the need for file descriptors at search-time, all per-segment
+binary data files are concatenated together in "cf.dat" at the close of each
+indexing session. Information about where each file begins and ends is stored
+in C<cfmeta.json>. When the segment is opened for reading, a single file
+descriptor per "cf.dat" file can be shared among several readers.
+
+=head1 A Typical Search
+
+Here's a simplified narrative, dramatizing how a search for "freedom" against
+a given segment plays out:
+
+=over
+
+=item 1
+
+The searcher asks the relevant Lexicon Index, "Do you know anything about
+'freedom'?" Lexicon Index replies, "Can't say for sure, but if the main
+Lexicon file does, 'freedom' is probably somewhere around byte 21008".
+
+=item 2
+
+The main Lexicon tells the searcher "One moment, let me scan our records...
+Yes, we have 2 documents which contain 'freedom'. You'll find them in
+seg_6/postings-4.dat starting at byte 66991."
+
+=item 3
+
+The Postings file says "Yep, we have 'freedom', all right! Document id 40
+has 1 'freedom', and document 44 has 8. If you need to know more, like if any
+'freedom' is part of the phrase 'freedom of speech', ask me about positions!
+
+=item 4
+
+If the searcher is only looking for 'freedom' in isolation, that's where it
+stops. It now knows enough to assign the documents scores against "freedom",
+with the 8-freedom document likely ranking higher than the single-freedom
+document.
+
+=back
+
+
diff --git a/perl/lib/Lucy/Docs/FileLocking.pm b/perl/lib/Lucy/Docs/FileLocking.pm
new file mode 100644
index 0000000..0a82858
--- /dev/null
+++ b/perl/lib/Lucy/Docs/FileLocking.pm
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ use Sys::Hostname qw( hostname );
+ my $hostname = hostname() or die "Can't get unique hostname";
+ my $manager = Lucy::Index::IndexManager->new( host => $hostname );
+
+ # Index time:
+ my $indexer = Lucy::Index::Indexer->new(
+ index => '/path/to/index',
+ manager => $manager,
+ );
+
+ # Search time:
+ my $reader = Lucy::Index::IndexReader->open(
+ index => '/path/to/index',
+ manager => $manager,
+ );
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $reader );
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Docs::FileLocking",
+ make_pod => { synopsis => $synopsis, },
+);
+
+
diff --git a/perl/lib/Lucy/Docs/IRTheory.pod b/perl/lib/Lucy/Docs/IRTheory.pod
new file mode 100644
index 0000000..7696ea8
--- /dev/null
+++ b/perl/lib/Lucy/Docs/IRTheory.pod
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::IRTheory - Crash course in information retrieval.
+
+=head1 ABSTRACT
+
+Just enough Information Retrieval theory to find your way around Apache Lucy.
+
+=head1 Terminology
+
+Lucy uses some terminology from the field of information retrieval which
+may be unfamiliar to many users. "Document" and "term" mean pretty much what
+you'd expect them to, but others such as "posting" and "inverted index" need a
+formal introduction:
+
+=over
+
+=item *
+
+I<document> - An atomic unit of retrieval.
+
+=item *
+
+I<term> - An attribute which describes a document.
+
+=item *
+
+I<posting> - One term indexing one document.
+
+=item *
+
+I<term list> - The complete list of terms which describe a document.
+
+=item *
+
+I<posting list> - The complete list of documents which a term indexes.
+
+=item *
+
+I<inverted index> - A data structure which maps from terms to documents.
+
+=back
+
+Since Lucy is a practical implementation of IR theory, it loads these
+abstract, distilled definitions down with useful traits. For instance, a
+"posting" in its most rarefied form is simply a term-document pairing; in
+Lucy, the class L<Lucy::Index::Posting::MatchPosting> fills this
+role. However, by associating additional information with a posting like the
+number of times the term occurs in the document, we can turn it into a
+L<ScorePosting|Lucy::Index::Posting::ScorePosting>, making it possible
+to rank documents by relevance rather than just list documents which happen to
+match in no particular order.
+
+=head1 TF/IDF ranking algorithm
+
+Lucy uses a variant of the well-established "Term Frequency / Inverse
+Document Frequency" weighting scheme. A thorough treatment of TF/IDF is too
+ambitious for our present purposes, but in a nutshell, it means that...
+
+=over
+
+=item
+
+in a search for C<skate park>, documents which score well for the
+comparatively rare term C<skate> will rank higher than documents which score
+well for the more common term C<park>.
+
+=item
+
+a 10-word text which has one occurrence each of both C<skate> and C<park> will
+rank higher than a 1000-word text which also contains one occurrence of each.
+
+=back
+
+A web search for "tf idf" will turn up many excellent explanations of the
+algorithm.
+
+=cut
+
diff --git a/perl/lib/Lucy/Docs/Tutorial.pod b/perl/lib/Lucy/Docs/Tutorial.pod
new file mode 100644
index 0000000..7ec7467
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial.pod
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial - Step-by-step introduction to Apache Lucy.
+
+=head1 ABSTRACT
+
+Explore Apache Lucy's basic functionality by starting with a minimalist CGI
+search app based on L<Lucy::Simple> and transforming it, step by step, into an
+"advanced search" interface utilizing more flexible core modules like
+L<Lucy::Index::Indexer> and L<Lucy::Search::IndexSearcher>.
+
+=head1 DESCRIPTION
+
+=head2 Chapters
+
+=over
+
+=item *
+
+L<Lucy::Docs::Tutorial::Simple> - Build a bare-bones search app using
+L<Lucy::Simple>.
+
+=item *
+
+L<Lucy::Docs::Tutorial::BeyondSimple> - Rebuild the app using core
+classes like L<Indexer|Lucy::Index::Indexer> and
+L<IndexSearcher|Lucy::Search::IndexSearcher> in place of Lucy::Simple.
+
+=item *
+
+L<Lucy::Docs::Tutorial::FieldType> - Experiment with different field
+characteristics using subclasses of L<Lucy::Plan::FieldType>.
+
+=item *
+
+L<Lucy::Docs::Tutorial::Analysis> - Examine how the choice of
+L<Lucy::Analysis::Analyzer> subclass affects search results.
+
+=item *
+
+L<Lucy::Docs::Tutorial::Highlighter> - Augment search results with
+highlighted excerpts.
+
+=item *
+
+L<Lucy::Docs::Tutorial::QueryObjects> - Unlock advanced search features
+by using Query objects instead of query strings.
+
+=back
+
+=head2 Source materials
+
+The source material used by the tutorial app -- a multi-text-file presentation
+of the United States constitution -- can be found in the C<sample> directory
+at the root of the Lucy distribution, along with finished indexing and search
+apps.
+
+ sample/indexer.pl # indexing app
+ sample/search.cgi # search app
+ sample/us_constitution # corpus
+
+=head2 Conventions
+
+The user is expected to be familiar with OO Perl and basic CGI programming.
+
+The code in this tutorial assumes a Unix-flavored operating system and the
+Apache webserver, but will work with minor modifications on other setups.
+
+=head1 SEE ALSO
+
+More advanced and esoteric subjects are covered in
+L<Lucy::Docs::Cookbook>.
+
+
diff --git a/perl/lib/Lucy/Docs/Tutorial/Analysis.pod b/perl/lib/Lucy/Docs/Tutorial/Analysis.pod
new file mode 100644
index 0000000..d935900
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial/Analysis.pod
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial::Analysis - How to choose and use Analyzers.
+
+=head1 DESCRIPTION
+
+Try swapping out the PolyAnalyzer in our Schema for a RegexTokenizer:
+
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ );
+
+Search for C<senate>, C<Senate>, and C<Senator> before and after making the
+change and re-indexing.
+
+Under PolyAnalyzer, the results are identical for all three searches, but
+under RegexTokenizer, searches are case-sensitive, and the result sets for
+C<Senate> and C<Senator> are distinct.
+
+=head2 PolyAnalyzer
+
+What's happening is that PolyAnalyzer is performing more aggressive processing
+than RegexTokenizer. In addition to tokenizing, it's also converting all text to
+lower case so that searches are case-insensitive, and using a "stemming"
+algorithm to reduce related words to a common stem (C<senat>, in this case).
+
+PolyAnalyzer is actually multiple Analyzers wrapped up in a single package.
+In this case, it's three-in-one, since specifying a PolyAnalyzer with
+C<< language => 'en' >> is equivalent to this snippet:
+
+ my $case_folder = Lucy::Analysis::CaseFolder->new;
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'en' );
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $tokenizer, $stemmer ],
+ );
+
+You can add or subtract Analyzers from there if you like. Try adding a fourth
+Analyzer, a SnowballStopFilter for suppressing "stopwords" like "the", "if",
+and "maybe".
+
+ my $stopfilter = Lucy::Analysis::SnowballStopFilter->new(
+ language => 'en',
+ );
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $tokenizer, $stopfilter, $stemmer ],
+ );
+
+Also, try removing the SnowballStemmer.
+
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $case_folder, $tokenizer ],
+ );
+
+The original choice of a stock English PolyAnalyzer probably still yields the
+best results for this document collection, but you get the idea: sometimes you
+want a different Analyzer.
+
+=head2 When the best Analyzer is no Analyzer
+
+Sometimes you don't want an Analyzer at all. That was true for our "url"
+field because we didn't need it to be searchable, but it's also true for
+certain types of searchable fields. For instance, "category" fields are often
+set up to match exactly or not at all, as are fields like "last_name" (because
+you may not want to conflate results for "Humphrey" and "Humphries").
+
+To specify that there should be no analysis performed at all, use StringType:
+
+ my $type = Lucy::Plan::StringType->new;
+ $schema->spec_field( name => 'category', type => $type );
+
+=head2 Highlighting up next
+
+In our next tutorial chapter, L<Lucy::Docs::Tutorial::Highlighter>,
+we'll add highlighted excerpts from the "content" field to our search results.
+
+
diff --git a/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod b/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod
new file mode 100644
index 0000000..768ec6c
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial/BeyondSimple.pod
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial::BeyondSimple - A more flexible app structure.
+
+=head1 DESCRIPTION
+
+=head2 Goal
+
+In this tutorial chapter, we'll refactor the apps we built in
+L<Lucy::Docs::Tutorial::Simple> so that they look exactly the same from
+the end user's point of view, but offer the developer greater possibilites for
+expansion.
+
+To achieve this, we'll ditch Lucy::Simple and replace it with the
+classes that it uses internally:
+
+=over
+
+=item *
+
+L<Lucy::Plan::Schema> - Plan out your index.
+
+=item *
+
+L<Lucy::Plan::FullTextType> - Field type for full text search.
+
+=item *
+
+L<Lucy::Analysis::PolyAnalyzer> - A one-size-fits-all parser/tokenizer.
+
+=item *
+
+L<Lucy::Index::Indexer> - Manipulate index content.
+
+=item *
+
+L<Lucy::Search::IndexSearcher> - Search an index.
+
+=item *
+
+L<Lucy::Search::Hits> - Iterate over hits returned by a Searcher.
+
+=back
+
+=head2 Adaptations to indexer.pl
+
+After we load our modules...
+
+ use Lucy::Plan::Schema;
+ use Lucy::Plan::FullTextType;
+ use Lucy::Analysis::PolyAnalyzer;
+ use Lucy::Index::Indexer;
+
+... the first item we're going need is a L<Schema|Lucy::Plan::Schema>.
+
+The primary job of a Schema is to specify what fields are available and how
+they're defined. We'll start off with three fields: title, content and url.
+
+ # Create Schema.
+ my $schema = Lucy::Plan::Schema->new;
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+ );
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ );
+ $schema->spec_field( name => 'title', type => $type );
+ $schema->spec_field( name => 'content', type => $type );
+ $schema->spec_field( name => 'url', type => $type );
+
+All of the fields are spec'd out using the "FullTextType" FieldType,
+indicating that they will be searchable as "full text" -- which means that
+they can be searched for individual words. The "analyzer", which is unique to
+FullTextType fields, is what breaks up the text into searchable tokens.
+
+Next, we'll swap our Lucy::Simple object out for a Lucy::Index::Indexer.
+The substitution will be straightforward because Simple has merely been
+serving as a thin wrapper around an inner Indexer, and we'll just be peeling
+away the wrapper.
+
+First, replace the constructor:
+
+ # Create Indexer.
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $path_to_index,
+ schema => $schema,
+ create => 1,
+ truncate => 1,
+ );
+
+Next, have the C<$indexer> object C<add_doc> where we were having the
+C<$lucy> object C<add_doc> before:
+
+ foreach my $filename (@filenames) {
+ my $doc = slurp_and_parse_file($filename);
+ $indexer->add_doc($doc);
+ }
+
+There's only one extra step required: at the end of the app, you must call
+commit() explicitly to close the indexing session and commit your changes.
+(Lucy::Simple hides this detail, calling commit() implicitly when it needs to).
+
+ $indexer->commit;
+
+=head2 Adaptations to search.cgi
+
+In our search app as in our indexing app, Lucy::Simple has served as a
+thin wrapper -- this time around L<Lucy::Search::IndexSearcher> and
+L<Lucy::Search::Hits>. Swapping out Simple for these two classes is
+also straightforward:
+
+ use Lucy::Search::IndexSearcher;
+
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => $path_to_index,
+ );
+ my $hits = $searcher->hits( # returns a Hits object, not a hit count
+ query => $q,
+ offset => $offset,
+ num_wanted => $page_size,
+ );
+ my $hit_count = $hits->total_hits; # get the hit count here
+
+ ...
+
+ while ( my $hit = $hits->next ) {
+ ...
+ }
+
+=head2 Hooray!
+
+Congratulations! Your apps do the same thing as before... but now they'll be
+easier to customize.
+
+In our next chapter, L<Lucy::Docs::Tutorial::FieldType>, we'll explore
+how to assign different behaviors to different fields.
+
+
diff --git a/perl/lib/Lucy/Docs/Tutorial/FieldType.pod b/perl/lib/Lucy/Docs/Tutorial/FieldType.pod
new file mode 100644
index 0000000..d34d31d
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial/FieldType.pod
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial::FieldType - Specify per-field properties and
+behaviors.
+
+=head1 DESCRIPTION
+
+The Schema we used in the last chapter specifies three fields:
+
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ );
+ $schema->spec_field( name => 'title', type => $type );
+ $schema->spec_field( name => 'content', type => $type );
+ $schema->spec_field( name => 'url', type => $type );
+
+Since they are all defined as "full text" fields, they are all searchable --
+including the C<url> field, a dubious choice. Some URLs contain meaningful
+information, but these don't, really:
+
+ http://example.com/us_constitution/amend1.txt
+
+We may as well not bother indexing the URL content. To achieve that we need
+to assign the C<url> field to a different FieldType.
+
+=head2 StringType
+
+Instead of FullTextType, we'll use a
+L<StringType|Lucy::Plan::StringType>, which doesn't use an
+Analyzer to break up text into individual fields. Furthermore, we'll mark
+this StringType as unindexed, so that its content won't be searchable at all.
+
+ my $url_type = Lucy::Plan::StringType( indexed => 0 );
+ $schema->spec_field( name => 'url', type => $url_type );
+
+To observe the change in behavior, try searching for C<us_constitution> both
+before and after changing the Schema and re-indexing.
+
+=head2 Toggling 'stored'
+
+For a taste of other FieldType possibilities, try turning off C<stored> for
+one or more fields.
+
+ my $content_type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ stored => 0,
+ );
+
+Turning off C<stored> for either C<title> or C<url> mangles our results page,
+but since we're not displaying C<content>, turning it off for C<content> has
+no effect -- except on index size.
+
+=head2 Analyzers up next
+
+Analyzers play a crucial role in the behavior of FullTextType fields. In our
+next tutorial chapter, L<Lucy::Docs::Tutorial::Analysis>, we'll see how
+changing up the Analyzer changes search results.
+
+
diff --git a/perl/lib/Lucy/Docs/Tutorial/Highlighter.pod b/perl/lib/Lucy/Docs/Tutorial/Highlighter.pod
new file mode 100644
index 0000000..9b6879c
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial/Highlighter.pod
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial::Highlighter - Augment search results with
+highlighted excerpts.
+
+=head1 DESCRIPTION
+
+Adding relevant excerpts with highlighted search terms to your search results
+display makes it much easier for end users to scan the page and assess which
+hits look promising, dramatically improving their search experience.
+
+=head2 Adaptations to indexer.pl
+
+L<Lucy::Highlight::Highlighter> uses information generated at index
+time. To save resources, highlighting is disabled by default and must be
+turned on for individual fields.
+
+ my $highlightable = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ highlightable => 1,
+ );
+ $schema->spec_field( name => 'content', type => $highlightable );
+
+=head2 Adaptations to search.cgi
+
+To add highlighting and excerpting to the search.cgi sample app, create a
+C<$highlighter> object outside the hits iterating loop...
+
+ my $highlighter = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content'
+ );
+
+... then modify the loop and the per-hit display to generate and include the
+excerpt.
+
+ # Create result list.
+ my $report = '';
+ while ( my $hit = $hits->next ) {
+ my $score = sprintf( "%0.3f", $hit->get_score );
+ my $excerpt = $highlighter->create_excerpt($hit);
+ $report .= qq|
+ <p>
+ <a href="$hit->{url}"><strong>$hit->{title}</strong></a>
+ <em>$score</em>
+ <br />
+ $excerpt
+ <br />
+ <span class="excerptURL">$hit->{url}</span>
+ </p>
+ |;
+ }
+
+=head2 Next chapter: Query objects
+
+Our next tutorial chapter, L<Lucy::Docs::Tutorial::QueryObjects>,
+illustrates how to build an "advanced search" interface using
+L<Query|Lucy::Search::Query> objects instead of query strings.
+
+
diff --git a/perl/lib/Lucy/Docs/Tutorial/QueryObjects.pod b/perl/lib/Lucy/Docs/Tutorial/QueryObjects.pod
new file mode 100644
index 0000000..4bf4e7f
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial/QueryObjects.pod
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial::QueryObjects - Use Query objects instead of query
+strings.
+
+=head1 DESCRIPTION
+
+Until now, our search app has had only a single search box. In this tutorial
+chapter, we'll move towards an "advanced search" interface, by adding a
+"category" drop-down menu. Three new classes will be required:
+
+=over
+
+=item *
+
+L<QueryParser|Lucy::Search::QueryParser> - Turn a query string into a
+L<Query|Lucy::Search::Query> object.
+
+=item *
+
+L<TermQuery|Lucy::Search::TermQuery> - Query for a specific term within
+a specific field.
+
+=item *
+
+L<ANDQuery|Lucy::Search::ANDQuery> - "AND" together multiple Query
+objects to produce an intersected result set.
+
+=back
+
+=head2 Adaptations to indexer.pl
+
+Our new "category" field will be a StringType field rather than a FullTextType
+field, because we will only be looking for exact matches. It needs to be
+indexed, but since we won't display its value, it doesn't need to be stored.
+
+ my $cat_type = Lucy::Plan::StringType->new( stored => 0 );
+ $schema->spec_field( name => 'category', type => $cat_type );
+
+There will be three possible values: "article", "amendment", and "preamble",
+which we'll hack out of the source file's name during our C<parse_file>
+subroutine:
+
+ my $category
+ = $filename =~ /art/ ? 'article'
+ : $filename =~ /amend/ ? 'amendment'
+ : $filename =~ /preamble/ ? 'preamble'
+ : die "Can't derive category for $filename";
+ return {
+ title => $title,
+ content => $bodytext,
+ url => "/us_constitution/$filename",
+ category => $category,
+ };
+
+=head2 Adaptations to search.cgi
+
+The "category" constraint will be added to our search interface using an HTML
+"select" element:
+
+ # Build up the HTML "select" object for the "category" field.
+ sub generate_category_select {
+ my $cat = shift;
+ my $select = qq|
+ <select name="category">
+ <option value="">All Sections</option>
+ <option value="article">Articles</option>
+ <option value="amendment">Amendments</option>
+ </select>|;
+ if ($cat) {
+ $select =~ s/"$cat"/"$cat" selected/;
+ }
+ return $select;
+ }
+
+We'll start off by loading our new modules and extracting our new CGI
+parameter.
+
+ use Lucy::Search::QueryParser;
+ use Lucy::Search::TermQuery;
+ use Lucy::Search::ANDQuery;
+
+ ...
+
+ my $category = decode( "UTF-8", $cgi->param('category') || '' );
+
+QueryParser's constructor requires a "schema" argument. We can get that from
+our IndexSearcher:
+
+ # Create an IndexSearcher and a QueryParser.
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => $path_to_index,
+ );
+ my $qparser = Lucy::Search::QueryParser->new(
+ schema => $searcher->get_schema,
+ );
+
+Previously, we have been handing raw query strings to IndexSearcher. Behind
+the scenes, IndexSearcher has been using a QueryParser to turn those query
+strings into Query objects. Now, we will bring QueryParser into the
+foreground and parse the strings explicitly.
+
+ my $query = $qparser->parse($q);
+
+If the user has specified a category, we'll use an ANDQuery to join our parsed
+query together with a TermQuery representing the category.
+
+ if ($category) {
+ my $category_query = Lucy::Search::TermQuery->new(
+ field => 'category',
+ term => $category,
+ );
+ $query = Lucy::Search::ANDQuery->new(
+ children => [ $query, $category_query ]
+ );
+ }
+
+Now when we execute the query...
+
+ # Execute the Query and get a Hits object.
+ my $hits = $searcher->hits(
+ query => $query,
+ offset => $offset,
+ num_wanted => $page_size,
+ );
+
+... we'll get a result set which is the intersection of the parsed query and
+the category query.
+
+=head1 Congratulations!
+
+You've made it to the end of the tutorial.
+
+=head1 SEE ALSO
+
+For additional thematic documentation, see the Apache Lucy
+L<Cookbook|Lucy::Docs::Cookbook>.
+
+ANDQuery has a companion class, L<ORQuery|Lucy::Search::ORQuery>, and a
+close relative,
+L<RequiredOptionalQuery|Lucy::Search::RequiredOptionalQuery>.
+
+
diff --git a/perl/lib/Lucy/Docs/Tutorial/Simple.pod b/perl/lib/Lucy/Docs/Tutorial/Simple.pod
new file mode 100644
index 0000000..435dd6c
--- /dev/null
+++ b/perl/lib/Lucy/Docs/Tutorial/Simple.pod
@@ -0,0 +1,300 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+=head1 NAME
+
+Lucy::Docs::Tutorial::Simple - Bare-bones search app.
+
+=head2 Setup
+
+Copy the text presentation of the US Constitution from the C<sample> directory
+of the Apache Lucy distribution to the base level of your web server's
+C<htdocs> directory.
+
+ $ cp -R sample/us_constitution /usr/local/apache2/htdocs/
+
+=head2 Indexing: indexer.pl
+
+Our first task will be to create an application called C<indexer.pl> which
+builds a searchable "inverted index" from a collection of documents.
+
+After we specify some configuration variables and load all necessary
+modules...
+
+ #!/usr/local/bin/perl
+ use strict;
+ use warnings;
+
+ # (Change configuration variables as needed.)
+ my $path_to_index = '/path/to/index';
+ my $uscon_source = '/usr/local/apache2/htdocs/us_constitution';
+
+ use Lucy::Simple;
+ use File::Spec::Functions qw( catfile );
+
+... we'll start by creating a Lucy::Simple object, telling it where we'd
+like the index to be located and the language of the source material.
+
+ my $lucy = Lucy::Simple->new(
+ path => $path_to_index,
+ language => 'en',
+ );
+
+Next, we'll add a subroutine which parses our sample documents.
+
+ # Parse a file from our US Constitution collection and return a hashref with
+ # the fields title, body, and url.
+ sub parse_file {
+ my $filename = shift;
+ my $filepath = catfile( $uscon_source, $filename );
+ open( my $fh, '<', $filepath ) or die "Can't open '$filepath': $!";
+ my $text = do { local $/; <$fh> }; # slurp file content
+ $text =~ /\A(.+?)^\s+(.*)/ms
+ or die "Can't extract title/bodytext from '$filepath'";
+ my $title = $1;
+ my $bodytext = $2;
+ return {
+ title => $title,
+ content => $bodytext,
+ url => "/us_constitution/$filename",
+ category => $category,
+ };
+ }
+
+Add some elementary directory reading code...
+
+ # Collect names of source files.
+ opendir( my $dh, $uscon_source )
+ or die "Couldn't opendir '$uscon_source': $!";
+ my @filenames = grep { $_ =~ /\.txt/ } readdir $dh;
+
+... and now we're ready for the meat of indexer.pl -- which occupies exactly
+one line of code.
+
+ foreach my $filename (@filenames) {
+ my $doc = parse_file($filename);
+ $lucy->add_doc($doc); # ta-da!
+ }
+
+=head2 Search: search.cgi
+
+As with our indexing app, the bulk of the code in our search script won't be
+Lucy-specific.
+
+The beginning is dedicated to CGI processing and configuration.
+
+ #!/usr/local/bin/perl -T
+ use strict;
+ use warnings;
+
+ # (Change configuration variables as needed.)
+ my $path_to_index = '/path/to/index';
+
+ use CGI;
+ use List::Util qw( max min );
+ use POSIX qw( ceil );
+ use Encode qw( decode );
+ use Lucy::Simple;
+
+ my $cgi = CGI->new;
+ my $q = decode( "UTF-8", $cgi->param('q') || '' );
+ my $offset = decode( "UTF-8", $cgi->param('offset') || 0 );
+ my $page_size = 10;
+
+Once that's out of the way, we create our Lucy::Simple object and feed
+it a query string.
+
+ my $lucy = Lucy::Simple->new(
+ path => $path_to_index,
+ language => 'en',
+ );
+ my $hit_count = $lucy->search(
+ query => $q,
+ offset => $offset,
+ num_wanted => $page_size,
+ );
+
+The value returned by search() is the total number of documents in the
+collection which matched the query. We'll show this hit count to the user,
+and also use it in conjunction with the parameters C<offset> and C<num_wanted>
+to break up results into "pages" of manageable size.
+
+Calling search() on our Simple object turns it into an iterator. Invoking
+next() now returns hits one at a time as L<Lucy::Document::HitDoc>
+objects, starting with the most relevant.
+
+ # Create result list.
+ my $report = '';
+ while ( my $hit = $lucy->next ) {
+ my $score = sprintf( "%0.3f", $hit->get_score );
+ $report .= qq|
+ <p>
+ <a href="$hit->{url}"><strong>$hit->{title}</strong></a>
+ <em>$score</em>
+ <br>
+ <span class="excerptURL">$hit->{url}</span>
+ </p>
+ |;
+ }
+
+The rest of the script is just text wrangling.
+
+ #---------------------------------------------------------------#
+ # No tutorial material below this point - just html generation. #
+ #---------------------------------------------------------------#
+
+ # Generate paging links and hit count, print and exit.
+ my $paging_links = generate_paging_info( $q, $hit_count );
+ blast_out_content( $q, $report, $paging_links );
+
+ # Create html fragment with links for paging through results n-at-a-time.
+ sub generate_paging_info {
+ my ( $query_string, $total_hits ) = @_;
+ my $escaped_q = CGI::escapeHTML($query_string);
+ my $paging_info;
+ if ( !length $query_string ) {
+ # No query? No display.
+ $paging_info = '';
+ }
+ elsif ( $total_hits == 0 ) {
+ # Alert the user that their search failed.
+ $paging_info
+ = qq|<p>No matches for <strong>$escaped_q</strong></p>|;
+ }
+ else {
+ # Calculate the nums for the first and last hit to display.
+ my $last_result = min( ( $offset + $page_size ), $total_hits );
+ my $first_result = min( ( $offset + 1 ), $last_result );
+
+ # Display the result nums, start paging info.
+ $paging_info = qq|
+ <p>
+ Results <strong>$first_result-$last_result</strong>
+ of <strong>$total_hits</strong>
+ for <strong>$escaped_q</strong>.
+ </p>
+ <p>
+ Results Page:
+ |;
+
+ # Calculate first and last hits pages to display / link to.
+ my $current_page = int( $first_result / $page_size ) + 1;
+ my $last_page = ceil( $total_hits / $page_size );
+ my $first_page = max( 1, ( $current_page - 9 ) );
+ $last_page = min( $last_page, ( $current_page + 10 ) );
+
+ # Create a url for use in paging links.
+ my $href = $cgi->url( -relative => 1 );
+ $href .= "?q=" . CGI::escape($query_string);
+ $href .= ";category=" . CGI::escape($category);
+ $href .= ";offset=" . CGI::escape($offset);
+
+ # Generate the "Prev" link.
+ if ( $current_page > 1 ) {
+ my $new_offset = ( $current_page - 2 ) * $page_size;
+ $href =~ s/(?<=offset=)\d+/$new_offset/;
+ $paging_info .= qq|<a href="$href"><= Prev</a>\n|;
+ }
+
+ # Generate paging links.
+ for my $page_num ( $first_page .. $last_page ) {
+ if ( $page_num == $current_page ) {
+ $paging_info .= qq|$page_num \n|;
+ }
+ else {
+ my $new_offset = ( $page_num - 1 ) * $page_size;
+ $href =~ s/(?<=offset=)\d+/$new_offset/;
+ $paging_info .= qq|<a href="$href">$page_num</a>\n|;
+ }
+ }
+
+ # Generate the "Next" link.
+ if ( $current_page != $last_page ) {
+ my $new_offset = $current_page * $page_size;
+ $href =~ s/(?<=offset=)\d+/$new_offset/;
+ $paging_info .= qq|<a href="$href">Next =></a>\n|;
+ }
+
+ # Close tag.
+ $paging_info .= "</p>\n";
+ }
+
+ return $paging_info;
+ }
+
+ # Print content to output.
+ sub blast_out_content {
+ my ( $query_string, $hit_list, $paging_info ) = @_;
+ my $escaped_q = CGI::escapeHTML($query_string);
+ binmode( STDOUT, ":encoding(UTF-8)" );
+ print qq|Content-type: text/html; charset=UTF-8\n\n|;
+ print qq|
+ <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+ <html>
+ <head>
+ <meta http-equiv="Content-type"
+ content="text/html;charset=UTF-8">
+ <link rel="stylesheet" type="text/css"
+ href="/us_constitution/uscon.css">
+ <title>Lucy: $escaped_q</title>
+ </head>
+
+ <body>
+
+ <div id="navigation">
+ <form id="usconSearch" action="">
+ <strong>
+ Search the
+ <a href="/us_constitution/index.html">US Constitution</a>:
+ </strong>
+ <input type="text" name="q" id="q" value="$escaped_q">
+ <input type="submit" value="=>">
+ </form>
+ </div><!--navigation-->
+
+ <div id="bodytext">
+
+ $hit_list
+
+ $paging_info
+
+ <p style="font-size: smaller; color: #666">
+ <em>
+ Powered by <a href="http://incubator.apache.org/lucy/"
+ >Apache Lucy<small><sup>TM</sup></small></a>
+ </em>
+ </p>
+ </div><!--bodytext-->
+
+ </body>
+
+ </html>
+ |;
+ }
+
+=head2 OK... now what?
+
+Lucy::Simple is perfectly adequate for some tasks, but it's not very flexible.
+Many people find that it doesn't do at least one or two things they can't live
+without.
+
+In our next tutorial chapter,
+L<BeyondSimple|Lucy::Docs::Tutorial::BeyondSimple>, we'll rewrite our
+indexing and search scripts using the classes that Lucy::Simple hides
+from view, opening up the possibilities for expansion; then, we'll spend the
+rest of the tutorial chapters exploring these possibilities.
+
+
diff --git a/perl/lib/Lucy/Document/Doc.pm b/perl/lib/Lucy/Document/Doc.pm
new file mode 100644
index 0000000..d67bb46
--- /dev/null
+++ b/perl/lib/Lucy/Document/Doc.pm
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Document::Doc;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Document::Doc
+
+SV*
+new(either_sv, ...)
+ SV *either_sv;
+CODE:
+{
+ SV* fields_sv = NULL;
+ int32_t doc_id = 0;
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Document::Doc::new_PARAMS",
+ ALLOT_SV(&fields_sv, "fields", 6, false),
+ ALLOT_I32(&doc_id, "doc_id", 6, false),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+
+ HV *fields = NULL;
+ if (fields_sv && XSBind_sv_defined(fields_sv)) {
+ if (SvROK(fields_sv)) {
+ fields = (HV*)SvRV(fields_sv);
+ }
+ if (!fields || SvTYPE((SV*)fields) != SVt_PVHV) {
+ CFISH_THROW(CFISH_ERR, "fields is not a hashref");
+ }
+ }
+
+ lucy_Doc *self = (lucy_Doc*)XSBind_new_blank_obj(either_sv);
+ lucy_Doc_init(self, fields, doc_id);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+
+SV*
+get_fields(self, ...)
+ lucy_Doc *self;
+CODE:
+ CHY_UNUSED_VAR(items);
+ RETVAL = newRV_inc((SV*)Lucy_Doc_Get_Fields(self));
+OUTPUT: RETVAL
+
+void
+set_fields(self, fields)
+ lucy_Doc *self;
+ HV *fields;
+PPCODE:
+ lucy_Doc_set_fields(self, fields);
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $doc = Lucy::Document::Doc->new(
+ fields => { foo => 'foo foo', bar => 'bar bar' },
+ );
+ $indexer->add_doc($doc);
+
+Doc objects allow access to field values via hashref overloading:
+
+ $doc->{foo} = 'new value for field "foo"';
+ print "foo: $doc->{foo}\n";
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $doc = Lucy::Document::Doc->new(
+ fields => { foo => 'foo foo', bar => 'bar bar' },
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Document::Doc",
+ xs_code => $xs_code,
+ bind_methods => [qw( Set_Doc_ID Get_Doc_ID )],
+ make_pod => {
+ methods => [qw( set_doc_id get_doc_id get_fields )],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Document/HitDoc.pm b/perl/lib/Lucy/Document/HitDoc.pm
new file mode 100644
index 0000000..da9e602
--- /dev/null
+++ b/perl/lib/Lucy/Document/HitDoc.pm
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Document::HitDoc;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Document::HitDoc
+
+SV*
+new(either_sv, ...)
+ SV *either_sv;
+CODE:
+{
+ SV *fields_sv = NULL;
+ int32_t doc_id = 0;
+ float score = 0.0f;
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Document::HitDoc::new_PARAMS",
+ ALLOT_SV(&fields_sv, "fields", 6, false),
+ ALLOT_I32(&doc_id, "doc_id", 6, false),
+ ALLOT_F32(&score, "score", 5, false),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+
+ HV *fields = NULL;
+ if (fields_sv && XSBind_sv_defined(fields_sv)) {
+ if (SvROK(fields_sv)) {
+ fields = (HV*)SvRV(fields_sv);
+ }
+ if (!fields || SvTYPE((SV*)fields) != SVt_PVHV) {
+ CFISH_THROW(CFISH_ERR, "fields is not a hashref");
+ }
+ }
+
+ lucy_HitDoc *self = (lucy_HitDoc*)XSBind_new_blank_obj(either_sv);
+ lucy_HitDoc_init(self, fields, doc_id, score);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ while ( my $hit_doc = $hits->next ) {
+ print "$hit_doc->{title}\n";
+ print $hit_doc->get_score . "\n";
+ ...
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Document::HitDoc",
+ bind_methods => [qw( Set_Score Get_Score )],
+ xs_code => $xs_code,
+ make_pod => {
+ methods => [qw( set_score get_score )],
+ synopsis => $synopsis,
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Highlight/HeatMap.pm b/perl/lib/Lucy/Highlight/HeatMap.pm
new file mode 100644
index 0000000..4463d9f
--- /dev/null
+++ b/perl/lib/Lucy/Highlight/HeatMap.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Highlight::HeatMap;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $heat_map = Lucy::Highlight::HeatMap->new(
+ spans => \@highlight_spans,
+ window => 100,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Highlight::HeatMap",
+ bind_methods => [
+ qw(
+ Calc_Proximity_Boost
+ Generate_Proximity_Boosts
+ Flatten_Spans
+ Get_Spans
+ )
+ ],
+ bind_constructors => ["new"],
+ #make_pod => {
+ # synopsis => " # TODO.\n",
+ # constructor => { sample => $constructor },
+ #},
+);
+
+
diff --git a/perl/lib/Lucy/Highlight/Highlighter.pm b/perl/lib/Lucy/Highlight/Highlighter.pm
new file mode 100644
index 0000000..97dc8a0
--- /dev/null
+++ b/perl/lib/Lucy/Highlight/Highlighter.pm
@@ -0,0 +1,93 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Highlight::Highlighter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $highlighter = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $query,
+ field => 'body'
+ );
+ my $hits = $searcher->hits( query => $query );
+ while ( my $hit = $hits->next ) {
+ my $excerpt = $highlighter->create_excerpt($hit);
+ ...
+ }
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $highlighter = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher, # required
+ query => $query, # required
+ field => 'content', # required
+ excerpt_length => 150, # default: 200
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Highlight::Highlighter",
+ bind_methods => [
+ qw(
+ Highlight
+ Encode
+ Create_Excerpt
+ _find_best_fragment|Find_Best_Fragment
+ _raw_excerpt|Raw_Excerpt
+ _highlight_excerpt|Highlight_Excerpt
+ Find_Sentences
+ Set_Pre_Tag
+ Get_Pre_Tag
+ Set_Post_Tag
+ Get_Post_Tag
+ Get_Searcher
+ Get_Query
+ Get_Compiler
+ Get_Excerpt_Length
+ Get_Field
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw(
+ create_excerpt
+ highlight
+ encode
+ set_pre_tag
+ get_pre_tag
+ set_post_tag
+ get_post_tag
+ get_searcher
+ get_query
+ get_compiler
+ get_excerpt_length
+ get_field
+ )
+ ]
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/BackgroundMerger.pm b/perl/lib/Lucy/Index/BackgroundMerger.pm
new file mode 100644
index 0000000..438533d
--- /dev/null
+++ b/perl/lib/Lucy/Index/BackgroundMerger.pm
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::BackgroundMerger;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $bg_merger = Lucy::Index::BackgroundMerger->new(
+ index => '/path/to/index',
+ );
+ $bg_merger->commit;
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $bg_merger = Lucy::Index::BackgroundMerger->new(
+ index => '/path/to/index', # required
+ manager => $manager # default: created internally
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::BackgroundMerger",
+ bind_methods => [
+ qw(
+ Commit
+ Prepare_Commit
+ Optimize
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [
+ qw(
+ commit
+ prepare_commit
+ optimize
+ )
+ ],
+ synopsis => $synopsis,
+ constructors => [ { sample => $constructor } ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/DataReader.pm b/perl/lib/Lucy/Index/DataReader.pm
new file mode 100644
index 0000000..8ac89ed
--- /dev/null
+++ b/perl/lib/Lucy/Index/DataReader.pm
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DataReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # Abstract base class.
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $reader = MyDataReader->new(
+ schema => $seg_reader->get_schema, # default undef
+ folder => $seg_reader->get_folder, # default undef
+ snapshot => $seg_reader->get_snapshot, # default undef
+ segments => $seg_reader->get_segments, # default undef
+ seg_tick => $seg_reader->get_seg_tick, # default -1
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DataReader",
+ bind_methods => [
+ qw(
+ Get_Schema
+ Get_Folder
+ Get_Segments
+ Get_Snapshot
+ Get_Seg_Tick
+ Get_Segment
+ Aggregator
+ Close
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor, },
+ methods => [
+ qw(
+ get_schema
+ get_folder
+ get_snapshot
+ get_segments
+ get_segment
+ get_seg_tick
+ aggregator
+ )
+ ]
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/DataWriter.pm b/perl/lib/Lucy/Index/DataWriter.pm
new file mode 100644
index 0000000..f7498bf
--- /dev/null
+++ b/perl/lib/Lucy/Index/DataWriter.pm
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DataWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<END_SYNOPSIS;
+ # Abstract base class.
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $writer = MyDataWriter->new(
+ snapshot => $snapshot, # required
+ segment => $segment, # required
+ polyreader => $polyreader, # required
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DataWriter",
+ bind_methods => [
+ qw(
+ Add_Inverted_Doc
+ Add_Segment
+ Delete_Segment
+ Merge_Segment
+ Finish
+ Format
+ Metadata
+ Get_Snapshot
+ Get_Segment
+ Get_PolyReader
+ Get_Schema
+ Get_Folder
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw(
+ add_inverted_doc
+ add_segment
+ delete_segment
+ merge_segment
+ finish
+ format
+ metadata
+ get_snapshot
+ get_segment
+ get_polyreader
+ get_schema
+ get_folder
+ )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/DeletionsReader.pm b/perl/lib/Lucy/Index/DeletionsReader.pm
new file mode 100644
index 0000000..088fbfb
--- /dev/null
+++ b/perl/lib/Lucy/Index/DeletionsReader.pm
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DeletionsReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DeletionsReader",
+ bind_constructors => ['new'],
+ bind_methods => [qw( Iterator Del_Count )],
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultDeletionsReader",
+ bind_constructors => ['new'],
+ bind_methods => [qw( Read_Deletions )],
+);
+
+
diff --git a/perl/lib/Lucy/Index/DeletionsWriter.pm b/perl/lib/Lucy/Index/DeletionsWriter.pm
new file mode 100644
index 0000000..207420f
--- /dev/null
+++ b/perl/lib/Lucy/Index/DeletionsWriter.pm
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DeletionsWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $polyreader = $del_writer->get_polyreader;
+ my $seg_readers = $polyreader->seg_readers;
+ for my $seg_reader (@$seg_readers) {
+ my $count = $del_writer->seg_del_count( $seg_reader->get_seg_name );
+ ...
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DeletionsWriter",
+ bind_methods => [
+ qw(
+ Generate_Doc_Map
+ Delete_By_Term
+ Delete_By_Query
+ Delete_By_Doc_ID
+ Updated
+ Seg_Deletions
+ Seg_Del_Count
+ )
+ ],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [
+ qw(
+ delete_by_term
+ delete_by_query
+ updated
+ seg_del_count
+ )
+ ],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultDeletionsWriter",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/DocReader.pm b/perl/lib/Lucy/Index/DocReader.pm
new file mode 100644
index 0000000..ddb1070
--- /dev/null
+++ b/perl/lib/Lucy/Index/DocReader.pm
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DocReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+ my $doc = $doc_reader->fetch_doc($doc_id);
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DocReader",
+ bind_constructors => ["new"],
+ bind_methods => [qw( Fetch_Doc )],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [qw( fetch_doc aggregator )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultDocReader",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/DocVector.pm b/perl/lib/Lucy/Index/DocVector.pm
new file mode 100644
index 0000000..4f490a1
--- /dev/null
+++ b/perl/lib/Lucy/Index/DocVector.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DocVector;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DocVector",
+ bind_methods => [qw( Term_Vector Field_Buf Add_Field_Buf )],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/DocWriter.pm b/perl/lib/Lucy/Index/DocWriter.pm
new file mode 100644
index 0000000..a7d9ecd
--- /dev/null
+++ b/perl/lib/Lucy/Index/DocWriter.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::DocWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DocWriter",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/FilePurger.pm b/perl/lib/Lucy/Index/FilePurger.pm
new file mode 100644
index 0000000..a118306
--- /dev/null
+++ b/perl/lib/Lucy/Index/FilePurger.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::FilePurger;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::FilePurger",
+ bind_methods => [qw( Purge )],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/HighlightReader.pm b/perl/lib/Lucy/Index/HighlightReader.pm
new file mode 100644
index 0000000..b893a23
--- /dev/null
+++ b/perl/lib/Lucy/Index/HighlightReader.pm
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::HighlightReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::HighlightReader",
+ bind_constructors => ["new"],
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultHighlightReader",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/HighlightWriter.pm b/perl/lib/Lucy/Index/HighlightWriter.pm
new file mode 100644
index 0000000..fc26615
--- /dev/null
+++ b/perl/lib/Lucy/Index/HighlightWriter.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::HighlightWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::HighlightWriter",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/IndexManager.pm b/perl/lib/Lucy/Index/IndexManager.pm
new file mode 100644
index 0000000..b2581a9
--- /dev/null
+++ b/perl/lib/Lucy/Index/IndexManager.pm
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::IndexManager;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ use Sys::Hostname qw( hostname );
+ my $hostname = hostname() or die "Can't get unique hostname";
+ my $manager = Lucy::Index::IndexManager->new(
+ host => $hostname,
+ );
+
+ # Index time:
+ my $indexer = Lucy::Index::Indexer->new(
+ index => '/path/to/index',
+ manager => $manager,
+ );
+
+ # Search time:
+ my $reader = Lucy::Index::IndexReader->open(
+ index => '/path/to/index',
+ manager => $manager,
+ );
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $reader );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $manager = Lucy::Index::IndexManager->new(
+ host => $hostname, # default: ""
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::IndexManager",
+ bind_constructors => ["new"],
+ bind_methods => [
+ qw(
+ Recycle
+ Make_Write_Lock
+ Make_Deletion_Lock
+ Make_Merge_Lock
+ Make_Snapshot_Read_Lock
+ Highest_Seg_Num
+ Make_Snapshot_Filename
+ Set_Folder
+ Get_Folder
+ Get_Host
+ Set_Write_Lock_Timeout
+ Get_Write_Lock_Timeout
+ Set_Write_Lock_Interval
+ Get_Write_Lock_Interval
+ Set_Merge_Lock_Timeout
+ Get_Merge_Lock_Timeout
+ Set_Merge_Lock_Interval
+ Get_Merge_Lock_Interval
+ Set_Deletion_Lock_Timeout
+ Get_Deletion_Lock_Timeout
+ Set_Deletion_Lock_Interval
+ Get_Deletion_Lock_Interval
+ )
+ ],
+ make_pod => {
+ methods => [
+ qw(
+ make_write_lock
+ recycle
+ set_folder
+ get_folder
+ get_host
+ set_write_lock_timeout
+ get_write_lock_timeout
+ set_write_lock_interval
+ get_write_lock_interval
+ )
+ ],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/IndexReader.pm b/perl/lib/Lucy/Index/IndexReader.pm
new file mode 100644
index 0000000..b68de63
--- /dev/null
+++ b/perl/lib/Lucy/Index/IndexReader.pm
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::IndexReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Index::IndexReader
+
+void
+set_race_condition_debug1(val_sv)
+ SV *val_sv;
+PPCODE:
+ LUCY_DECREF(lucy_PolyReader_race_condition_debug1);
+ lucy_PolyReader_race_condition_debug1 = (lucy_CharBuf*)
+ XSBind_maybe_sv_to_cfish_obj(val_sv, LUCY_CHARBUF, NULL);
+ if (lucy_PolyReader_race_condition_debug1) {
+ (void)LUCY_INCREF(lucy_PolyReader_race_condition_debug1);
+ }
+
+int32_t
+debug1_num_passes()
+CODE:
+ RETVAL = lucy_PolyReader_debug1_num_passes;
+OUTPUT: RETVAL
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $reader = Lucy::Index::IndexReader->open(
+ index => '/path/to/index',
+ );
+ my $seg_readers = $reader->seg_readers;
+ for my $seg_reader (@$seg_readers) {
+ my $seg_name = $seg_reader->get_segment->get_name;
+ my $num_docs = $seg_reader->doc_max;
+ print "Segment $seg_name ($num_docs documents):\n";
+ my $doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+ for my $doc_id ( 1 .. $num_docs ) {
+ my $doc = $doc_reader->fetch_doc($doc_id);
+ print " $doc_id: $doc->{title}\n";
+ }
+ }
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $reader = Lucy::Index::IndexReader->open(
+ index => '/path/to/index', # required
+ snapshot => $snapshot,
+ manager => $index_manager,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::IndexReader",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw( Doc_Max
+ Doc_Count
+ Del_Count
+ Fetch
+ Obtain
+ Seg_Readers
+ _offsets|Offsets
+ Get_Components
+ )
+ ],
+ bind_constructors => ['open|do_open'],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => {
+ name => 'open',
+ func => 'do_open',
+ sample => $constructor,
+ },
+ methods => [
+ qw(
+ doc_max
+ doc_count
+ del_count
+ seg_readers
+ offsets
+ fetch
+ obtain
+ )
+ ]
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/Indexer.pm b/perl/lib/Lucy/Index/Indexer.pm
new file mode 100644
index 0000000..f3329b9
--- /dev/null
+++ b/perl/lib/Lucy/Index/Indexer.pm
@@ -0,0 +1,209 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Indexer;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Index::Indexer
+
+int32_t
+CREATE(...)
+CODE:
+ CHY_UNUSED_VAR(items);
+ RETVAL = lucy_Indexer_CREATE;
+OUTPUT: RETVAL
+
+int32_t
+TRUNCATE(...)
+CODE:
+ CHY_UNUSED_VAR(items);
+ RETVAL = lucy_Indexer_TRUNCATE;
+OUTPUT: RETVAL
+
+void
+add_doc(self, ...)
+ lucy_Indexer *self;
+PPCODE:
+{
+ lucy_Doc *doc = NULL;
+ SV *doc_sv = NULL;
+ float boost = 1.0;
+
+ if (items == 2) {
+ doc_sv = ST(1);
+ }
+ else if (items > 2) {
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Index::Indexer::add_doc_PARAMS",
+ ALLOT_SV(&doc_sv, "doc", 3, true),
+ ALLOT_F32(&boost, "boost", 5, false),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+ }
+ else if (items == 1) {
+ CFISH_THROW(LUCY_ERR, "Missing required argument 'doc'");
+ }
+
+ // Either get a Doc or use the stock doc.
+ if (sv_isobject(doc_sv)
+ && sv_derived_from(doc_sv, "Lucy::Document::Doc")
+ ) {
+ IV tmp = SvIV(SvRV(doc_sv));
+ doc = INT2PTR(lucy_Doc*, tmp);
+ }
+ else if (XSBind_sv_defined(doc_sv) && SvROK(doc_sv)) {
+ HV *maybe_fields = (HV*)SvRV(doc_sv);
+ if (SvTYPE((SV*)maybe_fields) == SVt_PVHV) {
+ doc = Lucy_Indexer_Get_Stock_Doc(self);
+ Lucy_Doc_Set_Fields(doc, maybe_fields);
+ }
+ }
+ if (!doc) {
+ THROW(LUCY_ERR, "Need either a hashref or a %o",
+ Lucy_VTable_Get_Name(LUCY_DOC));
+ }
+
+ Lucy_Indexer_Add_Doc(self, doc, boost);
+}
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => '/path/to/index',
+ create => 1,
+ );
+ while ( my ( $title, $content ) = each %source_docs ) {
+ $indexer->add_doc({
+ title => $title,
+ content => $content,
+ });
+ }
+ $indexer->commit;
+END_SYNOPSIS
+
+my $constructor = <<'END_NEW';
+==head2 new( I<[labeled params]> )
+
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema, # required at index creation
+ index => '/path/to/index', # required
+ create => 1, # default: 0
+ truncate => 1, # default: 0
+ manager => $manager # default: created internally
+ );
+
+==over
+
+==item *
+
+B<schema> - A Schema. Required when index is being created; if not supplied,
+will be extracted from the index folder.
+
+==item *
+
+B<index> - Either a filepath to an index or a Folder.
+
+==item *
+
+B<create> - If true and the index directory does not exist, attempt to create
+it.
+
+==item *
+
+B<truncate> - If true, proceed with the intention of discarding all previous
+indexing data. The old data will remain intact and visible until commit()
+succeeds.
+
+==item *
+
+B<manager> - An IndexManager.
+
+==back
+END_NEW
+
+# Override is necessary because there's no standard way to explain
+# hash/hashref across multiple host languages.
+my $add_doc_pod = <<'END_ADD_DOC_POD';
+==head2 add_doc(...)
+
+ $indexer->add_doc($doc);
+ $indexer->add_doc( { field_name => $field_value } );
+ $indexer->add_doc(
+ doc => { field_name => $field_value },
+ boost => 2.5, # default: 1.0
+ );
+
+Add a document to the index. Accepts either a single argument or labeled
+params.
+
+==over
+
+==item *
+
+B<doc> - Either a Lucy::Document::Doc object, or a hashref (which will
+be attached to a Lucy::Document::Doc object internally).
+
+==item *
+
+B<boost> - A floating point weight which affects how this document scores.
+
+==back
+
+END_ADD_DOC_POD
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Indexer",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Delete_By_Term
+ Delete_By_Query
+ Add_Index
+ Commit
+ Prepare_Commit
+ Optimize
+ )
+ ],
+ bind_constructors => ["_new|init"],
+ make_pod => {
+ methods => [
+ { name => 'add_doc', pod => $add_doc_pod },
+ qw(
+ add_index
+ optimize
+ commit
+ prepare_commit
+ delete_by_term
+ delete_by_query
+ )
+ ],
+ synopsis => $synopsis,
+ constructors => [$constructor],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/Inverter.pm b/perl/lib/Lucy/Index/Inverter.pm
new file mode 100644
index 0000000..6b84a06
--- /dev/null
+++ b/perl/lib/Lucy/Index/Inverter.pm
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Inverter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Inverter",
+ bind_constructors => ["new"],
+ bind_methods => [
+ qw(
+ Get_Doc
+ Iterate
+ Next
+ Clear
+ Get_Field_Name
+ Get_Value
+ Get_Type
+ Get_Analyzer
+ Get_Similarity
+ Get_Inversion
+ )
+ ],
+);
+
+
diff --git a/perl/lib/Lucy/Index/Lexicon.pm b/perl/lib/Lucy/Index/Lexicon.pm
new file mode 100644
index 0000000..440d70a
--- /dev/null
+++ b/perl/lib/Lucy/Index/Lexicon.pm
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Lexicon;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $lex_reader = $seg_reader->obtain('Lucy::Index::LexiconReader');
+ my $lexicon = $lex_reader->lexicon( field => 'content' );
+ while ( $lexicon->next ) {
+ print $lexicon->get_term . "\n";
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Lexicon",
+ bind_methods => [qw( Seek Next Reset Get_Term Get_Field )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [qw( seek next get_term reset )],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/LexiconReader.pm b/perl/lib/Lucy/Index/LexiconReader.pm
new file mode 100644
index 0000000..d883d78
--- /dev/null
+++ b/perl/lib/Lucy/Index/LexiconReader.pm
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::LexiconReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $lex_reader = $seg_reader->obtain("Lucy::Index::LexiconReader");
+ my $lexicon = $lex_reader->lexicon( field => 'title' );
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::LexiconReader",
+ bind_methods => [qw( Lexicon Doc_Freq Fetch_Term_Info )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [qw( lexicon doc_freq )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultLexiconReader",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/LexiconWriter.pm b/perl/lib/Lucy/Index/LexiconWriter.pm
new file mode 100644
index 0000000..8de395c
--- /dev/null
+++ b/perl/lib/Lucy/Index/LexiconWriter.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::LexiconWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::LexiconWriter",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/PolyLexicon.pm b/perl/lib/Lucy/Index/PolyLexicon.pm
new file mode 100644
index 0000000..1292c87
--- /dev/null
+++ b/perl/lib/Lucy/Index/PolyLexicon.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::PolyLexicon;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::PolyLexicon",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/PolyReader.pm b/perl/lib/Lucy/Index/PolyReader.pm
new file mode 100644
index 0000000..85d35c7
--- /dev/null
+++ b/perl/lib/Lucy/Index/PolyReader.pm
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::PolyReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $polyreader = Lucy::Index::IndexReader->open(
+ index => '/path/to/index',
+ );
+ my $doc_reader = $polyreader->obtain("Lucy::Index::DocReader");
+ for my $doc_id ( 1 .. $polyreader->doc_max ) {
+ my $doc = $doc_reader->fetch_doc($doc_id);
+ print " $doc_id: $doc->{title}\n";
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::PolyReader",
+ bind_constructors => [ 'new', 'open|do_open' ],
+ bind_methods => [qw( Get_Seg_Readers )],
+ make_pod => { synopsis => $synopsis },
+);
+
+
diff --git a/perl/lib/Lucy/Index/Posting.pm b/perl/lib/Lucy/Index/Posting.pm
new file mode 100644
index 0000000..fda79a9
--- /dev/null
+++ b/perl/lib/Lucy/Index/Posting.pm
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Posting;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Posting",
+ bind_methods => [qw( Get_Doc_ID )],
+# make_pod => {
+# synopsis => " # Abstract base class.\n",
+# },
+);
+
+
diff --git a/perl/lib/Lucy/Index/Posting/MatchPosting.pm b/perl/lib/Lucy/Index/Posting/MatchPosting.pm
new file mode 100644
index 0000000..8831eb3
--- /dev/null
+++ b/perl/lib/Lucy/Index/Posting/MatchPosting.pm
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Posting::MatchPosting;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # MatchPosting is used indirectly, by specifying in FieldType subclass.
+ package MySchema::Category;
+ use base qw( Lucy::Plan::FullTextType );
+ sub posting {
+ my $self = shift;
+ return Lucy::Index::Posting::MatchPosting->new(@_);
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Posting::MatchPosting",
+ bind_constructors => ["new"],
+ bind_methods => [qw( Get_Freq )],
+# make_pod => {
+# synopsis => $synopsis,
+# }
+);
+
+
diff --git a/perl/lib/Lucy/Index/Posting/RichPosting.pm b/perl/lib/Lucy/Index/Posting/RichPosting.pm
new file mode 100644
index 0000000..cf4f79c
--- /dev/null
+++ b/perl/lib/Lucy/Index/Posting/RichPosting.pm
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Posting::RichPosting;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # RichPosting is used indirectly, by specifying in FieldType subclass.
+ package MySchema::Category;
+ use base qw( Lucy::Plan::FullTextType );
+ sub posting {
+ my $self = shift;
+ return Lucy::Index::Posting::RichPosting->new(@_);
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Posting::RichPosting",
+ bind_constructors => ["new"],
+# make_pod => {
+# synopsis => $synopsis,
+# }
+);
+
+
diff --git a/perl/lib/Lucy/Index/Posting/ScorePosting.pm b/perl/lib/Lucy/Index/Posting/ScorePosting.pm
new file mode 100644
index 0000000..f9409e7
--- /dev/null
+++ b/perl/lib/Lucy/Index/Posting/ScorePosting.pm
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Posting::ScorePosting;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Index::Posting::ScorePosting
+
+SV*
+get_prox(self)
+ lucy_ScorePosting *self;
+CODE:
+{
+ AV *out_av = newAV();
+ uint32_t *positions = Lucy_ScorePost_Get_Prox(self);
+ uint32_t i, max;
+
+ for (i = 0, max = Lucy_ScorePost_Get_Freq(self); i < max; i++) {
+ SV *pos_sv = newSVuv(positions[i]);
+ av_push(out_av, pos_sv);
+ }
+
+ RETVAL = newRV_noinc((SV*)out_av);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ # ScorePosting is used indirectly, by specifying in FieldType subclass.
+ package MySchema::Category;
+ use base qw( Lucy::Plan::FullTextType );
+ # (It's the default, so you don't need to spec it.)
+ # sub posting {
+ # my $self = shift;
+ # return Lucy::Index::Posting::ScorePosting->new(@_);
+ # }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Posting::ScorePosting",
+ xs_code => $xs_code,
+ bind_constructors => ["new"],
+# make_pod => {
+# synopsis => $synopsis,
+# }
+);
+
+
diff --git a/perl/lib/Lucy/Index/PostingList.pm b/perl/lib/Lucy/Index/PostingList.pm
new file mode 100644
index 0000000..5f0c3cc
--- /dev/null
+++ b/perl/lib/Lucy/Index/PostingList.pm
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::PostingList;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $posting_list_reader
+ = $seg_reader->obtain("Lucy::Index::PostingListReader");
+ my $posting_list = $posting_list_reader->posting_list(
+ field => 'content',
+ term => 'foo',
+ );
+ while ( my $doc_id = $posting_list->next ) {
+ say "Matching doc id: $doc_id";
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::PostingList",
+ bind_methods => [
+ qw(
+ Seek
+ Get_Posting
+ Get_Doc_Freq
+ Make_Matcher
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [
+ qw(
+ next
+ advance
+ get_doc_id
+ get_doc_freq
+ seek
+ )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/PostingListReader.pm b/perl/lib/Lucy/Index/PostingListReader.pm
new file mode 100644
index 0000000..86bb59d
--- /dev/null
+++ b/perl/lib/Lucy/Index/PostingListReader.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::PostingListReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $posting_list_reader
+ = $seg_reader->obtain("Lucy::Index::PostingListReader");
+ my $posting_list = $posting_list_reader->posting_list(
+ field => 'title',
+ term => 'foo',
+ );
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::PostingListReader",
+ bind_constructors => ["new"],
+ bind_methods => [qw( Posting_List Get_Lex_Reader )],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [qw( posting_list )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultPostingListReader",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/PostingListWriter.pm b/perl/lib/Lucy/Index/PostingListWriter.pm
new file mode 100644
index 0000000..e21a5e4
--- /dev/null
+++ b/perl/lib/Lucy/Index/PostingListWriter.pm
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::PostingListWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS';
+MODULE = Lucy PACKAGE = Lucy::Index::PostingListWriter
+
+void
+set_default_mem_thresh(mem_thresh)
+ size_t mem_thresh;
+PPCODE:
+ lucy_PListWriter_set_default_mem_thresh(mem_thresh);
+END_XS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::PostingListWriter",
+ xs_code => $xs_code,
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/SegLexicon.pm b/perl/lib/Lucy/Index/SegLexicon.pm
new file mode 100644
index 0000000..ee40ab5
--- /dev/null
+++ b/perl/lib/Lucy/Index/SegLexicon.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SegLexicon;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SegLexicon",
+ bind_methods => [qw( Get_Term_Info Get_Field_Num )],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/SegPostingList.pm b/perl/lib/Lucy/Index/SegPostingList.pm
new file mode 100644
index 0000000..0c0c8a7
--- /dev/null
+++ b/perl/lib/Lucy/Index/SegPostingList.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SegPostingList;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SegPostingList",
+ bind_methods => [qw( Get_Post_Stream Get_Count )],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/SegReader.pm b/perl/lib/Lucy/Index/SegReader.pm
new file mode 100644
index 0000000..54b6ab0
--- /dev/null
+++ b/perl/lib/Lucy/Index/SegReader.pm
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SegReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $polyreader = Lucy::Index::IndexReader->open(
+ index => '/path/to/index',
+ );
+ my $seg_readers = $polyreader->seg_readers;
+ for my $seg_reader (@$seg_readers) {
+ my $seg_name = $seg_reader->get_seg_name;
+ my $num_docs = $seg_reader->doc_max;
+ print "Segment $seg_name ($num_docs documents):\n";
+ my $doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+ for my $doc_id ( 1 .. $num_docs ) {
+ my $doc = $doc_reader->fetch_doc($doc_id);
+ print " $doc_id: $doc->{title}\n";
+ }
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SegReader",
+ bind_methods => [qw( Get_Seg_Name Get_Seg_Num Register )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [qw( Get_Seg_Name Get_Seg_Num )],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/SegWriter.pm b/perl/lib/Lucy/Index/SegWriter.pm
new file mode 100644
index 0000000..81da014
--- /dev/null
+++ b/perl/lib/Lucy/Index/SegWriter.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SegWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SegWriter",
+ bind_constructors => ["new"],
+ bind_methods => [
+ qw(
+ Add_Writer
+ Register
+ Fetch
+ )
+ ],
+ make_pod => {
+ methods => [
+ qw(
+ add_doc
+ add_writer
+ register
+ fetch
+ )
+ ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Index/Segment.pm b/perl/lib/Lucy/Index/Segment.pm
new file mode 100644
index 0000000..9f5c267
--- /dev/null
+++ b/perl/lib/Lucy/Index/Segment.pm
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Segment;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # Index-time.
+ package MyDataWriter;
+ use base qw( Lucy::Index::DataWriter );
+
+ sub finish {
+ my $self = shift;
+ my $segment = $self->get_segment;
+ my $metadata = $self->SUPER::metadata();
+ $metadata->{foo} = $self->get_foo;
+ $segment->store_metadata(
+ key => 'my_component',
+ metadata => $metadata
+ );
+ }
+
+ # Search-time.
+ package MyDataReader;
+ use base qw( Lucy::Index::DataReader );
+
+ sub new {
+ my $self = shift->SUPER::new(@_);
+ my $segment = $self->get_segment;
+ my $metadata = $segment->fetch_metadata('my_component');
+ if ($metadata) {
+ $self->set_foo( $metadata->{foo} );
+ ...
+ }
+ return $self;
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Segment",
+ bind_methods => [
+ qw(
+ Add_Field
+ _store_metadata|Store_Metadata
+ Fetch_Metadata
+ Field_Num
+ Field_Name
+ Get_Name
+ Get_Number
+ Set_Count
+ Get_Count
+ Write_File
+ Read_File
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [
+ qw(
+ add_field
+ store_metadata
+ fetch_metadata
+ field_num
+ field_name
+ get_name
+ get_number
+ set_count
+ get_count
+ )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/Similarity.pm b/perl/lib/Lucy/Index/Similarity.pm
new file mode 100644
index 0000000..7f5bd69
--- /dev/null
+++ b/perl/lib/Lucy/Index/Similarity.pm
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Similarity;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Index::Similarity
+
+SV*
+get_norm_decoder(self)
+ lucy_Similarity *self;
+CODE:
+ RETVAL = newSVpvn((char*)Lucy_Sim_Get_Norm_Decoder(self),
+ (256 * sizeof(float)));
+OUTPUT: RETVAL
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ package MySimilarity;
+
+ sub length_norm { return 1.0 } # disable length normalization
+
+ package MyFullTextType;
+ use base qw( Lucy::Plan::FullTextType );
+
+ sub make_similarity { MySimilarity->new }
+END_SYNOPSIS
+
+my $constructor = qq| my \$sim = Lucy::Index::Similarity->new;\n|;
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Similarity",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw( IDF
+ TF
+ Encode_Norm
+ Decode_Norm
+ Query_Norm
+ Length_Norm
+ Coord )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [qw( length_norm )],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Index/Snapshot.pm b/perl/lib/Lucy/Index/Snapshot.pm
new file mode 100644
index 0000000..872b622
--- /dev/null
+++ b/perl/lib/Lucy/Index/Snapshot.pm
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::Snapshot;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $snapshot = Lucy::Index::Snapshot->new;
+ $snapshot->read_file( folder => $folder ); # load most recent snapshot
+ my $files = $snapshot->list;
+ print "$_\n" for @$files;
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $snapshot = Lucy::Index::Snapshot->new;
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::Snapshot",
+ bind_methods => [
+ qw(
+ List
+ Num_Entries
+ Add_Entry
+ Delete_Entry
+ Read_File
+ Write_File
+ Set_Path
+ Get_Path
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw(
+ list
+ num_entries
+ add_entry
+ delete_entry
+ read_file
+ write_file
+ set_path
+ get_path
+ )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Index/SortCache.pm b/perl/lib/Lucy/Index/SortCache.pm
new file mode 100644
index 0000000..7e83a5c
--- /dev/null
+++ b/perl/lib/Lucy/Index/SortCache.pm
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SortCache;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Index::SortCache
+
+SV*
+value(self, ...)
+ lucy_SortCache *self;
+CODE:
+{
+ int32_t ord = 0;
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Index::SortCache::value_PARAMS",
+ ALLOT_I32(&ord, "ord", 3, false),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+ {
+ lucy_Obj *blank = Lucy_SortCache_Make_Blank(self);
+ lucy_Obj *value = Lucy_SortCache_Value(self, ord, blank);
+ RETVAL = XSBind_cfish_to_perl(value);
+ LUCY_DECREF(blank);
+ }
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SortCache",
+ xs_code => $xs_code,
+ bind_methods => [qw( Ordinal Find )],
+);
+
+
diff --git a/perl/lib/Lucy/Index/SortReader.pm b/perl/lib/Lucy/Index/SortReader.pm
new file mode 100644
index 0000000..b75f6c2
--- /dev/null
+++ b/perl/lib/Lucy/Index/SortReader.pm
@@ -0,0 +1,37 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SortReader;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SortReader",
+ bind_constructors => ["new"],
+ bind_methods => [qw( Fetch_Sort_Cache )],
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::DefaultSortReader",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/SortWriter.pm b/perl/lib/Lucy/Index/SortWriter.pm
new file mode 100644
index 0000000..e813e9a
--- /dev/null
+++ b/perl/lib/Lucy/Index/SortWriter.pm
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::SortWriter;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS';
+MODULE = Lucy PACKAGE = Lucy::Index::SortWriter
+
+void
+set_default_mem_thresh(mem_thresh)
+ size_t mem_thresh;
+PPCODE:
+ lucy_SortWriter_set_default_mem_thresh(mem_thresh);
+END_XS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::SortWriter",
+ xs_code => $xs_code,
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/TermInfo.pm b/perl/lib/Lucy/Index/TermInfo.pm
new file mode 100644
index 0000000..8272b8f
--- /dev/null
+++ b/perl/lib/Lucy/Index/TermInfo.pm
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::TermInfo;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::TermInfo",
+ bind_methods => [
+ qw(
+ Get_Doc_Freq
+ Get_Lex_FilePos
+ Get_Post_FilePos
+ Get_Skip_FilePos
+ Set_Doc_Freq
+ Set_Lex_FilePos
+ Set_Post_FilePos
+ Set_Skip_FilePos
+ Reset
+ )
+ ],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Index/TermVector.pm b/perl/lib/Lucy/Index/TermVector.pm
new file mode 100644
index 0000000..e8ec95d
--- /dev/null
+++ b/perl/lib/Lucy/Index/TermVector.pm
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Index::TermVector;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Index::TermVector",
+ bind_constructors => ["new"],
+ bind_methods => [
+ qw(
+ Get_Positions
+ Get_Start_Offsets
+ Get_End_Offsets
+ )
+ ],
+);
+
+
diff --git a/perl/lib/Lucy/Object/BitVector.pm b/perl/lib/Lucy/Object/BitVector.pm
new file mode 100644
index 0000000..c484f9c
--- /dev/null
+++ b/perl/lib/Lucy/Object/BitVector.pm
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::BitVector;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $bit_vec = Lucy::Object::BitVector->new( capacity => 8 );
+ my $other = Lucy::Object::BitVector->new( capacity => 8 );
+ $bit_vec->set($_) for ( 0, 2, 4, 6 );
+ $other->set($_) for ( 1, 3, 5, 7 );
+ $bit_vec->or($other);
+ print "$_\n" for @{ $bit_vec->to_array }; # prints 0 through 7.
+END_SYNOPSIS
+my $constructor = <<'END_CONSTRUCTOR';
+ my $bit_vec = Lucy::Object::BitVector->new(
+ capacity => $doc_max + 1, # default 0,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::BitVector",
+ bind_methods => [
+ qw( Get
+ Set
+ Clear
+ Clear_All
+ And
+ Or
+ And_Not
+ Xor
+ Flip
+ Flip_Block
+ Next_Hit
+ To_Array
+ Grow
+ Count
+ Get_Capacity
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw( get
+ set
+ clear
+ clear_all
+ and
+ or
+ and_not
+ xor
+ flip
+ flip_block
+ next_hit
+ to_array
+ grow
+ count
+ )
+ ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Object/ByteBuf.pm b/perl/lib/Lucy/Object/ByteBuf.pm
new file mode 100644
index 0000000..3cded5b
--- /dev/null
+++ b/perl/lib/Lucy/Object/ByteBuf.pm
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::ByteBuf;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::ByteBuf
+
+SV*
+new(either_sv, sv)
+ SV *either_sv;
+ SV *sv;
+CODE:
+{
+ STRLEN size;
+ char *ptr = SvPV(sv, size);
+ lucy_ByteBuf *self = (lucy_ByteBuf*)XSBind_new_blank_obj(either_sv);
+ lucy_BB_init(self, size);
+ Lucy_BB_Mimic_Bytes(self, ptr, size);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+
+SV*
+_deserialize(either_sv, instream)
+ SV *either_sv;
+ lucy_InStream *instream;
+CODE:
+ CHY_UNUSED_VAR(either_sv);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_BB_deserialize(NULL, instream));
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::ByteBuf",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Get_Size
+ Get_Capacity
+ Cat
+ )
+ ],
+);
+
+
diff --git a/perl/lib/Lucy/Object/CharBuf.pm b/perl/lib/Lucy/Object/CharBuf.pm
new file mode 100644
index 0000000..f3c4994
--- /dev/null
+++ b/perl/lib/Lucy/Object/CharBuf.pm
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::CharBuf;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::CharBuf
+
+SV*
+new(either_sv, sv)
+ SV *either_sv;
+ SV *sv;
+CODE:
+{
+ STRLEN size;
+ char *ptr = SvPVutf8(sv, size);
+ lucy_CharBuf *self = (lucy_CharBuf*)XSBind_new_blank_obj(either_sv);
+ lucy_CB_init(self, size);
+ Lucy_CB_Cat_Trusted_Str(self, ptr, size);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+
+SV*
+_clone(self)
+ lucy_CharBuf *self;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_CB_clone(self));
+OUTPUT: RETVAL
+
+SV*
+_deserialize(either_sv, instream)
+ SV *either_sv;
+ lucy_InStream *instream;
+CODE:
+ CHY_UNUSED_VAR(either_sv);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_CB_deserialize(NULL, instream));
+OUTPUT: RETVAL
+
+SV*
+to_perl(self)
+ lucy_CharBuf *self;
+CODE:
+ RETVAL = XSBind_cb_to_sv(self);
+OUTPUT: RETVAL
+
+MODULE = Lucy PACKAGE = Lucy::Object::ViewCharBuf
+
+SV*
+_new(unused, sv)
+ SV *unused;
+ SV *sv;
+CODE:
+{
+ STRLEN size;
+ char *ptr = SvPVutf8(sv, size);
+ lucy_ViewCharBuf *self
+ = lucy_ViewCB_new_from_trusted_utf8(ptr, size);
+ CHY_UNUSED_VAR(unused);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::CharBuf",
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/Lucy/Object/Err.pm b/perl/lib/Lucy/Object/Err.pm
new file mode 100644
index 0000000..333f1b8
--- /dev/null
+++ b/perl/lib/Lucy/Object/Err.pm
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::Err;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ use Scalar::Util qw( blessed );
+ my $bg_merger;
+ while (1) {
+ $bg_merger = eval {
+ Lucy::Index::BackgroundMerger->new( index => $index );
+ };
+ last if $bg_merger;
+ if ( blessed($@) and $@->isa("Lucy::Store::LockErr") ) {
+ warn "Retrying...\n";
+ }
+ else {
+ # Re-throw.
+ die "Failed to open BackgroundMerger: $@";
+ }
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::Err",
+ bind_methods => [qw( Cat_Mess Get_Mess )],
+ make_pod => { synopsis => $synopsis },
+ bind_constructors => ["_new"],
+);
+
+
diff --git a/perl/lib/Lucy/Object/Hash.pm b/perl/lib/Lucy/Object/Hash.pm
new file mode 100644
index 0000000..7e70b40
--- /dev/null
+++ b/perl/lib/Lucy/Object/Hash.pm
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::Hash;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::Hash
+
+SV*
+_deserialize(either_sv, instream)
+ SV *either_sv;
+ lucy_InStream *instream;
+CODE:
+ CHY_UNUSED_VAR(either_sv);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_Hash_deserialize(NULL, instream));
+OUTPUT: RETVAL
+
+SV*
+_fetch(self, key)
+ lucy_Hash *self;
+ const lucy_CharBuf *key;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV(lucy_Hash_fetch(self, (lucy_Obj*)key));
+OUTPUT: RETVAL
+
+void
+store(self, key, value);
+ lucy_Hash *self;
+ const lucy_CharBuf *key;
+ lucy_Obj *value;
+PPCODE:
+{
+ if (value) { LUCY_INCREF(value); }
+ lucy_Hash_store(self, (lucy_Obj*)key, value);
+}
+
+void
+next(self)
+ lucy_Hash *self;
+PPCODE:
+{
+ lucy_Obj *key;
+ lucy_Obj *val;
+
+ if (Lucy_Hash_Next(self, &key, &val)) {
+ SV *key_sv = (SV*)Lucy_Obj_To_Host(key);
+ SV *val_sv = (SV*)Lucy_Obj_To_Host(val);
+
+ XPUSHs(sv_2mortal(key_sv));
+ XPUSHs(sv_2mortal(val_sv));
+ XSRETURN(2);
+ }
+ else {
+ XSRETURN_EMPTY;
+ }
+}
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::Hash",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Fetch
+ Delete
+ Keys
+ Values
+ Find_Key
+ Clear
+ Iterate
+ Get_Size
+ )
+ ],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Object/Host.pm b/perl/lib/Lucy/Object/Host.pm
new file mode 100644
index 0000000..7237a05
--- /dev/null
+++ b/perl/lib/Lucy/Object/Host.pm
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::Host;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::Host
+
+=for comment
+
+These are all for testing purposes only.
+
+=cut
+
+IV
+_test(...)
+CODE:
+ RETVAL = items;
+OUTPUT: RETVAL
+
+SV*
+_test_obj(...)
+CODE:
+{
+ lucy_ByteBuf *test_obj = lucy_BB_new_bytes("blah", 4);
+ SV *pack_var = get_sv("Lucy::Object::Host::testobj", 1);
+ RETVAL = (SV*)Lucy_BB_To_Host(test_obj);
+ SvSetSV_nosteal(pack_var, RETVAL);
+ LUCY_DECREF(test_obj);
+ CHY_UNUSED_VAR(items);
+}
+OUTPUT: RETVAL
+
+void
+_callback(obj)
+ lucy_Obj *obj;
+PPCODE:
+{
+ lucy_ZombieCharBuf *blank = CFISH_ZCB_BLANK();
+ lucy_Host_callback(obj, "_test", 2,
+ CFISH_ARG_OBJ("nothing", (lucy_CharBuf*)blank),
+ CFISH_ARG_I32("foo", 3));
+}
+
+int64_t
+_callback_i64(obj)
+ lucy_Obj *obj;
+CODE:
+{
+ lucy_ZombieCharBuf *blank = CFISH_ZCB_BLANK();
+ RETVAL
+ = lucy_Host_callback_i64(obj, "_test", 2,
+ CFISH_ARG_OBJ("nothing", (lucy_CharBuf*)blank),
+ CFISH_ARG_I32("foo", 3));
+}
+OUTPUT: RETVAL
+
+double
+_callback_f64(obj)
+ lucy_Obj *obj;
+CODE:
+{
+ lucy_ZombieCharBuf *blank = CFISH_ZCB_BLANK();
+ RETVAL
+ = lucy_Host_callback_f64(obj, "_test", 2,
+ CFISH_ARG_OBJ("nothing", (lucy_CharBuf*)blank),
+ CFISH_ARG_I32("foo", 3));
+}
+OUTPUT: RETVAL
+
+SV*
+_callback_obj(obj)
+ lucy_Obj *obj;
+CODE:
+{
+ lucy_Obj *other = lucy_Host_callback_obj(obj, "_test_obj", 0);
+ RETVAL = (SV*)Lucy_Obj_To_Host(other);
+ LUCY_DECREF(other);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::Host",
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/Lucy/Object/I32Array.pm b/perl/lib/Lucy/Object/I32Array.pm
new file mode 100644
index 0000000..f5fd074
--- /dev/null
+++ b/perl/lib/Lucy/Object/I32Array.pm
@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::I32Array;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::I32Array
+
+SV*
+new(either_sv, ...)
+ SV *either_sv;
+CODE:
+{
+ SV *ints_sv = NULL;
+ lucy_I32Array *self = NULL;
+
+ chy_bool_t args_ok
+ = XSBind_allot_params(&(ST(0)), 1, items,
+ "Lucy::Object::I32Array::new_PARAMS",
+ ALLOT_SV(&ints_sv, "ints", 4, true),
+ NULL);
+ if (!args_ok) {
+ CFISH_RETHROW(LUCY_INCREF(cfish_Err_get_error()));
+ }
+
+ AV *ints_av = NULL;
+ if (SvROK(ints_sv)) {
+ ints_av = (AV*)SvRV(ints_sv);
+ }
+ if (ints_av && SvTYPE(ints_av) == SVt_PVAV) {
+ int32_t size = av_len(ints_av) + 1;
+ int32_t *ints = (int32_t*)LUCY_MALLOCATE(size * sizeof(int32_t));
+ int32_t i;
+
+ for (i = 0; i < size; i++) {
+ SV **const sv_ptr = av_fetch(ints_av, i, 0);
+ ints[i] = (sv_ptr && XSBind_sv_defined(*sv_ptr))
+ ? SvIV(*sv_ptr)
+ : 0;
+ }
+ self = (lucy_I32Array*)XSBind_new_blank_obj(either_sv);
+ lucy_I32Arr_init(self, ints, size);
+ }
+ else {
+ THROW(LUCY_ERR, "Required param 'ints' isn't an arrayref");
+ }
+
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+
+SV*
+to_arrayref(self)
+ lucy_I32Array *self;
+CODE:
+{
+ AV *out_av = newAV();
+ uint32_t i;
+ uint32_t size = Lucy_I32Arr_Get_Size(self);
+
+ av_extend(out_av, size);
+ for (i = 0; i < size; i++) {
+ int32_t result = Lucy_I32Arr_Get(self, i);
+ SV* result_sv = result == -1 ? newSV(0) : newSViv(result);
+ av_push(out_av, result_sv);
+ }
+ RETVAL = newRV_noinc((SV*)out_av);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::I32Array",
+ xs_code => $xs_code,
+ bind_methods => [qw( Get Get_Size )],
+);
+
+
diff --git a/perl/lib/Lucy/Object/LockFreeRegistry.pm b/perl/lib/Lucy/Object/LockFreeRegistry.pm
new file mode 100644
index 0000000..74ce1e9
--- /dev/null
+++ b/perl/lib/Lucy/Object/LockFreeRegistry.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::LockFreeRegistry;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::LockFreeRegistry",
+ bind_methods => [qw( Register Fetch )],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Object/Num.pm b/perl/lib/Lucy/Object/Num.pm
new file mode 100644
index 0000000..d7bddbf
--- /dev/null
+++ b/perl/lib/Lucy/Object/Num.pm
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::Num;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $float32_xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::Float32
+
+SV*
+new(either_sv, value)
+ SV *either_sv;
+ float value;
+CODE:
+{
+ lucy_Float32 *self = (lucy_Float32*)XSBind_new_blank_obj(either_sv);
+ lucy_Float32_init(self, value);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+my $float64_xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::Float64
+
+SV*
+new(either_sv, value)
+ SV *either_sv;
+ double value;
+CODE:
+{
+ lucy_Float64 *self = (lucy_Float64*)XSBind_new_blank_obj(either_sv);
+ lucy_Float64_init(self, value);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(self);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::Float32",
+ xs_code => $float32_xs_code,
+ bind_methods => [qw( Set_Value Get_Value )],
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::Float64",
+ xs_code => $float64_xs_code,
+ bind_methods => [qw( Set_Value Get_Value )],
+);
+
+
diff --git a/perl/lib/Lucy/Object/Obj.pm b/perl/lib/Lucy/Object/Obj.pm
new file mode 100644
index 0000000..c29285f
--- /dev/null
+++ b/perl/lib/Lucy/Object/Obj.pm
@@ -0,0 +1,249 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::Obj;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::Obj
+
+chy_bool_t
+is_a(self, class_name)
+ lucy_Obj *self;
+ const lucy_CharBuf *class_name;
+CODE:
+{
+ lucy_VTable *target = lucy_VTable_fetch_vtable(class_name);
+ RETVAL = Lucy_Obj_Is_A(self, target);
+}
+OUTPUT: RETVAL
+
+void
+STORABLE_freeze(self, ...)
+ lucy_Obj *self;
+PPCODE:
+{
+ CHY_UNUSED_VAR(self);
+ if (items < 2 || !SvTRUE(ST(1))) {
+ SV *retval;
+ lucy_ByteBuf *serialized_bb;
+ lucy_RAMFileHandle *file_handle
+ = lucy_RAMFH_open(NULL, LUCY_FH_WRITE_ONLY | LUCY_FH_CREATE, NULL);
+ lucy_OutStream *target = lucy_OutStream_open((lucy_Obj*)file_handle);
+
+ Lucy_Obj_Serialize(self, target);
+
+ Lucy_OutStream_Close(target);
+ serialized_bb
+ = Lucy_RAMFile_Get_Contents(Lucy_RAMFH_Get_File(file_handle));
+ retval = XSBind_bb_to_sv(serialized_bb);
+ LUCY_DECREF(file_handle);
+ LUCY_DECREF(target);
+
+ if (SvCUR(retval) == 0) { // Thwart Storable bug
+ THROW(LUCY_ERR, "Calling serialize produced an empty string");
+ }
+ ST(0) = sv_2mortal(retval);
+ XSRETURN(1);
+ }
+}
+
+=begin comment
+
+Calls deserialize(), and copies the object pointer. Since deserialize is an
+abstract method, it will confess() unless implemented.
+
+=end comment
+=cut
+
+void
+STORABLE_thaw(blank_obj, cloning, serialized_sv)
+ SV *blank_obj;
+ SV *cloning;
+ SV *serialized_sv;
+PPCODE:
+{
+ char *class_name = HvNAME(SvSTASH(SvRV(blank_obj)));
+ lucy_ZombieCharBuf *klass
+ = CFISH_ZCB_WRAP_STR(class_name, strlen(class_name));
+ lucy_VTable *vtable
+ = (lucy_VTable*)lucy_VTable_singleton((lucy_CharBuf*)klass, NULL);
+ STRLEN len;
+ char *ptr = SvPV(serialized_sv, len);
+ lucy_ViewByteBuf *contents = lucy_ViewBB_new(ptr, len);
+ lucy_RAMFile *ram_file = lucy_RAMFile_new((lucy_ByteBuf*)contents, true);
+ lucy_RAMFileHandle *file_handle
+ = lucy_RAMFH_open(NULL, LUCY_FH_READ_ONLY, ram_file);
+ lucy_InStream *instream = lucy_InStream_open((lucy_Obj*)file_handle);
+ lucy_Obj *self = Lucy_VTable_Foster_Obj(vtable, blank_obj);
+ lucy_Obj *deserialized = Lucy_Obj_Deserialize(self, instream);
+
+ CHY_UNUSED_VAR(cloning);
+ LUCY_DECREF(contents);
+ LUCY_DECREF(ram_file);
+ LUCY_DECREF(file_handle);
+ LUCY_DECREF(instream);
+
+ // Catch bad deserialize() override.
+ if (deserialized != self) {
+ THROW(LUCY_ERR, "Error when deserializing obj of class %o", klass);
+ }
+}
+
+void
+DESTROY(self)
+ lucy_Obj *self;
+PPCODE:
+ /*
+ {
+ char *perl_class = HvNAME(SvSTASH(SvRV(ST(0))));
+ warn("Destroying: 0x%x %s", (unsigned)self, perl_class);
+ }
+ */
+ Lucy_Obj_Destroy(self);
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ package MyObj;
+ use base qw( Lucy::Object::Obj );
+
+ # Inside-out member var.
+ my %foo;
+
+ sub new {
+ my ( $class, %args ) = @_;
+ my $foo = delete $args{foo};
+ my $self = $class->SUPER::new(%args);
+ $foo{$$self} = $foo;
+ return $self;
+ }
+
+ sub get_foo {
+ my $self = shift;
+ return $foo{$$self};
+ }
+
+ sub DESTROY {
+ my $self = shift;
+ delete $foo{$$self};
+ $self->SUPER::DESTROY;
+ }
+END_SYNOPSIS
+
+my $description = <<'END_DESCRIPTION';
+All objects in the Lucy:: hierarchy descend from
+Lucy::Object::Obj. All classes are implemented as blessed scalar
+references, with the scalar storing a pointer to a C struct.
+
+==head2 Subclassing
+
+The recommended way to subclass Lucy::Object::Obj and its descendants is
+to use the inside-out design pattern. (See L<Class::InsideOut> for an
+introduction to inside-out techniques.)
+
+Since the blessed scalar stores a C pointer value which is unique per-object,
+C<$$self> can be used as an inside-out ID.
+
+ # Accessor for 'foo' member variable.
+ sub get_foo {
+ my $self = shift;
+ return $foo{$$self};
+ }
+
+
+Caveats:
+
+==over
+
+==item *
+
+Inside-out aficionados will have noted that the "cached scalar id" stratagem
+recommended above isn't compatible with ithreads -- but Lucy doesn't
+support ithreads anyway, so it doesn't matter.
+
+==item *
+
+Overridden methods must not return undef unless the API specifies that
+returning undef is permissible. (Failure to adhere to this rule currently
+results in a segfault rather than an exception.)
+
+==back
+
+==head1 CONSTRUCTOR
+
+==head2 new()
+
+Abstract constructor -- must be invoked via a subclass. Attempting to
+instantiate objects of class "Lucy::Object::Obj" directly causes an
+error.
+
+Takes no arguments; if any are supplied, an error will be reported.
+
+==head1 DESTRUCTOR
+
+==head2 DESTROY
+
+All Lucy classes implement a DESTROY method; if you override it in a
+subclass, you must call C<< $self->SUPER::DESTROY >> to avoid leaking memory.
+END_DESCRIPTION
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::Obj",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Get_RefCount
+ Inc_RefCount
+ Dec_RefCount
+ Get_VTable
+ To_String
+ To_I64
+ To_F64
+ Dump
+ _load|Load
+ Clone
+ Mimic
+ Equals
+ Hash_Sum
+ Serialize
+ Deserialize
+ Destroy
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ description => $description,
+ methods => [
+ qw(
+ to_string
+ to_i64
+ to_f64
+ equals
+ dump
+ load
+ )
+ ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Object/VArray.pm b/perl/lib/Lucy/Object/VArray.pm
new file mode 100644
index 0000000..22110cf
--- /dev/null
+++ b/perl/lib/Lucy/Object/VArray.pm
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::VArray;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::VArray
+
+SV*
+shallow_copy(self)
+ lucy_VArray *self;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(Lucy_VA_Shallow_Copy(self));
+OUTPUT: RETVAL
+
+SV*
+_deserialize(either_sv, instream)
+ SV *either_sv;
+ lucy_InStream *instream;
+CODE:
+ CHY_UNUSED_VAR(either_sv);
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_VA_deserialize(NULL, instream));
+OUTPUT: RETVAL
+
+SV*
+_clone(self)
+ lucy_VArray *self;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(Lucy_VA_Clone(self));
+OUTPUT: RETVAL
+
+SV*
+shift(self)
+ lucy_VArray *self;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(Lucy_VA_Shift(self));
+OUTPUT: RETVAL
+
+SV*
+pop(self)
+ lucy_VArray *self;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(Lucy_VA_Pop(self));
+OUTPUT: RETVAL
+
+SV*
+delete(self, tick)
+ lucy_VArray *self;
+ uint32_t tick;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(Lucy_VA_Delete(self, tick));
+OUTPUT: RETVAL
+
+void
+store(self, tick, value);
+ lucy_VArray *self;
+ uint32_t tick;
+ lucy_Obj *value;
+PPCODE:
+{
+ if (value) { LUCY_INCREF(value); }
+ lucy_VA_store(self, tick, value);
+}
+
+SV*
+fetch(self, tick)
+ lucy_VArray *self;
+ uint32_t tick;
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV(Lucy_VA_Fetch(self, tick));
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::VArray",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Push
+ Push_VArray
+ Unshift
+ Excise
+ Resize
+ Get_Size
+ )
+ ],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Object/VTable.pm b/perl/lib/Lucy/Object/VTable.pm
new file mode 100644
index 0000000..f230898
--- /dev/null
+++ b/perl/lib/Lucy/Object/VTable.pm
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Object::VTable;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Object::VTable
+
+SV*
+_get_registry()
+CODE:
+ if (lucy_VTable_registry == NULL) {
+ lucy_VTable_init_registry();
+ }
+ RETVAL = (SV*)Lucy_Obj_To_Host((lucy_Obj*)lucy_VTable_registry);
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Object::VTable",
+ xs_code => $xs_code,
+ bind_methods => [qw( Get_Name Get_Parent )],
+);
+
+
diff --git a/perl/lib/Lucy/Plan/Architecture.pm b/perl/lib/Lucy/Plan/Architecture.pm
new file mode 100644
index 0000000..d51ebc9
--- /dev/null
+++ b/perl/lib/Lucy/Plan/Architecture.pm
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::Architecture;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ package MyArchitecture;
+ use base qw( Lucy::Plan::Architecture );
+
+ use LucyX::Index::ZlibDocWriter;
+ use LucyX::Index::ZlibDocReader;
+
+ sub register_doc_writer {
+ my ( $self, $seg_writer ) = @_;
+ my $doc_writer = LucyX::Index::ZlibDocWriter->new(
+ snapshot => $seg_writer->get_snapshot,
+ segment => $seg_writer->get_segment,
+ polyreader => $seg_writer->get_polyreader,
+ );
+ $seg_writer->register(
+ api => "Lucy::Index::DocReader",
+ component => $doc_writer,
+ );
+ $seg_writer->add_writer($doc_writer);
+ }
+
+ sub register_doc_reader {
+ my ( $self, $seg_reader ) = @_;
+ my $doc_reader = LucyX::Index::ZlibDocReader->new(
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ segments => $seg_reader->get_segments,
+ seg_tick => $seg_reader->get_seg_tick,
+ snapshot => $seg_reader->get_snapshot,
+ );
+ $seg_reader->register(
+ api => 'Lucy::Index::DocReader',
+ component => $doc_reader,
+ );
+ }
+
+ package MySchema;
+ use base qw( Lucy::Plan::Schema );
+
+ sub architecture {
+ shift;
+ return MyArchitecture->new(@_);
+ }
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $arch = Lucy::Plan::Architecture->new;
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::Architecture",
+ bind_methods => [
+ qw(
+ Index_Interval
+ Skip_Interval
+ Init_Seg_Reader
+ Register_Doc_Writer
+ Register_Doc_Reader
+ Register_Deletions_Writer
+ Register_Deletions_Reader
+ Register_Lexicon_Reader
+ Register_Posting_List_Writer
+ Register_Posting_List_Reader
+ Register_Sort_Writer
+ Register_Sort_Reader
+ Register_Highlight_Writer
+ Register_Highlight_Reader
+ Make_Similarity
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [
+ qw(
+ register_doc_writer
+ register_doc_reader
+ )
+ ],
+ constructors => [ { sample => $constructor } ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Plan/BlobType.pm b/perl/lib/Lucy/Plan/BlobType.pm
new file mode 100644
index 0000000..6d7fefd
--- /dev/null
+++ b/perl/lib/Lucy/Plan/BlobType.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::BlobType;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $string_type = Lucy::Plan::StringType->new;
+ my $blob_type = Lucy::Plan::BlobType->new( stored => 1 );
+ my $schema = Lucy::Plan::Schema->new;
+ $schema->spec_field( name => 'id', type => $string_type );
+ $schema->spec_field( name => 'jpeg', type => $blob_type );
+END_SYNOPSIS
+my $constructor = <<'END_CONSTRUCTOR';
+ my $blob_type = Lucy::Plan::BlobType->new(
+ stored => 1, # default: false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::BlobType",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Plan/FieldType.pm b/perl/lib/Lucy/Plan/FieldType.pm
new file mode 100644
index 0000000..050118c
--- /dev/null
+++ b/perl/lib/Lucy/Plan/FieldType.pm
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::FieldType;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopis = <<'END_SYNOPSIS';
+
+ my @sortable;
+ for my $field ( @{ $schema->all_fields } ) {
+ my $type = $schema->fetch_type($field);
+ next unless $type->sortable;
+ push @sortable, $field;
+ }
+
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::FieldType",
+ bind_methods => [
+ qw(
+ Get_Boost
+ Indexed
+ Stored
+ Sortable
+ Binary
+ Compare_Values
+ )
+ ],
+ bind_constructors => ["new|init2"],
+ make_pod => {
+ synopsis => $synopis,
+ methods => [
+ qw(
+ get_boost
+ indexed
+ stored
+ sortable
+ binary
+ )
+ ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Plan/Float32Type.pm b/perl/lib/Lucy/Plan/Float32Type.pm
new file mode 100644
index 0000000..f5c9e90
--- /dev/null
+++ b/perl/lib/Lucy/Plan/Float32Type.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::Float32Type;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $schema = Lucy::Plan::Schema->new;
+ my $float32_type = Lucy::Plan::FloatType->new;
+ $schema->spec_field( name => 'intensity', type => $float32_type );
+END_SYNOPSIS
+my $constructor = <<'END_CONSTRUCTOR';
+ my $float32_type = Lucy::Plan::Float32Type->new(
+ indexed => 0, # default true
+ stored => 0, # default true
+ sortable => 1, # default false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::Float32Type",
+ bind_constructors => ["new|init2"],
+ #make_pod => {
+ # synopsis => $synopsis,
+ # constructor => { sample => $constructor },
+ #},
+);
+
+
diff --git a/perl/lib/Lucy/Plan/Float64Type.pm b/perl/lib/Lucy/Plan/Float64Type.pm
new file mode 100644
index 0000000..790f2ed
--- /dev/null
+++ b/perl/lib/Lucy/Plan/Float64Type.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::Float64Type;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $schema = Lucy::Plan::Schema->new;
+ my $float64_type = Lucy::Plan::FloatType->new;
+ $schema->spec_field( name => 'intensity', type => $float64_type );
+END_SYNOPSIS
+my $constructor = <<'END_CONSTRUCTOR';
+ my $float64_type = Lucy::Plan::Float64Type->new(
+ indexed => 0 # default true
+ stored => 0, # default true
+ sortable => 1, # default false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::Float64Type",
+ bind_constructors => ["new|init2"],
+ #make_pod => {
+ # synopsis => $synopsis,
+ # constructor => { sample => $constructor },
+ #},
+);
+
+
diff --git a/perl/lib/Lucy/Plan/FullTextType.pm b/perl/lib/Lucy/Plan/FullTextType.pm
new file mode 100644
index 0000000..61138dd
--- /dev/null
+++ b/perl/lib/Lucy/Plan/FullTextType.pm
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::FullTextType;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+ );
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ );
+ my $schema = Lucy::Plan::Schema->new;
+ $schema->spec_field( name => 'title', type => $type );
+ $schema->spec_field( name => 'content', type => $type );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $analyzer, # required
+ boost => 2.0, # default: 1.0
+ indexed => 1, # default: true
+ stored => 1, # default: true
+ sortable => 1, # default: false
+ highlightable => 1, # default: false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::FullTextType",
+ bind_constructors => ["new|init2"],
+ bind_methods => [
+ qw(
+ Set_Highlightable
+ Highlightable
+ )
+ ],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw(
+ set_highlightable
+ highlightable
+ )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Plan/Int32Type.pm b/perl/lib/Lucy/Plan/Int32Type.pm
new file mode 100644
index 0000000..901874a
--- /dev/null
+++ b/perl/lib/Lucy/Plan/Int32Type.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::Int32Type;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $schema = Lucy::Plan::Schema->new;
+ my $int32_type = Lucy::Plan::Int32Type->new;
+ $schema->spec_field( name => 'count', type => $int32_type );
+END_SYNOPSIS
+my $constructor = <<'END_CONSTRUCTOR';
+ my $int32_type = Lucy::Plan::Int32Type->new(
+ indexed => 0, # default true
+ stored => 0, # default true
+ sortable => 1, # default false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::Int32Type",
+ bind_constructors => ["new|init2"],
+ #make_pod => {
+ # synopsis => $synopsis,
+ # constructor => { sample => $constructor },
+ #},
+);
+
+
diff --git a/perl/lib/Lucy/Plan/Int64Type.pm b/perl/lib/Lucy/Plan/Int64Type.pm
new file mode 100644
index 0000000..50d6bdd
--- /dev/null
+++ b/perl/lib/Lucy/Plan/Int64Type.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::Int64Type;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $schema = Lucy::Plan::Schema->new;
+ my $int64_type = Lucy::Plan::Int64Type->new;
+ $schema->spec_field( name => 'count', type => $int64_type );
+END_SYNOPSIS
+my $constructor = <<'END_CONSTRUCTOR';
+ my $int64_type = Lucy::Plan::Int64Type->new(
+ indexed => 0, # default true
+ stored => 0, # default true
+ sortable => 1, # default false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::Int64Type",
+ bind_constructors => ["new|init2"],
+ #make_pod => {
+ # synopsis => $synopsis,
+ # constructor => { sample => $constructor },
+ #},
+);
+
+
diff --git a/perl/lib/Lucy/Plan/Schema.pm b/perl/lib/Lucy/Plan/Schema.pm
new file mode 100644
index 0000000..0973533
--- /dev/null
+++ b/perl/lib/Lucy/Plan/Schema.pm
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::Schema;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ use Lucy::Plan::Schema;
+ use Lucy::Plan::FullTextType;
+ use Lucy::Analysis::PolyAnalyzer;
+
+ my $schema = Lucy::Plan::Schema->new;
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+ );
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ );
+ $schema->spec_field( name => 'title', type => $type );
+ $schema->spec_field( name => 'content', type => $type );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $schema = Lucy::Plan::Schema->new;
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::Schema",
+ bind_methods => [
+ qw(
+ Architecture
+ Get_Architecture
+ Get_Similarity
+ Fetch_Type
+ Fetch_Analyzer
+ Fetch_Sim
+ Num_Fields
+ All_Fields
+ Spec_Field
+ Write
+ Eat
+ )
+ ],
+ bind_constructors => [qw( new )],
+ make_pod => {
+ methods => [
+ qw(
+ spec_field
+ num_fields
+ all_fields
+ fetch_type
+ fetch_sim
+ architecture
+ get_architecture
+ get_similarity
+ )
+ ],
+ synopsis => $synopsis,
+ constructors => [ { sample => $constructor } ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Plan/StringType.pm b/perl/lib/Lucy/Plan/StringType.pm
new file mode 100644
index 0000000..ff37151
--- /dev/null
+++ b/perl/lib/Lucy/Plan/StringType.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Plan::StringType;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $type = Lucy::Plan::StringType->new;
+ my $schema = Lucy::Plan::Schema->new;
+ $schema->spec_field( name => 'category', type => $type );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $type = Lucy::Plan::StringType->new(
+ boost => 0.1, # default: 1.0
+ indexed => 1, # default: true
+ stored => 1, # default: true
+ sortable => 1, # default: false
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Plan::StringType",
+ bind_constructors => ["new|init2"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/ANDMatcher.pm b/perl/lib/Lucy/Search/ANDMatcher.pm
new file mode 100644
index 0000000..6e76be4
--- /dev/null
+++ b/perl/lib/Lucy/Search/ANDMatcher.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::ANDMatcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::ANDMatcher",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/ANDQuery.pm b/perl/lib/Lucy/Search/ANDQuery.pm
new file mode 100644
index 0000000..b1f756e
--- /dev/null
+++ b/perl/lib/Lucy/Search/ANDQuery.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::ANDQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $foo_and_bar_query = Lucy::Search::ANDQuery->new(
+ children => [ $foo_query, $bar_query ],
+ );
+ my $hits = $searcher->hits( query => $foo_and_bar_query );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $foo_and_bar_query = Lucy::Search::ANDQuery->new(
+ children => [ $foo_query, $bar_query ],
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::ANDQuery",
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [qw( add_child )],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/BitVecMatcher.pm b/perl/lib/Lucy/Search/BitVecMatcher.pm
new file mode 100644
index 0000000..c9c50a4
--- /dev/null
+++ b/perl/lib/Lucy/Search/BitVecMatcher.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::BitVecMatcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::BitVecMatcher",
+ bind_constructors => [qw( new )],
+);
+
+
diff --git a/perl/lib/Lucy/Search/Collector.pm b/perl/lib/Lucy/Search/Collector.pm
new file mode 100644
index 0000000..496b4dc
--- /dev/null
+++ b/perl/lib/Lucy/Search/Collector.pm
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Collector;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $constructor = <<'END_CONSTRUCTOR';
+ package MyCollector;
+ use base qw( Lucy::Search::Collector );
+ our %foo;
+ sub new {
+ my $self = shift->SUPER::new;
+ my %args = @_;
+ $foo{$$self} = $args{foo};
+ return $self;
+ }
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Collector",
+ bind_methods => [
+ qw(
+ Collect
+ Set_Reader
+ Set_Base
+ Set_Matcher
+ Need_Score
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => " # Abstract base class.\n",
+ constructor => { sample => $constructor },
+ methods => [qw( collect )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Collector::OffsetCollector",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/Collector/BitCollector.pm b/perl/lib/Lucy/Search/Collector/BitCollector.pm
new file mode 100644
index 0000000..5ebe49e
--- /dev/null
+++ b/perl/lib/Lucy/Search/Collector/BitCollector.pm
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Collector::BitCollector;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $bit_vec = Lucy::Object::BitVector->new(
+ capacity => $searcher->doc_max + 1,
+ );
+ my $bit_collector = Lucy::Search::Collector::BitCollector->new(
+ bit_vector => $bit_vec,
+ );
+ $searcher->collect(
+ collector => $bit_collector,
+ query => $query,
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $bit_collector = Lucy::Search::Collector::BitCollector->new(
+ bit_vector => $bit_vec, # required
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Collector::BitCollector",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [qw( collect )],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/Collector/SortCollector.pm b/perl/lib/Lucy/Search/Collector/SortCollector.pm
new file mode 100644
index 0000000..bb4030c
--- /dev/null
+++ b/perl/lib/Lucy/Search/Collector/SortCollector.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Collector::SortCollector;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Collector::SortCollector",
+ bind_methods => [qw( Pop_Match_Docs Get_Total_Hits )],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/Compiler.pm b/perl/lib/Lucy/Search/Compiler.pm
new file mode 100644
index 0000000..a92aa08
--- /dev/null
+++ b/perl/lib/Lucy/Search/Compiler.pm
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Compiler;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # (Compiler is an abstract base class.)
+ package MyCompiler;
+ use base qw( Lucy::Search::Compiler );
+
+ sub make_matcher {
+ my $self = shift;
+ return MyMatcher->new( @_, compiler => $self );
+ }
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR_CODE_SAMPLE';
+ my $compiler = MyCompiler->SUPER::new(
+ parent => $my_query,
+ searcher => $searcher,
+ similarity => $sim, # default: undef
+ boost => undef, # default: see below
+ );
+END_CONSTRUCTOR_CODE_SAMPLE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Compiler",
+ bind_methods => [
+ qw(
+ Make_Matcher
+ Get_Parent
+ Get_Similarity
+ Get_Weight
+ Sum_Of_Squared_Weights
+ Apply_Norm_Factor
+ Normalize
+ Highlight_Spans
+ )
+ ],
+ bind_constructors => ["do_new"],
+ make_pod => {
+ methods => [
+ qw(
+ make_matcher
+ get_weight
+ sum_of_squared_weights
+ apply_norm_factor
+ normalize
+ get_parent
+ get_similarity
+ highlight_spans
+ )
+ ],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/HitQueue.pm b/perl/lib/Lucy/Search/HitQueue.pm
new file mode 100644
index 0000000..03058ac
--- /dev/null
+++ b/perl/lib/Lucy/Search/HitQueue.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::HitQueue;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::HitQueue",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/Hits.pm b/perl/lib/Lucy/Search/Hits.pm
new file mode 100644
index 0000000..7f7e2c9
--- /dev/null
+++ b/perl/lib/Lucy/Search/Hits.pm
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Hits;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $hits = $searcher->hits(
+ query => $query,
+ offset => 0,
+ num_wanted => 10,
+ );
+ while ( my $hit = $hits->next ) {
+ print "<p>$hit->{title} <em>" . $hit->get_score . "</em></p>\n";
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Hits",
+ bind_methods => [
+ qw(
+ Total_Hits
+ Next
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ methods => [qw( next total_hits )],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/IndexSearcher.pm b/perl/lib/Lucy/Search/IndexSearcher.pm
new file mode 100644
index 0000000..6cabbd4
--- /dev/null
+++ b/perl/lib/Lucy/Search/IndexSearcher.pm
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::IndexSearcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => '/path/to/index'
+ );
+ my $hits = $searcher->hits(
+ query => 'foo bar',
+ offset => 0,
+ num_wanted => 100,
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => '/path/to/index'
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::IndexSearcher",
+ bind_methods => [qw( Get_Reader )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw( hits
+ collect
+ doc_max
+ doc_freq
+ fetch_doc
+ get_schema
+ get_reader )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/LeafQuery.pm b/perl/lib/Lucy/Search/LeafQuery.pm
new file mode 100644
index 0000000..fb9797a
--- /dev/null
+++ b/perl/lib/Lucy/Search/LeafQuery.pm
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::LeafQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ package MyQueryParser;
+ use base qw( Lucy::Search::QueryParser );
+
+ sub expand_leaf {
+ my ( $self, $leaf_query ) = @_;
+ if ( $leaf_query->get_text =~ /.\*\s*$/ ) {
+ return PrefixQuery->new(
+ query_string => $leaf_query->get_text,
+ field => $leaf_query->get_field,
+ );
+ }
+ else {
+ return $self->SUPER::expand_leaf($leaf_query);
+ }
+ }
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $leaf_query = Lucy::Search::LeafQuery->new(
+ text => '"three blind mice"', # required
+ field => 'content', # default: undef
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::LeafQuery",
+ bind_methods => [qw( Get_Field Get_Text )],
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [qw( get_field get_text )],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/MatchAllQuery.pm b/perl/lib/Lucy/Search/MatchAllQuery.pm
new file mode 100644
index 0000000..38c60e3
--- /dev/null
+++ b/perl/lib/Lucy/Search/MatchAllQuery.pm
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::MatchAllQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $match_all_query = Lucy::Search::MatchAllQuery->new;
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::MatchAllQuery",
+ bind_constructors => ["new"],
+ make_pod => { constructor => { sample => $constructor }, }
+);
+
+
diff --git a/perl/lib/Lucy/Search/MatchDoc.pm b/perl/lib/Lucy/Search/MatchDoc.pm
new file mode 100644
index 0000000..42767a2
--- /dev/null
+++ b/perl/lib/Lucy/Search/MatchDoc.pm
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::MatchDoc;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::MatchDoc",
+ bind_methods => [
+ qw(
+ Get_Doc_ID
+ Set_Doc_ID
+ Get_Score
+ Set_Score
+ Get_Values
+ Set_Values
+ )
+ ],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/Matcher.pm b/perl/lib/Lucy/Search/Matcher.pm
new file mode 100644
index 0000000..5174a09
--- /dev/null
+++ b/perl/lib/Lucy/Search/Matcher.pm
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Matcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # abstract base class
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR_CODE_SAMPLE';
+ my $matcher = MyMatcher->SUPER::new;
+END_CONSTRUCTOR_CODE_SAMPLE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Matcher",
+ bind_methods => [qw( Next Advance Get_Doc_ID Score Collect )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [qw( next advance get_doc_id score )],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/NOTMatcher.pm b/perl/lib/Lucy/Search/NOTMatcher.pm
new file mode 100644
index 0000000..cbfebb8
--- /dev/null
+++ b/perl/lib/Lucy/Search/NOTMatcher.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::NOTMatcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::NOTMatcher",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/NOTQuery.pm b/perl/lib/Lucy/Search/NOTQuery.pm
new file mode 100644
index 0000000..485d729
--- /dev/null
+++ b/perl/lib/Lucy/Search/NOTQuery.pm
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::NOTQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $not_bar_query = Lucy::Search::NOTQuery->new(
+ negated_query => $bar_query,
+ );
+ my $foo_and_not_bar_query = Lucy::Search::ANDQuery->new(
+ children => [ $foo_query, $not_bar_query ].
+ );
+ my $hits = $searcher->hits( query => $foo_and_not_bar_query );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $not_query = Lucy::Search::NOTQuery->new(
+ negated_query => $query,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::NOTQuery",
+ bind_constructors => ["new"],
+ bind_methods => [qw( Get_Negated_Query Set_Negated_Query )],
+ make_pod => {
+ methods => [qw( get_negated_query set_negated_query )],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/NoMatchQuery.pm b/perl/lib/Lucy/Search/NoMatchQuery.pm
new file mode 100644
index 0000000..d90a97e
--- /dev/null
+++ b/perl/lib/Lucy/Search/NoMatchQuery.pm
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::NoMatchQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $no_match_query = Lucy::Search::NoMatchQuery->new;
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::NoMatchQuery",
+ bind_constructors => ["new"],
+ make_pod => { constructor => { sample => $constructor }, }
+);
+
+
diff --git a/perl/lib/Lucy/Search/ORQuery.pm b/perl/lib/Lucy/Search/ORQuery.pm
new file mode 100644
index 0000000..02edbe3
--- /dev/null
+++ b/perl/lib/Lucy/Search/ORQuery.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::ORQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $foo_or_bar_query = Lucy::Search::ORQuery->new(
+ children => [ $foo_query, $bar_query ],
+ );
+ my $hits = $searcher->hits( query => $foo_or_bar_query );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $foo_or_bar_query = Lucy::Search::ORQuery->new(
+ children => [ $foo_query, $bar_query ],
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::ORQuery",
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [qw( add_child )],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor, }
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/ORScorer.pm b/perl/lib/Lucy/Search/ORScorer.pm
new file mode 100644
index 0000000..9ba96bb
--- /dev/null
+++ b/perl/lib/Lucy/Search/ORScorer.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::ORScorer;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::ORScorer",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/PhraseQuery.pm b/perl/lib/Lucy/Search/PhraseQuery.pm
new file mode 100644
index 0000000..4013f9d
--- /dev/null
+++ b/perl/lib/Lucy/Search/PhraseQuery.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::PhraseQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $phrase_query = Lucy::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( the who )],
+ );
+ my $hits = $searcher->hits( query => $phrase_query );
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::PhraseQuery",
+ bind_methods => [qw( Get_Field Get_Terms )],
+ bind_constructors => ["new"],
+ make_pod => {
+ constructor => { sample => '' },
+ synopsis => $synopsis,
+ methods => [qw( get_field get_terms )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::PhraseCompiler",
+ bind_constructors => ["do_new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/PolyCompiler.pm b/perl/lib/Lucy/Search/PolyCompiler.pm
new file mode 100644
index 0000000..67aa3f5
--- /dev/null
+++ b/perl/lib/Lucy/Search/PolyCompiler.pm
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::PolyCompiler;
+use Lucy;
+
+1;
+
+__END__
+
+
diff --git a/perl/lib/Lucy/Search/PolyQuery.pm b/perl/lib/Lucy/Search/PolyQuery.pm
new file mode 100644
index 0000000..62fb9dd
--- /dev/null
+++ b/perl/lib/Lucy/Search/PolyQuery.pm
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::PolyQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ sub walk {
+ my $query = shift;
+ if ( $query->isa("Lucy::Search::PolyQuery") ) {
+ if ( $query->isa("Lucy::Search::ORQuery") ) { ... }
+ elsif ( $query->isa("Lucy::Search::ANDQuery") ) { ... }
+ elsif ( $query->isa("Lucy::Search::RequiredOptionalQuery") ) {
+ ...
+ }
+ elsif ( $query->isa("Lucy::Search::NOTQuery") ) { ... }
+ }
+ else { ... }
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::PolyQuery",
+ bind_methods => [qw( Add_Child Set_Children Get_Children )],
+ bind_constructors => ["new"],
+ make_pod => { synopsis => $synopsis, },
+);
+
+
diff --git a/perl/lib/Lucy/Search/PolySearcher.pm b/perl/lib/Lucy/Search/PolySearcher.pm
new file mode 100644
index 0000000..4d5d822
--- /dev/null
+++ b/perl/lib/Lucy/Search/PolySearcher.pm
@@ -0,0 +1,66 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::PolySearcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $schema = MySchema->new;
+ for my $server_name (@server_names) {
+ push @searchers, LucyX::Remote::SearchClient->new(
+ peer_address => "$server_name:$port",
+ password => $pass,
+ schema => $schema,
+ );
+ }
+ my $poly_searcher = Lucy::Search::PolySearcher->new(
+ schema => $schema,
+ searchers => \@searchers,
+ );
+ my $hits = $poly_searcher->hits( query => $query );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $poly_searcher = Lucy::Search::PolySearcher->new(
+ schema => $schema,
+ searchers => \@searchers,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::PolySearcher",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw( hits
+ doc_max
+ doc_freq
+ fetch_doc
+ get_schema
+ )
+ ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/Query.pm b/perl/lib/Lucy/Search/Query.pm
new file mode 100644
index 0000000..5a10ae7
--- /dev/null
+++ b/perl/lib/Lucy/Search/Query.pm
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Query;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # Query is an abstract base class.
+ package MyQuery;
+ use base qw( Lucy::Search::Query );
+
+ sub make_compiler {
+ my $self = shift;
+ return MyCompiler->new( @_, parent => $self );
+ }
+
+ package MyCompiler;
+ use base ( Lucy::Search::Compiler );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR_CODE_SAMPLE';
+ my $query = MyQuery->SUPER::new(
+ boost => 2.5,
+ );
+END_CONSTRUCTOR_CODE_SAMPLE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Query",
+ bind_methods => [
+ qw( Set_Boost
+ Get_Boost
+ _make_compiler|Make_Compiler )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [qw( make_compiler set_boost get_boost )],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/QueryParser.pm b/perl/lib/Lucy/Search/QueryParser.pm
new file mode 100644
index 0000000..d300bc8
--- /dev/null
+++ b/perl/lib/Lucy/Search/QueryParser.pm
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::QueryParser;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $query_parser = Lucy::Search::QueryParser->new(
+ schema => $searcher->get_schema,
+ fields => ['body'],
+ );
+ my $query = $query_parser->parse( $query_string );
+ my $hits = $searcher->hits( query => $query );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $query_parser = Lucy::Search::QueryParser->new(
+ schema => $searcher->get_schema, # required
+ analyzer => $analyzer, # overrides schema
+ fields => ['bodytext'], # default: indexed fields
+ default_boolop => 'AND', # default: 'OR'
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::QueryParser",
+ bind_methods => [
+ qw(
+ Parse
+ Tree
+ Expand
+ Expand_Leaf
+ Prune
+ Heed_Colons
+ Set_Heed_Colons
+ Get_Analyzer
+ Get_Schema
+ Get_Fields
+ Make_Term_Query
+ Make_Phrase_Query
+ Make_AND_Query
+ Make_OR_Query
+ Make_NOT_Query
+ Make_Req_Opt_Query
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [
+ qw( parse
+ tree
+ expand
+ expand_leaf
+ prune
+ set_heed_colons
+ make_term_query
+ make_phrase_query
+ make_and_query
+ make_or_query
+ make_not_query
+ make_req_opt_query
+ )
+ ],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/RangeQuery.pm b/perl/lib/Lucy/Search/RangeQuery.pm
new file mode 100644
index 0000000..23414ca
--- /dev/null
+++ b/perl/lib/Lucy/Search/RangeQuery.pm
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::RangeQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ # Match all articles by "Foo" published since the year 2000.
+ my $range_query = Lucy::Search::RangeQuery->new(
+ field => 'publication_date',
+ lower_term => '2000-01-01',
+ include_lower => 1,
+ );
+ my $author_query = Lucy::Search::TermQuery->new(
+ field => 'author_last_name',
+ text => 'Foo',
+ );
+ my $and_query = Lucy::Search::ANDQuery->new(
+ children => [ $range_query, $author_query ],
+ );
+ my $hits = $searcher->hits( query => $and_query );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $range_query = Lucy::Search::RangeQuery->new(
+ field => 'product_number', # required
+ lower_term => '003', # see below
+ upper_term => '060', # see below
+ include_lower => 0, # default true
+ include_upper => 0, # default true
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::RangeQuery",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/RequiredOptionalMatcher.pm b/perl/lib/Lucy/Search/RequiredOptionalMatcher.pm
new file mode 100644
index 0000000..475289c
--- /dev/null
+++ b/perl/lib/Lucy/Search/RequiredOptionalMatcher.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::RequiredOptionalMatcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::RequiredOptionalMatcher",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/RequiredOptionalQuery.pm b/perl/lib/Lucy/Search/RequiredOptionalQuery.pm
new file mode 100644
index 0000000..7bf5e51
--- /dev/null
+++ b/perl/lib/Lucy/Search/RequiredOptionalQuery.pm
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::RequiredOptionalQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $foo_and_maybe_bar = Lucy::Search::RequiredOptionalQuery->new(
+ required_query => $foo_query,
+ optional_query => $bar_query,
+ );
+ my $hits = $searcher->hits( query => $foo_and_maybe_bar );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $reqopt_query = Lucy::Search::RequiredOptionalQuery->new(
+ required_query => $foo_query, # required
+ optional_query => $bar_query, # required
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::RequiredOptionalQuery",
+ bind_methods => [
+ qw( Get_Required_Query Set_Required_Query
+ Get_Optional_Query Set_Optional_Query )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [
+ qw( get_required_query set_required_query
+ get_optional_query set_optional_query )
+ ],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/Searcher.pm b/perl/lib/Lucy/Search/Searcher.pm
new file mode 100644
index 0000000..d78494f
--- /dev/null
+++ b/perl/lib/Lucy/Search/Searcher.pm
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Searcher;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $constructor = <<'END_CONSTRUCTOR';
+ package MySearcher;
+ use base qw( Lucy::Search::Searcher );
+ sub new {
+ my $self = shift->SUPER::new;
+ ...
+ return $self;
+ }
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Searcher",
+ bind_methods => [
+ qw( Doc_Max
+ Doc_Freq
+ Glean_Query
+ Hits
+ Collect
+ Top_Docs
+ Fetch_Doc
+ Fetch_Doc_Vec
+ Get_Schema
+ Close )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => " # Abstract base class.\n",
+ constructor => { sample => $constructor },
+ methods => [
+ qw(
+ hits
+ collect
+ glean_query
+ doc_max
+ doc_freq
+ fetch_doc
+ get_schema
+ )
+ ],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/SortRule.pm b/perl/lib/Lucy/Search/SortRule.pm
new file mode 100644
index 0000000..2b98b78
--- /dev/null
+++ b/perl/lib/Lucy/Search/SortRule.pm
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::SortRule;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Search::SortRule
+
+int32_t
+FIELD()
+CODE:
+ RETVAL = lucy_SortRule_FIELD;
+OUTPUT: RETVAL
+
+int32_t
+SCORE()
+CODE:
+ RETVAL = lucy_SortRule_SCORE;
+OUTPUT: RETVAL
+
+int32_t
+DOC_ID()
+CODE:
+ RETVAL = lucy_SortRule_DOC_ID;
+OUTPUT: RETVAL
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $sort_spec = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'date' ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ],
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $by_title = Lucy::Search::SortRule->new( field => 'title' );
+ my $by_score = Lucy::Search::SortRule->new( type => 'score' );
+ my $by_doc_id = Lucy::Search::SortRule->new( type => 'doc_id' );
+ my $reverse_date = Lucy::Search::SortRule->new(
+ field => 'date',
+ reverse => 1,
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::SortRule",
+ xs_code => $xs_code,
+ bind_constructors => ["_new"],
+ bind_methods => [qw( Get_Field Get_Reverse )],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [qw( get_field get_reverse )],
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/SortSpec.pm b/perl/lib/Lucy/Search/SortSpec.pm
new file mode 100644
index 0000000..b42507f
--- /dev/null
+++ b/perl/lib/Lucy/Search/SortSpec.pm
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::SortSpec;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $sort_spec = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'date' ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ],
+ );
+ my $hits = $searcher->hits(
+ query => $query,
+ sort_spec => $sort_spec,
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $sort_spec = Lucy::Search::SortSpec->new( rules => \@rules );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::SortSpec",
+ bind_methods => [qw( Get_Rules )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Search/Span.pm b/perl/lib/Lucy/Search/Span.pm
new file mode 100644
index 0000000..05db3fb
--- /dev/null
+++ b/perl/lib/Lucy/Search/Span.pm
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::Span;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $combined_length = $upper_span->get_length
+ + ( $upper_span->get_offset - $lower_span->get_offset );
+ my $combined_span = Lucy::Search::Span->new(
+ offset => $lower_span->get_offset,
+ length => $combined_length,
+ );
+ ...
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $span = Lucy::Search::Span->new(
+ offset => 75, # required
+ length => 7, # required
+ weight => 1.0, # default 0.0
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::Span",
+ bind_methods => [
+ qw( Set_Offset
+ Get_Offset
+ Set_Length
+ Get_Length
+ Set_Weight
+ Get_Weight )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw( set_offset
+ get_offset
+ set_length
+ get_length
+ set_weight
+ get_weight )
+ ],
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Search/TermQuery.pm b/perl/lib/Lucy/Search/TermQuery.pm
new file mode 100644
index 0000000..bb6e039
--- /dev/null
+++ b/perl/lib/Lucy/Search/TermQuery.pm
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::TermQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $term_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'foo',
+ );
+ my $hits = $searcher->hits( query => $term_query );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $term_query = Lucy::Search::TermQuery->new(
+ field => 'content', # required
+ term => 'foo', # required
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::TermQuery",
+ bind_methods => [qw( Get_Field Get_Term )],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [qw( get_field get_term )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::TermCompiler",
+ bind_constructors => ["do_new"],
+);
+
+
diff --git a/perl/lib/Lucy/Search/TopDocs.pm b/perl/lib/Lucy/Search/TopDocs.pm
new file mode 100644
index 0000000..bd56273
--- /dev/null
+++ b/perl/lib/Lucy/Search/TopDocs.pm
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Search::TopDocs;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Search::TopDocs",
+ bind_methods => [
+ qw(
+ Get_Match_Docs
+ Get_Total_Hits
+ Set_Total_Hits
+ )
+ ],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Simple.pm b/perl/lib/Lucy/Simple.pm
new file mode 100644
index 0000000..c52b928
--- /dev/null
+++ b/perl/lib/Lucy/Simple.pm
@@ -0,0 +1,270 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package Lucy::Simple;
+use Carp;
+use Scalar::Util qw( weaken reftype refaddr );
+
+use Lucy::Plan::Schema;
+use Lucy::Analysis::PolyAnalyzer;
+use Lucy::Index::Indexer;
+use Lucy::Search::IndexSearcher;
+
+my %obj_cache;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $path = delete $args{path};
+ my $language = lc( delete $args{language} );
+ confess("Missing required parameter 'path'") unless defined $path;
+ confess("Invalid language: '$language'")
+ unless $language =~ /^(?:da|de|en|es|fi|fr|it|nl|no|pt|ru|sv)$/;
+ my @remaining = keys %args;
+ confess("Invalid params: @remaining") if @remaining;
+ my $self = bless {
+ type => undef,
+ schema => undef,
+ indexer => undef,
+ searcher => undef,
+ hits => undef,
+ language => $language,
+ path => $path,
+ },
+ ref($either) || $either;
+
+ # Get type and schema.
+ my $analyzer = Lucy::Analysis::PolyAnalyzer->new( language => $language );
+ $self->{type} = Lucy::Plan::FullTextType->new( analyzer => $analyzer, );
+ my $schema = $self->{schema} = Lucy::Plan::Schema->new;
+
+ # Cache the object for later clean-up.
+ weaken( $obj_cache{ refaddr $self } = $self );
+
+ return $self;
+}
+
+sub _lazily_create_indexer {
+ my $self = shift;
+ if ( !defined $self->{indexer} ) {
+ $self->{indexer} = Lucy::Index::Indexer->new(
+ schema => $self->{schema},
+ index => $self->{path},
+ );
+ }
+}
+
+sub add_doc {
+ my ( $self, $hashref ) = @_;
+ my $schema = $self->{schema};
+ my $type = $self->{type};
+ croak("add_doc requires exactly one argument: a hashref")
+ unless ( @_ == 2 and reftype($hashref) eq 'HASH' );
+ $self->_lazily_create_indexer;
+ $schema->spec_field( name => $_, type => $type ) for keys %$hashref;
+ $self->{indexer}->add_doc($hashref);
+}
+
+sub _finish_indexing {
+ my $self = shift;
+
+ # Don't bother to throw an error if index not modified.
+ if ( defined $self->{indexer} ) {
+ $self->{indexer}->commit;
+
+ # Trigger searcher and indexer refresh.
+ undef $self->{indexer};
+ undef $self->{searcher};
+ }
+}
+
+sub search {
+ my ( $self, %args ) = @_;
+
+ # Flush recent adds; lazily create searcher.
+ $self->_finish_indexing;
+ if ( !defined $self->{searcher} ) {
+ $self->{searcher}
+ = Lucy::Search::IndexSearcher->new( index => $self->{path} );
+ }
+
+ $self->{hits} = $self->{searcher}->hits(%args);
+
+ return $self->{hits}->total_hits;
+}
+
+sub next {
+ my $self = shift;
+ return unless defined $self->{hits};
+
+ # Get the hit, bail if hits are exhausted.
+ my $hit = $self->{hits}->next;
+ if ( !defined $hit ) {
+ undef $self->{hits};
+ return;
+ }
+
+ return $hit;
+}
+
+sub DESTROY {
+ for (shift) {
+ $_->_finish_indexing;
+ delete $obj_cache{ refaddr $_ };
+ }
+}
+
+END {
+ # Finish indexing for any objects that still exist, since, if we wait
+ # until global destruction, our Indexer might no longer exist,
+ # (see bug #32689)
+ $_->_finish_indexing for values %obj_cache;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+Lucy::Simple - Basic search engine.
+
+=head1 SYNOPSIS
+
+First, build an index of your documents.
+
+ my $index = Lucy::Simple->new(
+ path => '/path/to/index/'
+ language => 'en',
+ );
+
+ while ( my ( $title, $content ) = each %source_docs ) {
+ $index->add_doc({
+ title => $title,
+ content => $content,
+ });
+ }
+
+Later, search the index.
+
+ my $total_hits = $index->search(
+ query => $query_string,
+ offset => 0,
+ num_wanted => 10,
+ );
+
+ print "Total hits: $total_hits\n";
+ while ( my $hit = $index->next ) {
+ print "$hit->{title}\n",
+ }
+
+=head1 DESCRIPTION
+
+Lucy::Simple is a stripped-down interface for the L<Apache Lucy|Lucy> search
+engine library.
+
+=head1 METHODS
+
+=head2 new
+
+ my $lucy = Lucy::Simple->new(
+ path => '/path/to/index/',
+ language => 'en',
+ );
+
+Create a Lucy::Simple object, which can be used for both indexing and
+searching. Two hash-style parameters are required.
+
+=over
+
+=item *
+
+B<path> - Where the index directory should be located. If no index is found
+at the specified location, one will be created.
+
+=item *
+
+B<language> - The language of the documents in your collection, indicated
+by a two-letter ISO code. 12 languages are supported:
+
+ |-----------------------|
+ | Language | ISO code |
+ |-----------------------|
+ | Danish | da |
+ | Dutch | nl |
+ | English | en |
+ | Finnish | fi |
+ | French | fr |
+ | German | de |
+ | Italian | it |
+ | Norwegian | no |
+ | Portuguese | pt |
+ | Spanish | es |
+ | Swedish | sv |
+ | Russian | ru |
+ |-----------------------|
+
+=back
+
+=head2 add_doc
+
+ $lucy->add_doc({
+ location => $url,
+ title => $title,
+ content => $content,
+ });
+
+Add a document to the index. The document must be supplied as a hashref, with
+field names as keys and content as values.
+
+=head2 search
+
+ my $total_hits = $lucy->search(
+ query => $query_string, # required
+ offset => 40, # default 0
+ num_wanted => 20, # default 10
+ );
+
+Search the index. Returns the total number of documents which match the
+query. (This number is unlikely to match C<num_wanted>.)
+
+=over
+
+=item *
+
+B<query> - A search query string.
+
+=item *
+
+B<offset> - The number of most-relevant hits to discard, typically used when
+"paging" through hits N at a time. Setting offset to 20 and num_wanted to 10
+retrieves hits 21-30, assuming that 30 hits can be found.
+
+=item *
+
+B<num_wanted> - The number of hits you would like to see after C<offset> is
+taken into account.
+
+=back
+
+=head1 BUGS
+
+Not thread-safe.
+
+=cut
diff --git a/perl/lib/Lucy/Store/FSFileHandle.pm b/perl/lib/Lucy/Store/FSFileHandle.pm
new file mode 100644
index 0000000..5841a29
--- /dev/null
+++ b/perl/lib/Lucy/Store/FSFileHandle.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::FSFileHandle;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::FSFileHandle",
+ bind_constructors => ['_open|do_open'],
+);
+
+
diff --git a/perl/lib/Lucy/Store/FSFolder.pm b/perl/lib/Lucy/Store/FSFolder.pm
new file mode 100644
index 0000000..b096a9c
--- /dev/null
+++ b/perl/lib/Lucy/Store/FSFolder.pm
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::FSFolder;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $folder = Lucy::Store::FSFolder->new(
+ path => '/path/to/folder',
+ );
+END_SYNOPSIS
+
+my $constructor = $synopsis;
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::FSFolder",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ },
+);
+
+
diff --git a/perl/lib/Lucy/Store/FileHandle.pm b/perl/lib/Lucy/Store/FileHandle.pm
new file mode 100644
index 0000000..8d51af2
--- /dev/null
+++ b/perl/lib/Lucy/Store/FileHandle.pm
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::FileHandle;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Store::FileHandle
+
+=for comment
+
+For testing purposes only. Track number of FileHandle objects in existence.
+
+=cut
+
+uint32_t
+FH_READ_ONLY()
+CODE:
+ RETVAL = LUCY_FH_READ_ONLY;
+OUTPUT: RETVAL
+
+uint32_t
+FH_WRITE_ONLY()
+CODE:
+ RETVAL = LUCY_FH_WRITE_ONLY;
+OUTPUT: RETVAL
+
+uint32_t
+FH_CREATE()
+CODE:
+ RETVAL = LUCY_FH_CREATE;
+OUTPUT: RETVAL
+
+uint32_t
+FH_EXCLUSIVE()
+CODE:
+ RETVAL = LUCY_FH_EXCLUSIVE;
+OUTPUT: RETVAL
+
+
+int32_t
+object_count()
+CODE:
+ RETVAL = lucy_FH_object_count;
+OUTPUT: RETVAL
+
+=for comment
+
+For testing purposes only. Used to help produce buffer alignment tests.
+
+=cut
+
+IV
+_BUF_SIZE()
+CODE:
+ RETVAL = LUCY_IO_STREAM_BUF_SIZE;
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::FileHandle",
+ xs_code => $xs_code,
+ bind_methods => [qw( Length Close )],
+ bind_constructors => ['_open|do_open'],
+);
+
+
diff --git a/perl/lib/Lucy/Store/Folder.pm b/perl/lib/Lucy/Store/Folder.pm
new file mode 100644
index 0000000..9cd2a41
--- /dev/null
+++ b/perl/lib/Lucy/Store/Folder.pm
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::Folder;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::Folder",
+ bind_methods => [
+ qw(
+ Open_Out
+ Open_In
+ MkDir
+ List_R
+ Exists
+ Rename
+ Hard_Link
+ Delete
+ Slurp_File
+ Close
+ Get_Path
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => { synopsis => " # Abstract base class.\n", },
+);
+
+
diff --git a/perl/lib/Lucy/Store/InStream.pm b/perl/lib/Lucy/Store/InStream.pm
new file mode 100644
index 0000000..039e789
--- /dev/null
+++ b/perl/lib/Lucy/Store/InStream.pm
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::InStream;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Store::InStream
+
+void
+read(self, buffer_sv, len, ...)
+ lucy_InStream *self;
+ SV *buffer_sv;
+ size_t len;
+PPCODE:
+{
+ UV offset = items == 4 ? SvUV(ST(3)) : 0;
+ char *ptr;
+ size_t total_len = offset + len;
+ SvUPGRADE(buffer_sv, SVt_PV);
+ if (!SvPOK(buffer_sv)) { SvCUR_set(buffer_sv, 0); }
+ ptr = SvGROW(buffer_sv, total_len + 1);
+ Lucy_InStream_Read_Bytes(self, ptr + offset, len);
+ SvPOK_on(buffer_sv);
+ if (SvCUR(buffer_sv) < total_len) {
+ SvCUR_set(buffer_sv, total_len);
+ *(SvEND(buffer_sv)) = '\0';
+ }
+}
+
+SV*
+read_string(self)
+ lucy_InStream *self;
+CODE:
+{
+ char *ptr;
+ size_t len = Lucy_InStream_Read_C32(self);
+ RETVAL = newSV(len + 1);
+ SvCUR_set(RETVAL, len);
+ SvPOK_on(RETVAL);
+ SvUTF8_on(RETVAL); // Trust source. Reconsider if API goes public.
+ *SvEND(RETVAL) = '\0';
+ ptr = SvPVX(RETVAL);
+ Lucy_InStream_Read_Bytes(self, ptr, len);
+}
+OUTPUT: RETVAL
+
+int
+read_raw_c64(self, buffer_sv)
+ lucy_InStream *self;
+ SV *buffer_sv;
+CODE:
+{
+ char *ptr;
+ SvUPGRADE(buffer_sv, SVt_PV);
+ ptr = SvGROW(buffer_sv, 10 + 1);
+ RETVAL = Lucy_InStream_Read_Raw_C64(self, ptr);
+ SvPOK_on(buffer_sv);
+ SvCUR_set(buffer_sv, RETVAL);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::InStream",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Seek
+ Tell
+ Length
+ Reopen
+ Close
+ Read_I8
+ Read_I32
+ Read_I64
+ Read_U8
+ Read_U32
+ Read_U64
+ Read_C32
+ Read_C64
+ Read_F32
+ Read_F64
+ )
+ ],
+ bind_constructors => ['open|do_open'],
+);
+
+
diff --git a/perl/lib/Lucy/Store/Lock.pm b/perl/lib/Lucy/Store/Lock.pm
new file mode 100644
index 0000000..69de12d
--- /dev/null
+++ b/perl/lib/Lucy/Store/Lock.pm
@@ -0,0 +1,86 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::Lock;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $lock = $lock_factory->make_lock(
+ name => 'write',
+ timeout => 5000,
+ );
+ $lock->obtain or die "can't get lock for " . $lock->get_name;
+ do_stuff();
+ $lock->release;
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $lock = Lucy::Store::Lock->new(
+ name => 'commit', # required
+ folder => $folder, # required
+ host => $hostname, # required
+ timeout => 5000, # default: 0
+ interval => 1000, # default: 100
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::Lock",
+ bind_methods => [
+ qw(
+ Obtain
+ Request
+ Is_Locked
+ Release
+ Clear_Stale
+ Get_Name
+ Get_Lock_Path
+ Get_Host
+ )
+ ],
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ methods => [
+ qw(
+ obtain
+ request
+ release
+ is_locked
+ clear_stale
+ )
+ ],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::LockFileLock",
+ bind_constructors => ["new"],
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::SharedLock",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Store/LockErr.pm b/perl/lib/Lucy/Store/LockErr.pm
new file mode 100644
index 0000000..6dd4a8e
--- /dev/null
+++ b/perl/lib/Lucy/Store/LockErr.pm
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::LockErr;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ while (1) {
+ my $bg_merger = eval {
+ Lucy::Index::BackgroundMerger->new( index => $index );
+ };
+ if ( blessed($@) and $@->isa("Lucy::Store::LockErr") ) {
+ warn "Retrying...\n";
+ }
+ elsif (!$bg_merger) {
+ # Re-throw.
+ die "Failed to open BackgroundMerger: $@";
+ }
+ ...
+ }
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::LockErr",
+ make_pod => { synopsis => $synopsis }
+);
+
+
diff --git a/perl/lib/Lucy/Store/LockFactory.pm b/perl/lib/Lucy/Store/LockFactory.pm
new file mode 100644
index 0000000..c399591
--- /dev/null
+++ b/perl/lib/Lucy/Store/LockFactory.pm
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::LockFactory;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ use Sys::Hostname qw( hostname );
+ my $hostname = hostname() or die "Can't get unique hostname";
+ my $folder = Lucy::Store::FSFolder->new(
+ path => '/path/to/index',
+ );
+ my $lock_factory = Lucy::Store::LockFactory->new(
+ folder => $folder,
+ host => $hostname,
+ );
+ my $write_lock = $lock_factory->make_lock(
+ name => 'write',
+ timeout => 5000,
+ interval => 100,
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $lock_factory = Lucy::Store::LockFactory->new(
+ folder => $folder, # required
+ host => $hostname, # required
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::LockFactory",
+ bind_methods => [qw( Make_Lock Make_Shared_Lock )],
+ bind_constructors => ["new"],
+ make_pod => {
+ methods => [qw( make_lock make_shared_lock)],
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Store/OutStream.pm b/perl/lib/Lucy/Store/OutStream.pm
new file mode 100644
index 0000000..c0e5fc8
--- /dev/null
+++ b/perl/lib/Lucy/Store/OutStream.pm
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::OutStream;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Store::OutStream
+
+void
+print(self, ...)
+ lucy_OutStream *self;
+PPCODE:
+{
+ int i;
+ for (i = 1; i < items; i++) {
+ STRLEN len;
+ char *ptr = SvPV(ST(i), len);
+ Lucy_OutStream_Write_Bytes(self, ptr, len);
+ }
+}
+
+void
+write_string(self, aSV)
+ lucy_OutStream *self;
+ SV *aSV;
+PPCODE:
+{
+ STRLEN len = 0;
+ char *ptr = SvPVutf8(aSV, len);
+ Lucy_OutStream_Write_C32(self, len);
+ Lucy_OutStream_Write_Bytes(self, ptr, len);
+}
+END_XS_CODE
+
+my $synopsis = <<'END_SYNOPSIS'; # Don't use this yet.
+ my $outstream = $folder->open_out($filename) or die $@;
+ $outstream->write_u64($file_position);
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::OutStream",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Tell
+ Length
+ Flush
+ Close
+ Absorb
+ Write_I8
+ Write_I32
+ Write_I64
+ Write_U8
+ Write_U32
+ Write_U64
+ Write_C32
+ Write_C64
+ Write_F32
+ Write_F64
+ )
+ ],
+ bind_constructors => ['open|do_open'],
+);
+
+
diff --git a/perl/lib/Lucy/Store/RAMFile.pm b/perl/lib/Lucy/Store/RAMFile.pm
new file mode 100644
index 0000000..c409df5
--- /dev/null
+++ b/perl/lib/Lucy/Store/RAMFile.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::RAMFile;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::RAMFile",
+ bind_methods => [qw( Get_Contents )],
+ bind_constructors => ['new'],
+);
+
+
diff --git a/perl/lib/Lucy/Store/RAMFileHandle.pm b/perl/lib/Lucy/Store/RAMFileHandle.pm
new file mode 100644
index 0000000..cf72a64
--- /dev/null
+++ b/perl/lib/Lucy/Store/RAMFileHandle.pm
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::RAMFileHandle;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::RAMFileHandle",
+ bind_methods => [qw( Get_File )],
+ bind_constructors => ['_open|do_open'],
+);
+
+
diff --git a/perl/lib/Lucy/Store/RAMFolder.pm b/perl/lib/Lucy/Store/RAMFolder.pm
new file mode 100644
index 0000000..0a5987b
--- /dev/null
+++ b/perl/lib/Lucy/Store/RAMFolder.pm
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Store::RAMFolder;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $folder = Lucy::Store::RAMFolder->new;
+
+ # or sometimes...
+ my $folder = Lucy::Store::RAMFolder->new(
+ path => $relative_path,
+ );
+END_SYNOPSIS
+
+my $constructor = <<'END_CONSTRUCTOR';
+ my $folder = Lucy::Store::RAMFolder->new(
+ path => $relative_path, # default: empty string
+ );
+END_CONSTRUCTOR
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Store::RAMFolder",
+ bind_constructors => ["new"],
+ make_pod => {
+ synopsis => $synopsis,
+ constructor => { sample => $constructor },
+ }
+);
+
+
diff --git a/perl/lib/Lucy/Test.pm b/perl/lib/Lucy/Test.pm
new file mode 100644
index 0000000..8c033ec
--- /dev/null
+++ b/perl/lib/Lucy/Test.pm
@@ -0,0 +1,308 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Test;
+use Lucy;
+
+# Set the default memory threshold for PostingListWriter to a low number so
+# that we simulate large indexes by performing a lot of PostingPool flushes.
+Lucy::Index::PostingListWriter::set_default_mem_thresh(0x1000);
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Test::TestUtils
+
+SV*
+doc_set()
+CODE:
+ RETVAL = CFISH_OBJ_TO_SV_NOINC(lucy_TestUtils_doc_set());
+OUTPUT: RETVAL
+
+MODULE = Lucy PACKAGE = Lucy::Test
+
+void
+run_tests(package)
+ char *package;
+PPCODE:
+{
+ // Lucy::Analysis
+ if (strEQ(package, "TestAnalyzer")) {
+ lucy_TestAnalyzer_run_tests();
+ }
+ else if (strEQ(package, "TestCaseFolder")) {
+ lucy_TestCaseFolder_run_tests();
+ }
+ else if (strEQ(package, "TestPolyAnalyzer")) {
+ lucy_TestPolyAnalyzer_run_tests();
+ }
+ else if (strEQ(package, "TestSnowballStopFilter")) {
+ lucy_TestSnowStop_run_tests();
+ }
+ else if (strEQ(package, "TestSnowStemmer")) {
+ lucy_TestSnowStemmer_run_tests();
+ }
+ else if (strEQ(package, "TestRegexTokenizer")) {
+ lucy_TestRegexTokenizer_run_tests();
+ }
+ // Lucy::Object
+ else if (strEQ(package, "TestObj")) {
+ lucy_TestObj_run_tests();
+ }
+ else if (strEQ(package, "TestI32Array")) {
+ lucy_TestI32Arr_run_tests();
+ }
+ else if (strEQ(package, "TestByteBuf")) {
+ lucy_TestBB_run_tests();
+ }
+ else if (strEQ(package, "TestLockFreeRegistry")) {
+ lucy_TestLFReg_run_tests();
+ }
+ // Lucy::Plan
+ else if (strEQ(package, "TestBlobType")) {
+ lucy_TestBlobType_run_tests();
+ }
+ else if (strEQ(package, "TestFieldType")) {
+ lucy_TestFType_run_tests();
+ }
+ else if (strEQ(package, "TestFullTextType")) {
+ lucy_TestFullTextType_run_tests();
+ }
+ else if (strEQ(package, "TestNumericType")) {
+ lucy_TestNumericType_run_tests();
+ }
+ else if (strEQ(package, "TestSchema")) {
+ lucy_TestSchema_run_tests();
+ }
+ // Lucy::Index
+ else if (strEQ(package, "TestDocWriter")) {
+ lucy_TestDocWriter_run_tests();
+ }
+ else if (strEQ(package, "TestHighlightWriter")) {
+ lucy_TestHLWriter_run_tests();
+ }
+ else if (strEQ(package, "TestIndexManager")) {
+ lucy_TestIxManager_run_tests();
+ }
+ else if (strEQ(package, "TestPolyReader")) {
+ lucy_TestPolyReader_run_tests();
+ }
+ else if (strEQ(package, "TestPostingListWriter")) {
+ lucy_TestPListWriter_run_tests();
+ }
+ else if (strEQ(package, "TestSegment")) {
+ lucy_TestSeg_run_tests();
+ }
+ else if (strEQ(package, "TestSegWriter")) {
+ lucy_TestSegWriter_run_tests();
+ }
+ else if (strEQ(package, "TestSnapshot")) {
+ lucy_TestSnapshot_run_tests();
+ }
+ // Lucy::Search
+ else if (strEQ(package, "TestANDQuery")) {
+ lucy_TestANDQuery_run_tests();
+ }
+ else if (strEQ(package, "TestLeafQuery")) {
+ lucy_TestLeafQuery_run_tests();
+ }
+ else if (strEQ(package, "TestMatchAllQuery")) {
+ lucy_TestMatchAllQuery_run_tests();
+ }
+ else if (strEQ(package, "TestNoMatchQuery")) {
+ lucy_TestNoMatchQuery_run_tests();
+ }
+ else if (strEQ(package, "TestNOTQuery")) {
+ lucy_TestNOTQuery_run_tests();
+ }
+ else if (strEQ(package, "TestORQuery")) {
+ lucy_TestORQuery_run_tests();
+ }
+ else if (strEQ(package, "TestPhraseQuery")) {
+ lucy_TestPhraseQuery_run_tests();
+ }
+ else if (strEQ(package, "TestQueryParserLogic")) {
+ lucy_TestQPLogic_run_tests();
+ }
+ else if (strEQ(package, "TestSeriesMatcher")) {
+ lucy_TestSeriesMatcher_run_tests();
+ }
+ else if (strEQ(package, "TestRangeQuery")) {
+ lucy_TestRangeQuery_run_tests();
+ }
+ else if (strEQ(package, "TestReqOptQuery")) {
+ lucy_TestReqOptQuery_run_tests();
+ }
+ else if (strEQ(package, "TestTermQuery")) {
+ lucy_TestTermQuery_run_tests();
+ }
+ // Lucy::Store
+ else if (strEQ(package, "TestCompoundFileReader")) {
+ lucy_TestCFReader_run_tests();
+ }
+ else if (strEQ(package, "TestCompoundFileWriter")) {
+ lucy_TestCFWriter_run_tests();
+ }
+ else if (strEQ(package, "TestFileHandle")) {
+ lucy_TestFH_run_tests();
+ }
+ else if (strEQ(package, "TestFolder")) {
+ lucy_TestFolder_run_tests();
+ }
+ else if (strEQ(package, "TestFSDirHandle")) {
+ lucy_TestFSDH_run_tests();
+ }
+ else if (strEQ(package, "TestFSFolder")) {
+ lucy_TestFSFolder_run_tests();
+ }
+ else if (strEQ(package, "TestFSFileHandle")) {
+ lucy_TestFSFH_run_tests();
+ }
+ else if (strEQ(package, "TestInStream")) {
+ lucy_TestInStream_run_tests();
+ }
+ else if (strEQ(package, "TestIOChunks")) {
+ lucy_TestIOChunks_run_tests();
+ }
+ else if (strEQ(package, "TestIOPrimitives")) {
+ lucy_TestIOPrimitives_run_tests();
+ }
+ else if (strEQ(package, "TestRAMDirHandle")) {
+ lucy_TestRAMDH_run_tests();
+ }
+ else if (strEQ(package, "TestRAMFileHandle")) {
+ lucy_TestRAMFH_run_tests();
+ }
+ else if (strEQ(package, "TestRAMFolder")) {
+ lucy_TestRAMFolder_run_tests();
+ }
+ // Lucy::Util
+ else if (strEQ(package, "TestAtomic")) {
+ lucy_TestAtomic_run_tests();
+ }
+ else if (strEQ(package, "TestBitVector")) {
+ lucy_TestBitVector_run_tests();
+ }
+ else if (strEQ(package, "TestCharBuf")) {
+ lucy_TestCB_run_tests();
+ }
+ else if (strEQ(package, "TestHash")) {
+ lucy_TestHash_run_tests();
+ }
+ else if (strEQ(package, "TestJson")) {
+ lucy_TestJson_run_tests();
+ }
+ else if (strEQ(package, "TestMemory")) {
+ lucy_TestMemory_run_tests();
+ }
+ else if (strEQ(package, "TestIndexFileNames")) {
+ lucy_TestIxFileNames_run_tests();
+ }
+ else if (strEQ(package, "TestNumberUtils")) {
+ lucy_TestNumUtil_run_tests();
+ }
+ else if (strEQ(package, "TestNum")) {
+ lucy_TestNum_run_tests();
+ }
+ else if (strEQ(package, "TestPriorityQueue")) {
+ lucy_TestPriQ_run_tests();
+ }
+ else if (strEQ(package, "TestStringHelper")) {
+ lucy_TestStrHelp_run_tests();
+ }
+ else if (strEQ(package, "TestMemoryPool")) {
+ lucy_TestMemPool_run_tests();
+ }
+ else if (strEQ(package, "TestVArray")) {
+ lucy_TestVArray_run_tests();
+ }
+ else {
+ THROW(LUCY_ERR, "Unknown test id: %s", package);
+ }
+}
+
+MODULE = Lucy PACKAGE = Lucy::Test::Search::TestQueryParserSyntax
+
+void
+run_tests(index);
+ lucy_Folder *index;
+PPCODE:
+ lucy_TestQPSyntax_run_tests(index);
+END_XS_CODE
+
+my $charm_xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Test::TestCharmonizer
+
+void
+run_tests(which)
+ char *which;
+PPCODE:
+{
+ chaz_TestBatch *batch = NULL;
+ chaz_Test_init();
+
+ if (strcmp(which, "dirmanip") == 0) {
+ batch = chaz_TestDirManip_prepare();
+ }
+ else if (strcmp(which, "integers") == 0) {
+ batch = chaz_TestIntegers_prepare();
+ }
+ else if (strcmp(which, "func_macro") == 0) {
+ batch = chaz_TestFuncMacro_prepare();
+ }
+ else if (strcmp(which, "headers") == 0) {
+ batch = chaz_TestHeaders_prepare();
+ }
+ else if (strcmp(which, "large_files") == 0) {
+ batch = chaz_TestLargeFiles_prepare();
+ }
+ else if (strcmp(which, "unused_vars") == 0) {
+ batch = chaz_TestUnusedVars_prepare();
+ }
+ else if (strcmp(which, "variadic_macros") == 0) {
+ batch = chaz_TestVariadicMacros_prepare();
+ }
+ else {
+ THROW(LUCY_ERR, "Unknown test identifier: '%s'", which);
+ }
+
+ batch->run_test(batch);
+ batch->destroy(batch);
+}
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Test::TestSchema",
+ bind_constructors => ["new"],
+);
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Test",
+ xs_code => $xs_code,
+);
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Test::TestCharmonizer",
+ xs_code => $charm_xs_code,
+);
+
+
diff --git a/perl/lib/Lucy/Test/Util/BBSortEx.pm b/perl/lib/Lucy/Test/Util/BBSortEx.pm
new file mode 100644
index 0000000..3ffc7d6
--- /dev/null
+++ b/perl/lib/Lucy/Test/Util/BBSortEx.pm
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Test::Util::BBSortEx;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Test::Util::BBSortEx
+
+SV*
+fetch(self)
+ lucy_BBSortEx *self;
+CODE:
+{
+ void *address = Lucy_BBSortEx_Fetch(self);
+ if (address) {
+ RETVAL = XSBind_cfish_to_perl(*(lucy_Obj**)address);
+ LUCY_DECREF(*(lucy_Obj**)address);
+ }
+ else {
+ RETVAL = newSV(0);
+ }
+}
+OUTPUT: RETVAL
+
+SV*
+peek(self)
+ lucy_BBSortEx *self;
+CODE:
+{
+ void *address = Lucy_BBSortEx_Peek(self);
+ if (address) {
+ RETVAL = XSBind_cfish_to_perl(*(lucy_Obj**)address);
+ }
+ else {
+ RETVAL = newSV(0);
+ }
+}
+OUTPUT: RETVAL
+
+void
+feed(self, bb)
+ lucy_BBSortEx *self;
+ lucy_ByteBuf *bb;
+CODE:
+ LUCY_INCREF(bb);
+ Lucy_BBSortEx_Feed(self, &bb);
+
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Test::Util::BBSortEx",
+ bind_constructors => ["new"],
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/Lucy/Util/Debug.pm b/perl/lib/Lucy/Util/Debug.pm
new file mode 100644
index 0000000..3713a97
--- /dev/null
+++ b/perl/lib/Lucy/Util/Debug.pm
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::Debug;
+use strict;
+use warnings;
+
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Util::Debug
+
+#include "Lucy/Util/Debug.h"
+
+void
+DEBUG_PRINT(message)
+ char *message;
+PPCODE:
+ LUCY_DEBUG_PRINT("%s", message);
+
+void
+DEBUG(message)
+ char *message;
+PPCODE:
+ LUCY_DEBUG("%s", message);
+
+chy_bool_t
+DEBUG_ENABLED()
+CODE:
+ RETVAL = LUCY_DEBUG_ENABLED;
+OUTPUT: RETVAL
+
+=for comment
+
+Keep track of any Lucy objects that have been assigned to global Perl
+variables. This is useful when accounting how many objects should have been
+destroyed and diagnosing memory leaks.
+
+=cut
+
+void
+track_globals(...)
+PPCODE:
+{
+ CHY_UNUSED_VAR(items);
+ LUCY_IFDEF_DEBUG(lucy_Debug_num_globals++;);
+}
+
+void
+set_env_cache(str)
+ char *str;
+PPCODE:
+ lucy_Debug_set_env_cache(str);
+
+void
+ASSERT(maybe)
+ int maybe;
+PPCODE:
+ LUCY_ASSERT(maybe, "XS ASSERT binding test");
+
+IV
+num_allocated()
+CODE:
+ RETVAL = lucy_Debug_num_allocated;
+OUTPUT: RETVAL
+
+IV
+num_freed()
+CODE:
+ RETVAL = lucy_Debug_num_freed;
+OUTPUT: RETVAL
+
+IV
+num_globals()
+CODE:
+ RETVAL = lucy_Debug_num_globals;
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::Debug",
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/Lucy/Util/IndexFileNames.pm b/perl/lib/Lucy/Util/IndexFileNames.pm
new file mode 100644
index 0000000..de095c7
--- /dev/null
+++ b/perl/lib/Lucy/Util/IndexFileNames.pm
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::IndexFileNames;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Util::IndexFileNames
+
+uint64_t
+extract_gen(name)
+ const lucy_CharBuf *name;
+CODE:
+ RETVAL = lucy_IxFileNames_extract_gen(name);
+OUTPUT: RETVAL
+
+SV*
+latest_snapshot(folder)
+ lucy_Folder *folder;
+CODE:
+{
+ lucy_CharBuf *latest = lucy_IxFileNames_latest_snapshot(folder);
+ RETVAL = XSBind_cb_to_sv(latest);
+ LUCY_DECREF(latest);
+}
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::IndexFileNames",
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/Lucy/Util/Json.pm b/perl/lib/Lucy/Util/Json.pm
new file mode 100644
index 0000000..25bb3b8
--- /dev/null
+++ b/perl/lib/Lucy/Util/Json.pm
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::Json;
+use Lucy;
+
+1;
+
+__END__
+
+
diff --git a/perl/lib/Lucy/Util/MemoryPool.pm b/perl/lib/Lucy/Util/MemoryPool.pm
new file mode 100644
index 0000000..e86fc44
--- /dev/null
+++ b/perl/lib/Lucy/Util/MemoryPool.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::MemoryPool;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::MemoryPool",
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Util/PriorityQueue.pm b/perl/lib/Lucy/Util/PriorityQueue.pm
new file mode 100644
index 0000000..a35827d
--- /dev/null
+++ b/perl/lib/Lucy/Util/PriorityQueue.pm
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::PriorityQueue;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::PriorityQueue",
+ bind_methods => [
+ qw(
+ Less_Than
+ Insert
+ Pop
+ Pop_All
+ Peek
+ Get_Size
+ )
+ ],
+ bind_constructors => ["new"],
+);
+
+
diff --git a/perl/lib/Lucy/Util/SortExternal.pm b/perl/lib/Lucy/Util/SortExternal.pm
new file mode 100644
index 0000000..bde53d3
--- /dev/null
+++ b/perl/lib/Lucy/Util/SortExternal.pm
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::SortExternal;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Util::SortExternal
+
+IV
+_DEFAULT_MEM_THRESHOLD()
+CODE:
+ RETVAL = LUCY_SORTEX_DEFAULT_MEM_THRESHOLD;
+OUTPUT: RETVAL
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::SortExternal",
+ xs_code => $xs_code,
+ bind_methods => [
+ qw(
+ Flush
+ Flip
+ Add_Run
+ Refill
+ Sort_Cache
+ Cache_Count
+ Clear_Cache
+ Set_Mem_Thresh
+ )
+ ],
+);
+
+
diff --git a/perl/lib/Lucy/Util/Stepper.pm b/perl/lib/Lucy/Util/Stepper.pm
new file mode 100644
index 0000000..4afbcfb
--- /dev/null
+++ b/perl/lib/Lucy/Util/Stepper.pm
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::Stepper;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::Stepper",
+ bind_methods => [qw( Read_Record )],
+);
+
+
diff --git a/perl/lib/Lucy/Util/StringHelper.pm b/perl/lib/Lucy/Util/StringHelper.pm
new file mode 100644
index 0000000..568d82c
--- /dev/null
+++ b/perl/lib/Lucy/Util/StringHelper.pm
@@ -0,0 +1,122 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package Lucy::Util::StringHelper;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $xs_code = <<'END_XS_CODE';
+MODULE = Lucy PACKAGE = Lucy::Util::StringHelper
+
+=for comment
+
+Turn an SV's UTF8 flag on. Equivalent to Encode::_utf8_on, but we don't have
+to load Encode.
+
+=cut
+
+void
+utf8_flag_on(sv)
+ SV *sv;
+PPCODE:
+ SvUTF8_on(sv);
+
+=for comment
+
+Turn an SV's UTF8 flag off.
+
+=cut
+
+void
+utf8_flag_off(sv)
+ SV *sv;
+PPCODE:
+ SvUTF8_off(sv);
+
+SV*
+to_base36(num)
+ uint64_t num;
+CODE:
+{
+ char base36[lucy_StrHelp_MAX_BASE36_BYTES];
+ size_t size = lucy_StrHelp_to_base36(num, &base36);
+ RETVAL = newSVpvn(base36, size);
+}
+OUTPUT: RETVAL
+
+IV
+from_base36(str)
+ char *str;
+CODE:
+ RETVAL = strtol(str, NULL, 36);
+OUTPUT: RETVAL
+
+=for comment
+
+Upgrade a SV to UTF8, converting Latin1 if necessary. Equivalent to
+utf::upgrade().
+
+=cut
+
+void
+utf8ify(sv)
+ SV *sv;
+PPCODE:
+ sv_utf8_upgrade(sv);
+
+chy_bool_t
+utf8_valid(sv)
+ SV *sv;
+CODE:
+{
+ STRLEN len;
+ char *ptr = SvPV(sv, len);
+ RETVAL = lucy_StrHelp_utf8_valid(ptr, len);
+}
+OUTPUT: RETVAL
+
+=for comment
+
+Concatenate one scalar onto the end of the other, ignoring UTF-8 status of the
+second scalar. This is necessary because $not_utf8 . $utf8 results in a
+scalar which has been infected by the UTF-8 flag of the second argument.
+
+=cut
+
+void
+cat_bytes(sv, catted)
+ SV *sv;
+ SV *catted;
+PPCODE:
+{
+ STRLEN len;
+ char *ptr = SvPV(catted, len);
+ if (SvUTF8(sv)) { CFISH_THROW(LUCY_ERR, "Can't cat_bytes onto a UTF-8 SV"); }
+ sv_catpvn(sv, ptr, len);
+}
+END_XS_CODE
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "Lucy::Util::StringHelper",
+ xs_code => $xs_code,
+);
+
+
diff --git a/perl/lib/LucyX/Index/ByteBufDocReader.pm b/perl/lib/LucyX/Index/ByteBufDocReader.pm
new file mode 100644
index 0000000..e105b9e
--- /dev/null
+++ b/perl/lib/LucyX/Index/ByteBufDocReader.pm
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Index::ByteBufDocReader;
+use base qw( Lucy::Index::DocReader );
+use Lucy::Document::HitDoc;
+use Carp;
+
+# Inside-out member vars.
+our %width;
+our %field;
+our %instream;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $width = delete $args{width};
+ my $field = delete $args{field};
+ my $self = $either->SUPER::new(%args);
+ confess("Missing required param 'width'") unless defined $width;
+ confess("Missing required param 'field'") unless $field;
+ if ( $width < 1 ) { confess("'width' must be at least 1") }
+ $width{$$self} = $width;
+ $field{$$self} = $field;
+
+ my $segment = $self->get_segment;
+ my $metadata = $self->get_segment->fetch_metadata("bytebufdocs");
+ if ($metadata) {
+ if ( $metadata->{format} != 1 ) {
+ confess("Unrecognized format: '$metadata->{format}'");
+ }
+ my $filename = $segment->get_name . "/bytebufdocs.dat";
+ $instream{$$self} = $self->get_folder->open_in($filename)
+ or confess Lucy->error;
+ }
+
+ return $self;
+}
+
+sub fetch_doc {
+ my ( $self, $doc_id ) = @_;
+ my $field = $field{$$self};
+ my %fields = ( $field => '' );
+ $self->read_record( $doc_id, \$fields{$field} );
+ return Lucy::Document::HitDoc->new(
+ doc_id => $doc_id,
+ fields => \%fields,
+ );
+}
+
+sub read_record {
+ my ( $self, $doc_id, $buf ) = @_;
+ my $instream = $instream{$$self};
+ if ($instream) {
+ my $width = $width{$$self};
+ $instream->seek( $width * $doc_id );
+ $instream->read( $$buf, $width );
+ }
+}
+
+sub close {
+ my $self = shift;
+ delete $width{$$self};
+ delete $instream{$$self};
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $width{$$self};
+ delete $field{$$self};
+ delete $instream{$$self};
+ $self->SUPER::DESTROY;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+LucyX::Index::ByteBufDocReader - Read a Doc as a fixed-width byte array.
+
+=head1 SYNOPSIS
+
+ # See LucyX::Index::ByteBufDocWriter
+
+=head1 DESCRIPTION
+
+This is a proof-of-concept class to demonstrate alternate implementations for
+fetching documents. It is unsupported.
+
+=cut
+
diff --git a/perl/lib/LucyX/Index/ByteBufDocWriter.pm b/perl/lib/LucyX/Index/ByteBufDocWriter.pm
new file mode 100644
index 0000000..856b2a3
--- /dev/null
+++ b/perl/lib/LucyX/Index/ByteBufDocWriter.pm
@@ -0,0 +1,199 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Index::ByteBufDocWriter;
+use base qw( Lucy::Index::DataWriter );
+use Carp;
+use Scalar::Util qw( blessed );
+use bytes;
+no bytes;
+
+# Inside-out member vars.
+our %field;
+our %width;
+our %outstream;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $width = delete $args{width};
+ my $field = delete $args{field};
+ my $self = $either->SUPER::new(%args);
+ confess("Missing required param 'width'") unless defined $width;
+ confess("Missing required param 'field'") unless defined $field;
+ if ( $width < 1 ) { confess("'width' must be at least 1") }
+ $field{$$self} = $field;
+ $width{$$self} = $width;
+ return $self;
+}
+
+sub _lazy_init {
+ my $self = shift;
+
+ # Get outstream. Skip past non-doc #0.
+ my $folder = $self->get_folder;
+ my $filename = $self->get_segment->get_name . "/bytebufdocs.dat";
+ my $outstream = $outstream{$$self} = $folder->open_out($filename)
+ or confess Lucy->error;
+ my $nulls = "\0" x $width{$$self};
+ $outstream->print($nulls);
+
+ return $outstream;
+}
+
+sub add_inverted_doc {
+ my ( $self, %args ) = @_;
+ my $outstream = $outstream{$$self} || _lazy_init($self);
+ my $fields = $args{inverter}->get_doc->get_fields;
+ my $width = $width{$$self};
+ my $field = $field{$$self};
+ if ( bytes::length( $fields->{$field} ) != $width ) {
+ confess("Width of '$fields->{$field}' not $width");
+ }
+ $outstream->print( $fields->{$field} );
+}
+
+sub add_segment {
+ my ( $self, %args ) = @_;
+ my $seg_reader = $args{reader};
+ my $doc_map = $args{doc_map};
+ my $doc_max = $seg_reader->doc_max;
+
+ # Bail if the supplied segment is empty. */
+ return unless $doc_max;
+
+ my $outstream = $outstream{$$self} || _lazy_init($self);
+ my $doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+ confess("Not a ByteBufDocReader")
+ unless ( blessed($doc_reader)
+ and $doc_reader->isa("LucyX::Index::ByteBufDocReader") );
+
+ for ( my $i = 1; $i <= $doc_max; $i++ ) {
+ next unless $doc_map->get($i);
+ my $buf;
+ $doc_reader->read_record( $i, \$buf );
+ $outstream->print($buf);
+ }
+}
+
+sub finish {
+ my $self = shift;
+ my $outstream = $outstream{$$self};
+ if ($outstream) {
+ $outstream->close;
+ my $segment = $self->get_segment;
+ $segment->store_metadata(
+ key => 'bytebufdocs',
+ metadata => $self->metadata
+ );
+ }
+}
+
+sub format {1}
+
+sub DESTROY {
+ my $self = shift;
+ delete $field{$$self};
+ delete $width{$$self};
+ delete $outstream{$$self};
+ $self->SUPER::DESTROY;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+LucyX::Index::ByteBufDocWriter - Write a Doc as a fixed-width byte array.
+
+=head1 SYNOPSIS
+
+Create an L<Architecture|Lucy::Plan::Architecture> subclass which
+overrides register_doc_writer() and register_doc_reader():
+
+ package MyArchitecture;
+ use base qw( Lucy::Plan::Architecture );
+ use LucyX::Index::ByteBufDocReader;
+ use LucyX::Index::ByteBufDocWriter;
+
+ sub register_doc_writer {
+ my ( $self, $seg_writer ) = @_;
+ my $doc_writer = LucyX::Index::ByteBufDocWriter->new(
+ width => 16,
+ field => 'value',
+ snapshot => $seg_writer->get_snapshot,
+ segment => $seg_writer->get_segment,
+ polyreader => $seg_writer->get_polyreader,
+ );
+ $seg_writer->register(
+ api => "Lucy::Index::DocReader",
+ component => $doc_writer,
+ );
+ $seg_writer->add_writer($doc_writer);
+ }
+
+ sub register_doc_reader {
+ my ( $self, $seg_reader ) = @_;
+ my $doc_reader = LucyX::Index::ByteBufDocReader->new(
+ width => 16,
+ field => 'value',
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ segments => $seg_reader->get_segments,
+ seg_tick => $seg_reader->get_seg_tick,
+ snapshot => $seg_reader->get_snapshot,
+ );
+ $seg_reader->register(
+ api => 'Lucy::Index::DocReader',
+ component => $doc_reader,
+ );
+ }
+
+ package MySchema;
+ use base qw( Lucy::Plan::Schema );
+
+ sub architecture { MyArchitecture->new }
+
+Proceed as normal in your indexer app, making sure that every supplied
+document supplies a valid value for the field in question:
+
+ $indexer->add_doc({
+ title => $title,
+ content => $content,
+ id => $id, # <---- Must meet spec.
+ });
+
+Then, in your search app:
+
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => '/path/to/index',
+ );
+ my $hits = $searcher->hits( query => $query );
+ while ( my $id = $hits->next ) {
+ my $real_doc = $external_document_source->fetch( $doc->{value} );
+ ...
+ }
+
+=head1 DESCRIPTION
+
+This is a proof-of-concept class to demonstrate alternate implementations for
+fetching documents. It is unsupported.
+
+=cut
diff --git a/perl/lib/LucyX/Index/LongFieldSim.pm b/perl/lib/LucyX/Index/LongFieldSim.pm
new file mode 100644
index 0000000..cdf1019
--- /dev/null
+++ b/perl/lib/LucyX/Index/LongFieldSim.pm
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Index::LongFieldSim;
+use base qw( Lucy::Index::Similarity );
+
+sub length_norm {
+ my ( $self, $num_tokens ) = @_;
+ $num_tokens = $num_tokens < 100 ? 100 : $num_tokens;
+ return 1 / sqrt($num_tokens);
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+LucyX::Index::LongFieldSim - Similarity optimized for long fields.
+
+=head1 SYNOPSIS
+
+ package MySchema::body;
+ use base qw( Lucy::Plan::FullTextType );
+ use LucyX::Index::LongFieldSim;
+ sub make_similarity { LucyX::Index::LongFieldSim->new }
+
+=head1 DESCRIPTION
+
+Apache Lucy's default L<Similarity|Lucy::Index::Similarity>
+implmentation produces a bias towards extremely short fields.
+
+ Lucy::Index::Similarity
+
+ | more weight
+ | *
+ | **
+ | ***
+ | **********
+ | ********************
+ | *******************************
+ | less weight ****
+ |------------------------------------------------------------------------
+ fewer tokens more tokens
+
+LongFieldSim eliminates this bias.
+
+ LucyX::Index::LongFieldSim
+
+ | more weight
+ |
+ |
+ |
+ |*****************
+ | ********************
+ | *******************************
+ | less weight ****
+ |------------------------------------------------------------------------
+ fewer tokens more tokens
+
+In most cases, the default bias towards short fields is desirable. For
+instance, say you have two documents:
+
+=over
+
+=item *
+
+"George Washington"
+
+=item *
+
+"George Washington Carver"
+
+=back
+
+If a user searches for "george washington", we want the exact title match to
+appear first. Under the default Similarity implementation it will, because
+the "Carver" in "George Washington Carver" dilutes the impact of the other two
+tokens.
+
+However, under LongFieldSim, the two titles will yield equal scores. That
+would be bad in this particular case, but it could be good in another.
+
+ "George Washington Carver is cool."
+
+ "George Washington Carver was born on the eve of the US Civil War, in
+ 1864. His exact date of birth is unknown... Carver's research in crop
+ rotation revolutionized agriculture..."
+
+The first document is succinct, but useless. Unfortunately, the default
+similarity will assess it as extremely relevant to a query of "george
+washington carver". However, under LongFieldSim, the short-field bias is
+eliminated, and the addition of other mentions of Carver's name in the second
+document yield a higher score and a higher rank.
+
+=cut
+
+
diff --git a/perl/lib/LucyX/Index/ZlibDocReader.pm b/perl/lib/LucyX/Index/ZlibDocReader.pm
new file mode 100644
index 0000000..b98d497
--- /dev/null
+++ b/perl/lib/LucyX/Index/ZlibDocReader.pm
@@ -0,0 +1,146 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Index::ZlibDocReader;
+use base qw( Lucy::Index::DocReader );
+use Lucy::Util::StringHelper qw( utf8_valid utf8_flag_on );
+use Compress::Zlib qw( uncompress );
+use Carp;
+
+# Inside-out member vars.
+our %ix_in;
+our %dat_in;
+our %binary_fields;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $self = $either->SUPER::new(%args);
+
+ # Validate metadata. Only open streams if the segment has data we
+ # recognize.
+ my $segment = $self->get_segment;
+ my $metadata = $segment->fetch_metadata("zdocs");
+ if ($metadata) {
+ if ( $metadata->{format} != 1 ) {
+ confess("Unrecognized format: '$metadata->{format}'");
+ }
+
+ # Open InStreams.
+ my $dat_filename = $segment->get_name . "/zdocs.dat";
+ my $ix_filename = $segment->get_name . "/zdocs.ix";
+ my $folder = $self->get_folder;
+ $ix_in{$$self} = $folder->open_in($ix_filename)
+ or confess Lucy->error;
+ $dat_in{$$self} = $folder->open_in($dat_filename)
+ or confess Lucy->error;
+
+ # Remember which fields are binary.
+ my $schema = $self->get_schema;
+ my $bin_fields = $binary_fields{$$self} = {};
+ $bin_fields->{$_} = 1
+ for grep { $schema->fetch_type($_)->binary }
+ @{ $schema->all_fields };
+ }
+
+ return $self;
+}
+
+sub fetch_doc {
+ my ( $self, $doc_id ) = @_;
+ my $dat_in = $dat_in{$$self};
+ my $ix_in = $ix_in{$$self};
+ my $bin_fields = $binary_fields{$$self};
+
+ # Bail if no data in segment.
+ return unless $ix_in;
+
+ # Read index information.
+ $ix_in->seek( $doc_id * 8 );
+ my $start = $ix_in->read_i64;
+ my $len = $ix_in->read_i64 - $start;
+ my $compressed;
+
+ # Read main data.
+ $dat_in->seek($start);
+ $dat_in->read( $compressed, $len );
+ my $inflated = uncompress($compressed);
+ my $num_fields = unpack( "w", $inflated );
+ my $pack_template = "w ";
+ $pack_template .= "w/a*" x ( $num_fields * 2 );
+ my ( undef, %fields ) = unpack( $pack_template, $inflated );
+
+ # Turn on UTF-8 flag for string fields.
+ for my $field ( keys %fields ) {
+ next if $bin_fields->{$field};
+ utf8_flag_on( $fields{$field} );
+ confess("Invalid UTF-8 read for doc $doc_id field '$field'")
+ unless utf8_valid( $fields{$field} );
+ }
+
+ return Lucy::Document::HitDoc->new(
+ fields => \%fields,
+ doc_id => $doc_id,
+ );
+}
+
+sub read_record {
+ my ( $self, $doc_id, $buf ) = @_;
+ my $dat_in = $dat_in{$$self};
+ my $ix_in = $ix_in{$$self};
+ my $bin_fields = $binary_fields{$$self};
+
+ if ($ix_in) {
+ $ix_in->seek( $doc_id * 8 );
+ my $start = $ix_in->read_i64;
+ my $len = $ix_in->read_i64 - $start;
+ $dat_in->seek($start);
+ $dat_in->read( $$buf, $len );
+ }
+}
+
+sub close {
+ my $self = shift;
+ delete $ix_in{$$self};
+ delete $dat_in{$$self};
+ delete $binary_fields{$$self};
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $ix_in{$$self};
+ delete $dat_in{$$self};
+ delete $binary_fields{$$self};
+ $self->SUPER::DESTROY;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+LucyX::Index::ZlibDocReader - Compressed doc storage.
+
+=head1 DESCRIPTION
+
+This is a proof-of-concept class to demonstrate alternate implementations for
+fetching documents. It is unsupported.
+
+=cut
diff --git a/perl/lib/LucyX/Index/ZlibDocWriter.pm b/perl/lib/LucyX/Index/ZlibDocWriter.pm
new file mode 100644
index 0000000..ed7273c
--- /dev/null
+++ b/perl/lib/LucyX/Index/ZlibDocWriter.pm
@@ -0,0 +1,153 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+
+package LucyX::Index::ZlibDocWriter;
+use base qw( Lucy::Index::DataWriter );
+use Carp;
+use Scalar::Util qw( blessed );
+use Compress::Zlib qw( compress );
+use Lucy::Util::StringHelper qw( cat_bytes );
+use Lucy qw( to_perl );
+use bytes;
+no bytes;
+
+# Inside-out member vars.
+our %ix_out;
+our %dat_out;
+
+# Inherit constructor.
+
+sub _lazy_init {
+ my $self = shift;
+
+ # Get outstreams. Skip past non-doc #0.
+ my $folder = $self->get_folder;
+ my $ix_file = $self->get_segment->get_name . "/zdocs.ix";
+ my $dat_file = $self->get_segment->get_name . "/zdocs.dat";
+ $ix_out{$$self} = $folder->open_out($ix_file)
+ or confess Lucy->error;
+ $dat_out{$$self} = $folder->open_out($dat_file)
+ or confess Lucy->error;
+ $ix_out{$$self}->write_i64(0);
+}
+
+sub add_inverted_doc {
+ my ( $self, %args ) = @_;
+ _lazy_init($self) unless $ix_out{$$self};
+ my $inverter = $args{inverter};
+ my $ix_out = $ix_out{$$self};
+ my $dat_out = $dat_out{$$self};
+
+ # Check doc id.
+ my $expected = $ix_out->tell / 8;
+ confess("Expected doc id $expected, got '$args{doc_id}'")
+ unless $args{doc_id} == $expected;
+
+ my $to_compress = "";
+ my $count = 0;
+ my $schema = $self->get_schema;
+ $inverter->iterate;
+ while ( $inverter->next ) {
+ next unless $inverter->get_type->stored;
+ my $name = $inverter->get_field_name;
+ my $value = $inverter->get_value;
+ cat_bytes( $to_compress, pack( "w", bytes::length($name) ) );
+ cat_bytes( $to_compress, $name );
+ cat_bytes( $to_compress, pack( "w", bytes::length($value) ) );
+ cat_bytes( $to_compress, $value );
+ $count++;
+ }
+ # Prepend count of fields to store in this Doc.
+ $to_compress = pack( "w", $count ) . $to_compress;
+
+ # Write file pointer to index file. Write compressed serialized string to
+ # main file.
+ $ix_out->write_i64( $dat_out->tell );
+ $dat_out->print( compress($to_compress) );
+}
+
+sub add_segment {
+ my ( $self, %args ) = @_;
+ my $seg_reader = $args{reader};
+ my $doc_map = $args{doc_map};
+ my $doc_max = $seg_reader->doc_max;
+
+ # Bail if the supplied segment is empty. */
+ return unless $doc_max;
+
+ _lazy_init($self) unless $ix_out{$$self};
+ my $ix_out = $ix_out{$$self};
+ my $dat_out = $dat_out{$$self};
+ my $doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+ confess("Not a ZlibDocReader")
+ unless ( blessed($doc_reader)
+ and $doc_reader->isa("LucyX::Index::ZlibDocReader") );
+
+ for ( my $i = 1; $i <= $doc_max; $i++ ) {
+ next unless $doc_map->get($i);
+ my $buf;
+ $doc_reader->read_record( $i, \$buf );
+ $ix_out->write_i64( $dat_out->tell );
+ $dat_out->print($buf);
+ }
+}
+
+sub finish {
+ my $self = shift;
+ my $ix_out = $ix_out{$$self};
+ my $dat_out = $dat_out{$$self};
+ if ($ix_out) {
+ # Write one extra file pointer so that we can always derive record
+ # length.
+ $ix_out->write_i64( $dat_out->tell );
+
+ # Close streams and store metadata.
+ $ix_out->close;
+ $dat_out->close;
+ my $segment = $self->get_segment;
+ $segment->store_metadata(
+ key => 'zdocs',
+ metadata => $self->metadata,
+ );
+ }
+}
+
+sub format {1}
+
+sub DESTROY {
+ my $self = shift;
+ delete $ix_out{$$self};
+ delete $dat_out{$$self};
+ $self->SUPER::DESTROY;
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 NAME
+
+LucyX::Index::ZlibDocWriter - Compressed doc storage.
+
+=head1 DESCRIPTION
+
+This is a proof-of-concept class to demonstrate alternate implementations for
+fetching documents. It is unsupported.
+
+=cut
diff --git a/perl/lib/LucyX/Remote/SearchClient.pm b/perl/lib/LucyX/Remote/SearchClient.pm
new file mode 100644
index 0000000..0fb3174
--- /dev/null
+++ b/perl/lib/LucyX/Remote/SearchClient.pm
@@ -0,0 +1,183 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Remote::SearchClient;
+BEGIN { our @ISA = qw( Lucy::Search::Searcher ) }
+use Carp;
+use Storable qw( nfreeze thaw );
+use bytes;
+no bytes;
+
+# Inside-out member vars.
+our %peer_address;
+our %password;
+our %sock;
+
+use IO::Socket::INET;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $peer_address = delete $args{peer_address};
+ my $password = delete $args{password};
+ my $self = $either->SUPER::new(%args);
+ $peer_address{$$self} = $peer_address;
+ $password{$$self} = $password;
+
+ # Establish a connection.
+ my $sock = $sock{$$self} = IO::Socket::INET->new(
+ PeerAddr => $peer_address,
+ Proto => 'tcp',
+ );
+ confess("No socket: $!") unless $sock;
+ $sock->autoflush(1);
+
+ # Verify password.
+ print $sock "$password\n";
+ chomp( my $response = <$sock> );
+ confess("Failed to connect: '$response'") unless $response =~ /accept/i;
+
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $peer_address{$$self};
+ delete $password{$$self};
+ delete $sock{$$self};
+ $self->SUPER::DESTROY;
+}
+
+=for comment
+
+Make a remote procedure call. For every call that does not close/terminate
+the socket connection, expect a response back that's been serialized using
+Storable.
+
+=cut
+
+sub _rpc {
+ my ( $self, $method, $args ) = @_;
+ my $sock = $sock{$$self};
+
+ my $serialized = nfreeze($args);
+ my $packed_len = pack( 'N', bytes::length($serialized) );
+ print $sock "$method\n$packed_len$serialized";
+
+ # Bail out if we're either closing or shutting down the server remotely.
+ return if $method eq 'done';
+ return if $method eq 'terminate';
+
+ # Decode response.
+ $sock->read( $packed_len, 4 );
+ my $arg_len = unpack( 'N', $packed_len );
+ my $check_val = read( $sock, $serialized, $arg_len );
+ confess("Tried to read $arg_len bytes, got $check_val")
+ unless ( defined $arg_len and $check_val == $arg_len );
+ my $response = thaw($serialized);
+ if ( exists $response->{retval} ) {
+ return $response->{retval};
+ }
+ return;
+}
+
+sub top_docs {
+ my $self = shift;
+ return $self->_rpc( 'top_docs', {@_} );
+}
+
+sub terminate {
+ my $self = shift;
+ return $self->_rpc( 'terminate', {} );
+}
+
+sub fetch_doc {
+ my ( $self, $doc_id ) = @_;
+ return $self->_rpc( 'fetch_doc', { doc_id => $doc_id } );
+}
+
+sub fetch_doc_vec {
+ my ( $self, $doc_id ) = @_;
+ return $self->_rpc( 'fetch_doc_vec', { doc_id => $doc_id } );
+}
+
+sub doc_max {
+ my $self = shift;
+ return $self->_rpc( 'doc_max', {} );
+}
+
+sub doc_freq {
+ my $self = shift;
+ return $self->_rpc( 'doc_freq', {@_} );
+}
+
+sub close {
+ my $self = shift;
+ $self->_rpc( 'done', {} );
+ my $sock = $sock{$$self};
+ close $sock or confess("Error when closing socket: $!");
+ delete $sock{$$self};
+}
+
+sub NUKE {
+ my $self = shift;
+ $self->close if defined $sock{$$self};
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+LucyX::Remote::SearchClient - Connect to a remote SearchServer.
+
+=head1 SYNOPSIS
+
+ my $client = LucyX::Remote::SearchClient->new(
+ peer_address => 'searchserver1:7890',
+ password => $pass,
+ );
+ my $hits = $client->hits( query => $query );
+
+=head1 DESCRIPTION
+
+SearchClient is a subclass of L<Lucy::Search::Searcher> which can be
+used to search an index on a remote machine made accessible via
+L<SearchServer|LucyX::Remote::SearchServer>.
+
+=head1 METHODS
+
+=head2 new
+
+Constructor. Takes hash-style params.
+
+=over
+
+=item *
+
+B<peer_address> - The name/IP and the port number which the client should
+attempt to connect to.
+
+=item *
+
+B<password> - Password to be supplied to the SearchServer when initializing
+socket connection.
+
+=back
+
+=cut
diff --git a/perl/lib/LucyX/Remote/SearchServer.pm b/perl/lib/LucyX/Remote/SearchServer.pm
new file mode 100644
index 0000000..c9501ed
--- /dev/null
+++ b/perl/lib/LucyX/Remote/SearchServer.pm
@@ -0,0 +1,234 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Remote::SearchServer;
+BEGIN { our @ISA = qw( Lucy::Object::Obj ) }
+use Carp;
+use Storable qw( nfreeze thaw );
+use bytes;
+no bytes;
+
+# Inside-out member vars.
+our %searcher;
+our %port;
+our %password;
+our %sock;
+
+use IO::Socket::INET;
+use IO::Select;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $searcher = delete $args{searcher};
+ my $password = delete $args{password};
+ my $port = delete $args{port};
+ my $self = $either->SUPER::new(%args);
+ $searcher{$$self} = $searcher;
+ confess("Missing required param 'password'") unless defined $password;
+ $password{$$self} = $password;
+
+ # Establish a listening socket.
+ $port{$$self} = $port;
+ confess("Invalid port: $port") unless $port =~ /^\d+$/;
+ my $sock = IO::Socket::INET->new(
+ LocalPort => $port,
+ Proto => 'tcp',
+ Listen => SOMAXCONN,
+ Reuse => 1,
+ );
+ confess("No socket: $!") unless $sock;
+ $sock->autoflush(1);
+ $sock{$$self} = $sock;
+
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $searcher{$$self};
+ delete $port{$$self};
+ delete $password{$$self};
+ delete $sock{$$self};
+ $self->SUPER::DESTROY;
+}
+
+my %dispatch = (
+ doc_max => \&do_doc_max,
+ doc_freq => \&do_doc_freq,
+ top_docs => \&do_top_docs,
+ fetch_doc => \&do_fetch_doc,
+ fetch_doc_vec => \&do_fetch_doc_vec,
+ terminate => undef,
+);
+
+sub serve {
+ my $self = shift;
+ my $main_sock = $sock{$$self};
+ my $read_set = IO::Select->new($main_sock);
+
+ while ( my @ready = $read_set->can_read ) {
+ for my $readhandle (@ready) {
+ # If this is the main handle, we have a new client, so accept.
+ if ( $readhandle == $main_sock ) {
+ my $client_sock = $main_sock->accept;
+
+ # Verify password.
+ my $pass = <$client_sock>;
+ chomp($pass) if defined $pass;
+ if ( defined $pass && $pass eq $password{$$self} ) {
+ $read_set->add($client_sock);
+ print $client_sock "accepted\n";
+ }
+ else {
+ print $client_sock "password incorrect\n";
+ }
+ }
+ # Otherwise it's a client sock, so process the request.
+ else {
+ my $client_sock = $readhandle;
+ my ( $check_val, $buf, $len, $method, $args );
+ chomp( $method = <$client_sock> );
+
+ # If "done", the client's closing.
+ if ( $method eq 'done' ) {
+ $read_set->remove($client_sock);
+ $client_sock->close;
+ next;
+ }
+ # Remote signal to close the server.
+ elsif ( $method eq 'terminate' ) {
+ $read_set->remove($client_sock);
+ $client_sock->close;
+ $main_sock->close;
+ return;
+ }
+ # Sanity check the method name.
+ elsif ( !$dispatch{$method} ) {
+ print $client_sock "ERROR: Bad method name: $method\n";
+ next;
+ }
+
+ # Process the method call.
+ read( $client_sock, $buf, 4 );
+ $len = unpack( 'N', $buf );
+ read( $client_sock, $buf, $len );
+ my $response = $dispatch{$method}->( $self, thaw($buf) );
+ my $frozen = nfreeze($response);
+ my $packed_len = pack( 'N', bytes::length($frozen) );
+ print $client_sock $packed_len . $frozen;
+ }
+ }
+ }
+}
+
+sub do_doc_freq {
+ my ( $self, $args ) = @_;
+ return { retval => $searcher{$$self}->doc_freq(%$args) };
+}
+
+sub do_top_docs {
+ my ( $self, $args ) = @_;
+ my $top_docs = $searcher{$$self}->top_docs(%$args);
+ return { retval => $top_docs };
+}
+
+sub do_doc_max {
+ my ( $self, $args ) = @_;
+ my $doc_max = $searcher{$$self}->doc_max;
+ return { retval => $doc_max };
+}
+
+sub do_fetch_doc {
+ my ( $self, $args ) = @_;
+ my $doc = $searcher{$$self}->fetch_doc( $args->{doc_id} );
+ return { retval => $doc };
+}
+
+sub do_fetch_doc_vec {
+ my ( $self, $args ) = @_;
+ my $doc_vec = $searcher{$$self}->fetch_doc_vec( $args->{doc_id} );
+ return { retval => $doc_vec };
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+LucyX::Remote::SearchServer - Make a Searcher remotely accessible.
+
+=head1 SYNOPSIS
+
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => '/path/to/index'
+ );
+ my $search_server = LucyX::Remote::SearchServer->new(
+ searcher => $searcher,
+ port => 7890,
+ password => $pass,
+ );
+ $search_server->serve;
+
+=head1 DESCRIPTION
+
+The SearchServer class, in conjunction with
+L<SearchClient|LucyX::Remote::SearchClient>, makes it possible to run
+a search on one machine and report results on another.
+
+By aggregating several SearchClients under a
+L<PolySearcher|Lucy::Search::PolySearcher>, the cost of searching
+what might have been a prohibitively large monolithic index can be distributed
+across multiple nodes, each with its own, smaller index.
+
+=head1 METHODS
+
+=head2 new
+
+ my $search_server = LucyX::Remote::SearchServer->new(
+ searcher => $searcher, # required
+ port => 7890, # required
+ password => $pass, # required
+ );
+
+Constructor. Takes hash-style parameters.
+
+=over
+
+=item *
+
+B<searcher> - the L<Searcher|Lucy::Search::IndexSearcher> that the SearchServer
+will wrap.
+
+=item *
+
+B<port> - the port on localhost that the server should open and listen on.
+
+=item *
+
+B<password> - a password which must be supplied by clients.
+
+=back
+
+=head2 serve
+
+ $search_server->serve;
+
+Open a listening socket on localhost and wait for SearchClients to connect.
+
+=cut
diff --git a/perl/lib/LucyX/Search/Filter.pm b/perl/lib/LucyX/Search/Filter.pm
new file mode 100644
index 0000000..364b2eb
--- /dev/null
+++ b/perl/lib/LucyX/Search/Filter.pm
@@ -0,0 +1,263 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package LucyX::Search::Filter;
+BEGIN { our @ISA = qw( Lucy::Search::Query ) }
+use Carp;
+use Storable qw( nfreeze thaw );
+use Scalar::Util qw( blessed weaken );
+use bytes;
+no bytes;
+
+# Inside-out member vars.
+our %query;
+our %cached_bits;
+
+sub new {
+ my ( $either, %args ) = @_;
+ my $query = delete $args{query};
+ confess("required parameter query is not a Lucy::Search::Query")
+ unless ( blessed($query)
+ && $query->isa('Lucy::Search::Query') );
+ my $self = $either->SUPER::new(%args);
+ $self->_init_cache;
+ $query{$$self} = $query;
+ $self->set_boost(0);
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $query{$$self};
+ delete $cached_bits{$$self};
+ $self->SUPER::DESTROY;
+}
+
+sub make_compiler {
+ my $self = shift;
+ return LucyX::Search::FilterCompiler->new( @_, parent => $self );
+}
+
+sub serialize {
+ my ( $self, $outstream ) = @_;
+ $self->SUPER::serialize($outstream);
+ my $frozen = nfreeze( $query{$$self} );
+ $outstream->write_c32( bytes::length($frozen) );
+ $outstream->print($frozen);
+}
+
+sub deserialize {
+ my ( $self, $instream ) = @_;
+ $self->SUPER::deserialize($instream);
+ my $len = $instream->read_c32;
+ my $frozen;
+ $instream->read( $frozen, $len );
+ $query{$$self} = thaw($frozen);
+ return $self;
+}
+
+sub equals {
+ my ( $self, $other ) = @_;
+ return 0 unless $other->isa(__PACKAGE__);
+ return 0 unless $query{$$self}->equals( $query{$$other} );
+ return 0 unless $self->get_boost == $other->get_boost;
+ return 1;
+}
+
+sub to_string {
+ my $self = shift;
+ return 'Filter(' . $query{$$self}->to_string . ')';
+}
+
+sub _bits {
+ my ( $self, $seg_reader ) = @_;
+
+ my $cached_bits = $self->_fetch_cached_bits($seg_reader);
+
+ # Fill the cache.
+ if ( !defined $cached_bits ) {
+ $cached_bits = Lucy::Object::BitVector->new(
+ capacity => $seg_reader->doc_max + 1 );
+ $self->_store_cached_bits( $seg_reader, $cached_bits );
+
+ my $collector = Lucy::Search::Collector::BitCollector->new(
+ bit_vector => $cached_bits );
+
+ my $polyreader = Lucy::Index::PolyReader->new(
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ snapshot => $seg_reader->get_snapshot,
+ sub_readers => [$seg_reader],
+ );
+ my $searcher
+ = Lucy::Search::IndexSearcher->new( index => $polyreader );
+
+ # Perform the search.
+ $searcher->collect(
+ query => $query{$$self},
+ collector => $collector,
+ );
+ }
+
+ return $cached_bits;
+}
+
+# Store a cached BitVector associated with a particular SegReader. Store a
+# weak reference to the SegReader as an indicator of cache validity.
+sub _store_cached_bits {
+ my ( $self, $seg_reader, $bits ) = @_;
+ my $pair = { seg_reader => $seg_reader, bits => $bits };
+ weaken( $pair->{seg_reader} );
+ $cached_bits{$$self}{ $seg_reader->hash_sum } = $pair;
+}
+
+# Retrieve a cached BitVector associated with a particular SegReader. As a
+# side effect, clear away any BitVectors which are no longer valid because
+# their SegReaders have gone away.
+sub _fetch_cached_bits {
+ my ( $self, $seg_reader ) = @_;
+ my $cached_bits = $cached_bits{$$self};
+
+ # Sweep.
+ while ( my ( $hash_sum, $pair ) = each %$cached_bits ) {
+ # If weak ref has decomposed into undef, SegReader is gone... so
+ # delete.
+ next if defined $pair->{seg_reader};
+ delete $cached_bits->{$hash_sum};
+ }
+
+ # Fetch.
+ my $pair = $cached_bits->{ $seg_reader->hash_sum };
+ return $pair->{bits} if defined $pair;
+ return;
+}
+
+# Kill any existing cached filters.
+sub _init_cache {
+ my $self = shift;
+ $cached_bits{$$self} = {};
+}
+
+# Testing only.
+sub _cached_count {
+ my $self = shift;
+ return scalar grep { defined $cached_bits{$$self}{$_}{seg_reader} }
+ keys %{ $cached_bits{$$self} };
+}
+
+package LucyX::Search::FilterCompiler;
+BEGIN { our @ISA = qw( Lucy::Search::Compiler ) }
+
+sub new {
+ my ( $class, %args ) = @_;
+ $args{similarity} ||= $args{searcher}->get_schema->get_similarity;
+ return $class->SUPER::new(%args);
+}
+
+sub make_matcher {
+ my ( $self, %args ) = @_;
+ my $seg_reader = $args{reader};
+ my $bits = $self->get_parent->_bits($seg_reader);
+ return LucyX::Search::FilterMatcher->new(
+ bits => $bits,
+ doc_max => $seg_reader->doc_max,
+ );
+}
+
+package LucyX::Search::FilterMatcher;
+BEGIN { our @ISA = qw( Lucy::Search::Matcher ) }
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "LucyX::Search::FilterMatcher",
+ bind_constructors => ["new"],
+);
+
+__POD__
+
+=head1 NAME
+
+LucyX::Search::Filter - Build a caching filter based on results of a Query.
+
+=head1 SYNOPSIS
+
+ my %category_filters;
+ for my $category (qw( sweet sour salty bitter )) {
+ my $cat_query = Lucy::Search::TermQuery->new(
+ field => 'category',
+ term => $category,
+ );
+ $category_filters{$category} = LucyX::Search::Filter->new(
+ query => $cat_query,
+ );
+ }
+
+ while ( my $cgi = CGI::Fast->new ) {
+ my $user_query = $cgi->param('q');
+ my $filter = $category_filters{ $cgi->param('category') };
+ my $and_query = Lucy::Search::ANDQuery->new;
+ $and_query->add_child($user_query);
+ $and_query->add_child($filter);
+ my $hits = $searcher->hits( query => $and_query );
+ ...
+
+=head1 DESCRIPTION
+
+A Filter is a L<Lucy::Search::Query> subclass that can be used to filter
+the results of another Query. The effect is very similar to simply using the
+wrapped inner query, but there are two important differences:
+
+=over
+
+=item
+
+A Filter does not contribute to the score of the documents it matches.
+
+=item
+
+A Filter caches its results, so it is more efficient if you use it more than
+once.
+
+=back
+
+To obtain logically equivalent results to the Filter but avoid the caching,
+substitute the wrapped query but use set_boost() to set its C<boost> to 0.
+
+=head1 METHODS
+
+=head2 new
+
+ my $filter = LucyX::Search::Filter->new(
+ query => $query;
+ );
+
+Constructor. Takes one hash-style parameter, C<query>, which must be an
+object belonging to a subclass of L<Lucy::Search::Query>.
+
+=head1 BUGS
+
+Filters do not cache when used in a search cluster with LucyX::Remote's
+SearchServer and SearchClient.
+
+=cut
diff --git a/perl/lib/LucyX/Search/MockMatcher.pm b/perl/lib/LucyX/Search/MockMatcher.pm
new file mode 100644
index 0000000..b985103
--- /dev/null
+++ b/perl/lib/LucyX/Search/MockMatcher.pm
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use Lucy;
+
+package LucyX::Search::MockMatcher;
+
+sub new {
+ my ( $either, %args ) = @_;
+ confess("Missing doc_ids") unless ref( $args{doc_ids} ) eq 'ARRAY';
+ my $doc_ids = Lucy::Object::I32Array->new( ints => $args{doc_ids} );
+ my $size = $doc_ids->get_size;
+ my $scores;
+ if ( ref( $args{scores} ) eq 'ARRAY' ) {
+ confess("Mismatch between scores and doc_ids array sizes")
+ unless scalar @{ $args{scores} } == $size;
+ $scores = Lucy::Object::ByteBuf->new(
+ pack( "f$size", @{ $args{scores} } ) );
+ }
+
+ return $either->_new(
+ doc_ids => $doc_ids,
+ scores => $scores,
+ );
+}
+
+1;
+
+__END__
+
+__BINDING__
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "LucyX::Search::MockMatcher",
+ bind_constructors => ["_new|init"],
+);
+
+__POD__
+
+=head1 NAME
+
+LucyX::Search::MockMatcher - Matcher with arbitrary docs and scores.
+
+=head1 DESCRIPTION
+
+Used for testing combining L<Matchers|Lucy::Search::Matcher> such as
+ANDMatcher, MockMatcher allows arbitrary match criteria to be supplied,
+obviating the need for clever index construction to cover corner cases.
+
+MockMatcher is a testing and demonstration class; it is unsupported.
+
+=head1 CONSTRUCTORS
+
+=head2 new( [I<labeled params>] )
+
+=over
+
+=item *
+
+B<doc_ids> - A sorted array of L<doc_ids|Lucy::Docs::DocIDs>.
+
+=item *
+
+B<scores> - An array of scores, one for each doc_id.
+
+=back
+
+=cut
diff --git a/perl/lib/LucyX/Search/ProximityQuery.pm b/perl/lib/LucyX/Search/ProximityQuery.pm
new file mode 100644
index 0000000..c152376
--- /dev/null
+++ b/perl/lib/LucyX/Search/ProximityQuery.pm
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+package LucyX::Search::ProximityQuery;
+use Lucy;
+
+1;
+
+__END__
+
+__BINDING__
+
+my $synopsis = <<'END_SYNOPSIS';
+ my $proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [qw( the who )],
+ within => 10, # match within 10 positions
+ );
+ my $hits = $searcher->hits( query => $proximity_query );
+END_SYNOPSIS
+
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "LucyX::Search::ProximityQuery",
+ bind_methods => [qw( Get_Field Get_Terms )],
+ bind_constructors => ["new"],
+ make_pod => {
+ constructor => { sample => '' },
+ synopsis => $synopsis,
+ methods => [qw( get_field get_terms get_within )],
+ },
+);
+Clownfish::Binding::Perl::Class->register(
+ parcel => "Lucy",
+ class_name => "LucyX::Search::ProximityCompiler",
+ bind_constructors => ["do_new"],
+);
+
+
diff --git a/perl/sample/FlatQueryParser.pm b/perl/sample/FlatQueryParser.pm
new file mode 100644
index 0000000..cfef01e
--- /dev/null
+++ b/perl/sample/FlatQueryParser.pm
@@ -0,0 +1,100 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package FlatQueryParser;
+use base qw( Lucy::Search::QueryParser );
+use Lucy::Search::TermQuery;
+use Lucy::Search::PhraseQuery;
+use Lucy::Search::ORQuery;
+use PrefixQuery;
+use Carp;
+
+# Inherit new()
+
+sub parse {
+ my ( $self, $query_string ) = @_;
+ my $tokens = $self->_tokenize($query_string);
+ my $or_query = Lucy::Search::ORQuery->new;
+ for my $token (@$tokens) {
+ my $leaf_query = Lucy::Search::LeafQuery->new( text => $token );
+ $or_query->add_child($leaf_query);
+ }
+ return $self->expand($or_query);
+}
+
+sub _tokenize {
+ my ( $self, $query_string ) = @_;
+ my @tokens;
+ while ( length $query_string ) {
+ if ( $query_string =~ s/^\s+// ) {
+ next; # skip whitespace
+ }
+ elsif ( $query_string =~ s/^("[^"]*(?:"|$))// ) {
+ push @tokens, $1; # double-quoted phrase
+ }
+ else {
+ $query_string =~ s/(\S+)//;
+ push @tokens, $1; # single word
+ }
+ }
+ return \@tokens;
+}
+
+sub expand_leaf {
+ my ( $self, $leaf_query ) = @_;
+ my $text = $leaf_query->get_text;
+ if ( $text =~ /\*$/ ) {
+ my $or_query = Lucy::Search::ORQuery->new;
+ for my $field ( @{ $self->get_fields } ) {
+ my $prefix_query = PrefixQuery->new(
+ field => $field,
+ query_string => $text,
+ );
+ $or_query->add_child($prefix_query);
+ }
+ return $or_query;
+ }
+ else {
+ return $self->SUPER::expand_leaf($leaf_query);
+ }
+}
+
+1;
+
+__END__
+
+=head1 NAME
+
+FlatQueryParser - Simple query parser, with no boolean operators.
+
+=head1 SYNOPSIS
+
+ my $searcher = Lucy::Search::IndexSearcher->new(
+ index => '/path/to/index'
+ );
+ my $parser = FlatQueryParser->new( $searcher->get_schema );
+ my $query = $parser->parse($query_string);
+ my $hits = $searcher->hits( query => $query );
+ ...
+
+=head1 DESCRIPTION
+
+See L<Lucy::Docs::Cookbook::CustomQueryParser>.
+
+=cut
+
diff --git a/perl/sample/PrefixQuery.pm b/perl/sample/PrefixQuery.pm
new file mode 100644
index 0000000..b59ca81
--- /dev/null
+++ b/perl/sample/PrefixQuery.pm
@@ -0,0 +1,193 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package PrefixQuery;
+use base qw( Lucy::Search::Query );
+use Carp;
+use Scalar::Util qw( blessed );
+
+# Inside-out member vars and hand-rolled accessors.
+my %query_string;
+my %field;
+sub get_query_string { my $self = shift; return $query_string{$$self} }
+sub get_field { my $self = shift; return $field{$$self} }
+
+sub new {
+ my ( $class, %args ) = @_;
+ my $query_string = delete $args{query_string};
+ my $field = delete $args{field};
+ my $self = $class->SUPER::new(%args);
+ confess("'query_string' param is required")
+ unless defined $query_string;
+ confess("Invalid query_string: '$query_string'")
+ unless $query_string =~ /\*\s*$/;
+ confess("'field' param is required")
+ unless defined $field;
+ $query_string{$$self} = $query_string;
+ $field{$$self} = $field;
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $query_string{$$self};
+ delete $field{$$self};
+ $self->SUPER::DESTROY;
+}
+
+sub equals {
+ my ( $self, $other ) = @_;
+ return 0 unless blessed($other);
+ return 0 unless $other->isa("PrefixQuery");
+ return 0 unless $field{$$self} eq $field{$$other};
+ return 0 unless $query_string{$$self} eq $query_string{$$other};
+ return 1;
+}
+
+sub to_string {
+ my $self = shift;
+ return "$field{$$self}:$query_string{$$self}";
+}
+
+sub make_compiler {
+ my $self = shift;
+ return PrefixCompiler->new( @_, parent => $self );
+}
+
+package PrefixCompiler;
+use base qw( Lucy::Search::Compiler );
+
+sub make_matcher {
+ my ( $self, %args ) = @_;
+ my $seg_reader = $args{reader};
+
+ # Retrieve low-level components LexiconReader and PostingListReader.
+ my $lex_reader
+ = $seg_reader->obtain("Lucy::Index::LexiconReader");
+ my $plist_reader
+ = $seg_reader->obtain("Lucy::Index::PostingListReader");
+
+ # Acquire a Lexicon and seek it to our query string.
+ my $substring = $self->get_parent->get_query_string;
+ $substring =~ s/\*.\s*$//;
+ my $field = $self->get_parent->get_field;
+ my $lexicon = $lex_reader->lexicon( field => $field );
+ return unless $lexicon;
+ $lexicon->seek($substring);
+
+ # Accumulate PostingLists for each matching term.
+ my @posting_lists;
+ while ( defined( my $term = $lexicon->get_term ) ) {
+ last unless $term =~ /^\Q$substring/;
+ my $posting_list = $plist_reader->posting_list(
+ field => $field,
+ term => $term,
+ );
+ if ($posting_list) {
+ push @posting_lists, $posting_list;
+ }
+ last unless $lexicon->next;
+ }
+ return unless @posting_lists;
+
+ return PrefixMatcher->new( posting_lists => \@posting_lists );
+}
+
+package PrefixMatcher;
+use base qw( Lucy::Search::Matcher );
+
+# Inside-out member vars.
+my %doc_ids;
+my %tally;
+my %tick;
+
+sub new {
+ my ( $class, %args ) = @_;
+ my $posting_lists = delete $args{posting_lists};
+ my $self = $class->SUPER::new(%args);
+
+ # Cheesy but simple way of interleaving PostingList doc sets.
+ my %all_doc_ids;
+ for my $posting_list (@$posting_lists) {
+ while ( my $doc_id = $posting_list->next ) {
+ $all_doc_ids{$doc_id} = undef;
+ }
+ }
+ my @doc_ids = sort { $a <=> $b } keys %all_doc_ids;
+ $doc_ids{$$self} = \@doc_ids;
+
+ $tick{$$self} = -1;
+ $tally{$$self} = Lucy::Search::Tally->new;
+ $tally{$$self}->set_score(1.0); # fixed score of 1.0
+
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $doc_ids{$$self};
+ delete $tick{$$self};
+ delete $tally{$$self};
+ $self->SUPER::DESTROY;
+}
+
+sub next {
+ my $self = shift;
+ my $doc_ids = $doc_ids{$$self};
+ my $tick = ++$tick{$$self};
+ return 0 if $tick >= scalar @$doc_ids;
+ return $doc_ids->[$tick];
+}
+
+sub get_doc_id {
+ my $self = shift;
+ my $tick = $tick{$$self};
+ my $doc_ids = $doc_ids{$$self};
+ return $tick < scalar @$doc_ids ? $doc_ids->[$tick] : 0;
+}
+
+sub tally {
+ my $self = shift;
+ return $tally{$$self};
+}
+
+1;
+
+__END__
+
+__POD__
+
+=head1 SAMPLE CLASS
+
+PrefixQuery - Sample subclass of Lucy::Query, supporting trailing
+wildcards.
+
+=head1 SYNOPSIS
+
+ my $prefix_query = PrefixQuery->new(
+ field => 'content',
+ query_string => 'foo*',
+ );
+ my $hits = $searcher->hits( query => $prefix_query );
+
+=head1 DESCRIPTION
+
+See L<Lucy::Docs::Cookbook::CustomQuery>.
+
+=cut
+
diff --git a/perl/sample/README.txt b/perl/sample/README.txt
new file mode 100644
index 0000000..4ad7a06
--- /dev/null
+++ b/perl/sample/README.txt
@@ -0,0 +1,2 @@
+For more information on the sample code in this directory, see
+Lucy::Docs::Tutorial and Lucy::Docs::Cookbook.
diff --git a/perl/sample/indexer.pl b/perl/sample/indexer.pl
new file mode 100644
index 0000000..559f31f
--- /dev/null
+++ b/perl/sample/indexer.pl
@@ -0,0 +1,97 @@
+#!/usr/local/bin/perl
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+# (Change configuration variables as needed.)
+my $path_to_index = '/path/to/index';
+my $uscon_source = '/usr/local/apache2/htdocs/us_constitution';
+
+use File::Spec::Functions qw( catfile );
+use Lucy::Plan::Schema;
+use Lucy::Plan::FullTextType;
+use Lucy::Analysis::PolyAnalyzer;
+use Lucy::Index::Indexer;
+
+# Create Schema.
+my $schema = Lucy::Plan::Schema->new;
+my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ language => 'en',
+);
+my $title_type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+);
+my $content_type = Lucy::Plan::FullTextType->new(
+ analyzer => $polyanalyzer,
+ highlightable => 1,
+);
+my $url_type = Lucy::Plan::StringType->new( indexed => 0, );
+my $cat_type = Lucy::Plan::StringType->new( stored => 0, );
+$schema->spec_field( name => 'title', type => $title_type );
+$schema->spec_field( name => 'content', type => $content_type );
+$schema->spec_field( name => 'url', type => $url_type );
+$schema->spec_field( name => 'category', type => $cat_type );
+
+# Create an Indexer object.
+my $indexer = Lucy::Index::Indexer->new(
+ index => $path_to_index,
+ schema => $schema,
+ create => 1,
+ truncate => 1,
+);
+
+# Collect names of source files.
+opendir( my $dh, $uscon_source )
+ or die "Couldn't opendir '$uscon_source': $!";
+my @filenames = grep { $_ =~ /\.txt/ } readdir $dh;
+
+# Iterate over list of source files.
+for my $filename (@filenames) {
+ print "Indexing $filename\n";
+ my $doc = parse_file($filename);
+ $indexer->add_doc($doc);
+}
+
+# Finalize the index and print a confirmation message.
+$indexer->commit;
+print "Finished.\n";
+
+# Parse a file from our US Constitution collection and return a hashref with
+# the fields title, body, url, and category.
+sub parse_file {
+ my $filename = shift;
+ my $filepath = catfile( $uscon_source, $filename );
+ open( my $fh, '<', $filepath ) or die "Can't open '$filepath': $!";
+ my $text = do { local $/; <$fh> }; # slurp file content
+ $text =~ /\A(.+?)^\s+(.*)/ms
+ or die "Can't extract title/bodytext from '$filepath'";
+ my $title = $1;
+ my $bodytext = $2;
+ my $category
+ = $filename =~ /art/ ? 'article'
+ : $filename =~ /amend/ ? 'amendment'
+ : $filename =~ /preamble/ ? 'preamble'
+ : die "Can't derive category for $filename";
+ return {
+ title => $title,
+ content => $bodytext,
+ url => "/us_constitution/$filename",
+ category => $category,
+ };
+}
+
diff --git a/perl/sample/search.cgi b/perl/sample/search.cgi
new file mode 100755
index 0000000..290a76f
--- /dev/null
+++ b/perl/sample/search.cgi
@@ -0,0 +1,243 @@
+#!/usr/local/bin/perl -T
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+# (Change configuration variables as needed.)
+my $path_to_index = '/path/to/index';
+
+use CGI;
+use List::Util qw( max min );
+use POSIX qw( ceil );
+use Encode qw( decode );
+use Lucy::Search::IndexSearcher;
+use Lucy::Highlight::Highlighter;
+use Lucy::Search::QueryParser;
+use Lucy::Search::TermQuery;
+use Lucy::Search::ANDQuery;
+
+my $cgi = CGI->new;
+my $q = decode( "UTF-8", $cgi->param('q') || '' );
+my $offset = decode( "UTF-8", $cgi->param('offset') || 0 );
+my $category = decode( "UTF-8", $cgi->param('category') || '' );
+my $page_size = 10;
+
+# Create an IndexSearcher and a QueryParser.
+my $searcher = Lucy::Search::IndexSearcher->new(
+ index => $path_to_index,
+);
+my $qparser = Lucy::Search::QueryParser->new(
+ schema => $searcher->get_schema,
+);
+
+# Build up a Query.
+my $query = $qparser->parse($q);
+if ($category) {
+ my $category_query = Lucy::Search::TermQuery->new(
+ field => 'category',
+ term => $category,
+ );
+ $query = Lucy::Search::ANDQuery->new(
+ children => [ $query, $category_query ]
+ );
+}
+
+# Execute the Query and get a Hits object.
+my $hits = $searcher->hits(
+ query => $query,
+ offset => $offset,
+ num_wanted => $page_size,
+);
+my $hit_count = $hits->total_hits;
+
+# Arrange for highlighted excerpts to be created.
+my $highlighter = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content'
+);
+
+# Create result list.
+my $report = '';
+while ( my $hit = $hits->next ) {
+ my $score = sprintf( "%0.3f", $hit->get_score );
+ my $excerpt = $highlighter->create_excerpt($hit);
+ $report .= qq|
+ <p>
+ <a href="$hit->{url}"><strong>$hit->{title}</strong></a>
+ <em>$score</em>
+ <br />
+ $excerpt
+ <br />
+ <span class="excerptURL">$hit->{url}</span>
+ </p>
+ |;
+}
+
+#--------------------------------------------------------------------#
+# No Lucy tutorial material below this point - just html generation. #
+#--------------------------------------------------------------------#
+
+# Generate html, print and exit.
+my $paging_links = generate_paging_info( $q, $hit_count );
+my $cat_select = generate_category_select($category);
+blast_out_content( $q, $report, $paging_links, $cat_select );
+
+# Create html fragment with links for paging through results n-at-a-time.
+sub generate_paging_info {
+ my ( $query_string, $total_hits ) = @_;
+ my $escaped_q = CGI::escapeHTML($query_string);
+ my $paging_info;
+ if ( !length $query_string ) {
+ # No query? No display.
+ $paging_info = '';
+ }
+ elsif ( $total_hits == 0 ) {
+ # Alert the user that their search failed.
+ $paging_info
+ = qq|<p>No matches for <strong>$escaped_q</strong></p>|;
+ }
+ else {
+ # Calculate the nums for the first and last hit to display.
+ my $last_result = min( ( $offset + $page_size ), $total_hits );
+ my $first_result = min( ( $offset + 1 ), $last_result );
+
+ # Display the result nums, start paging info.
+ $paging_info = qq|
+ <p>
+ Results <strong>$first_result-$last_result</strong>
+ of <strong>$total_hits</strong>
+ for <strong>$escaped_q</strong>.
+ </p>
+ <p>
+ Results Page:
+ |;
+
+ # Calculate first and last hits pages to display / link to.
+ my $current_page = int( $first_result / $page_size ) + 1;
+ my $last_page = ceil( $total_hits / $page_size );
+ my $first_page = max( 1, ( $current_page - 9 ) );
+ $last_page = min( $last_page, ( $current_page + 10 ) );
+
+ # Create a url for use in paging links.
+ my $href = $cgi->url( -relative => 1 );
+ $href .= "?q=" . CGI::escape($query_string);
+ $href .= ";category=" . CGI::escape($category);
+ $href .= ";offset=" . CGI::escape($offset);
+
+ # Generate the "Prev" link.
+ if ( $current_page > 1 ) {
+ my $new_offset = ( $current_page - 2 ) * $page_size;
+ $href =~ s/(?<=offset=)\d+/$new_offset/;
+ $paging_info .= qq|<a href="$href"><= Prev</a>\n|;
+ }
+
+ # Generate paging links.
+ for my $page_num ( $first_page .. $last_page ) {
+ if ( $page_num == $current_page ) {
+ $paging_info .= qq|$page_num \n|;
+ }
+ else {
+ my $new_offset = ( $page_num - 1 ) * $page_size;
+ $href =~ s/(?<=offset=)\d+/$new_offset/;
+ $paging_info .= qq|<a href="$href">$page_num</a>\n|;
+ }
+ }
+
+ # Generate the "Next" link.
+ if ( $current_page != $last_page ) {
+ my $new_offset = $current_page * $page_size;
+ $href =~ s/(?<=offset=)\d+/$new_offset/;
+ $paging_info .= qq|<a href="$href">Next =></a>\n|;
+ }
+
+ # Close tag.
+ $paging_info .= "</p>\n";
+ }
+
+ return $paging_info;
+}
+
+# Build up the HTML "select" object for the "category" field.
+sub generate_category_select {
+ my $cat = shift;
+ my $select = qq|
+ <select name="category">
+ <option value="">All Sections</option>
+ <option value="article">Articles</option>
+ <option value="amendment">Amendments</option>
+ </select>|;
+ if ($cat) {
+ $select =~ s/"$cat"/"$cat" selected/;
+ }
+ return $select;
+}
+
+# Print content to output.
+sub blast_out_content {
+ my ( $query_string, $hit_list, $paging_info, $category_select ) = @_;
+ my $escaped_q = CGI::escapeHTML($query_string);
+ binmode( STDOUT, ":encoding(UTF-8)" );
+ print qq|Content-type: text/html; charset=UTF-8\n\n|;
+ print qq|
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<head>
+ <meta http-equiv="Content-type"
+ content="text/html;charset=UTF-8">
+ <link rel="stylesheet" type="text/css"
+ href="/us_constitution/uscon.css">
+ <title>Lucy: $escaped_q</title>
+</head>
+
+<body>
+
+ <div id="navigation">
+ <form id="usconSearch" action="">
+ <strong>
+ Search the
+ <a href="/us_constitution/index.html">US Constitution</a>:
+ </strong>
+ <input type="text" name="q" id="q" value="$escaped_q">
+ $category_select
+ <input type="submit" value="=>">
+ </form>
+ </div><!--navigation-->
+
+ <div id="bodytext">
+
+ $hit_list
+
+ $paging_info
+
+ </div><!--bodytext-->
+ <p style="font-size: smaller; color: #666">
+ <em>
+ Powered by <a href="http://incubator.apache.org/lucy/"
+ >Apache Lucy<small><sup>TM</sup></small></a>
+ </em>
+ </p>
+ </div><!--bodytext-->
+
+</body>
+
+</html>
+|;
+}
+
diff --git a/perl/sample/us_constitution/amend1.txt b/perl/sample/us_constitution/amend1.txt
new file mode 100644
index 0000000..f410f65
--- /dev/null
+++ b/perl/sample/us_constitution/amend1.txt
@@ -0,0 +1,7 @@
+Amendment I
+
+Congress shall make no law respecting an establishment of religion, or
+prohibiting the free exercise thereof; or abridging the freedom of speech, or
+of the press; or the right of the people peaceably to assemble, and to
+petition the Government for a redress of grievances.
+
diff --git a/perl/sample/us_constitution/amend10.txt b/perl/sample/us_constitution/amend10.txt
new file mode 100644
index 0000000..10fd290
--- /dev/null
+++ b/perl/sample/us_constitution/amend10.txt
@@ -0,0 +1,6 @@
+Amendment X
+
+The powers not delegated to the United States by the Constitution, nor
+prohibited by it to the States, are reserved to the States respectively, or
+to the people.
+
diff --git a/perl/sample/us_constitution/amend11.txt b/perl/sample/us_constitution/amend11.txt
new file mode 100644
index 0000000..c2a7e3d
--- /dev/null
+++ b/perl/sample/us_constitution/amend11.txt
@@ -0,0 +1,7 @@
+Amendment XI
+
+The Judicial power of the United States shall not be construed to extend to
+any suit in law or equity, commenced or prosecuted against one of the United
+States by Citizens of another State, or by Citizens or Subjects of any
+Foreign State.
+
diff --git a/perl/sample/us_constitution/amend12.txt b/perl/sample/us_constitution/amend12.txt
new file mode 100644
index 0000000..0906c89
--- /dev/null
+++ b/perl/sample/us_constitution/amend12.txt
@@ -0,0 +1,39 @@
+Amendment XII
+
+The Electors shall meet in their respective states, and vote by ballot for
+President and Vice-President, one of whom, at least, shall not be an
+inhabitant of the same state with themselves; they shall name in their
+ballots the person voted for as President, and in distinct ballots the person
+voted for as Vice-President, and they shall make distinct lists of all
+persons voted for as President, and of all persons voted for as
+Vice-President and of the number of votes for each, which lists they shall
+sign and certify, and transmit sealed to the seat of the government of the
+United States, directed to the President of the Senate;
+
+The President of the Senate shall, in the presence of the Senate and House of
+Representatives, open all the certificates and the votes shall then be
+counted;
+
+The person having the greatest Number of votes for President, shall be the
+President, if such number be a majority of the whole number of Electors
+appointed; and if no person have such majority, then from the persons having
+the highest numbers not exceeding three on the list of those voted for as
+President, the House of Representatives shall choose immediately, by ballot,
+the President. But in choosing the President, the votes shall be taken by
+states, the representation from each state having one vote; a quorum for this
+purpose shall consist of a member or members from two-thirds of the states,
+and a majority of all the states shall be necessary to a choice. And if the
+House of Representatives shall not choose a President whenever the right of
+choice shall devolve upon them, before the fourth day of March next
+following, then the Vice-President shall act as President, as in the case of
+the death or other constitutional disability of the President.
+
+The person having the greatest number of votes as Vice-President, shall be
+the Vice-President, if such number be a majority of the whole number of
+Electors appointed, and if no person have a majority, then from the two
+highest numbers on the list, the Senate shall choose the Vice-President; a
+quorum for the purpose shall consist of two-thirds of the whole number of
+Senators, and a majority of the whole number shall be necessary to a choice.
+But no person constitutionally ineligible to the office of President shall be
+eligible to that of Vice-President of the United States.
+
diff --git a/perl/sample/us_constitution/amend13.txt b/perl/sample/us_constitution/amend13.txt
new file mode 100644
index 0000000..4a6afd5
--- /dev/null
+++ b/perl/sample/us_constitution/amend13.txt
@@ -0,0 +1,9 @@
+Amendment XIII
+
+1. Neither slavery nor involuntary servitude, except as a punishment for
+crime whereof the party shall have been duly convicted, shall exist within
+the United States, or any place subject to their jurisdiction.
+
+2. Congress shall have power to enforce this article by appropriate
+legislation.
+
diff --git a/perl/sample/us_constitution/amend14.txt b/perl/sample/us_constitution/amend14.txt
new file mode 100644
index 0000000..74cb04c
--- /dev/null
+++ b/perl/sample/us_constitution/amend14.txt
@@ -0,0 +1,43 @@
+Amendment XIV
+
+1. All persons born or naturalized in the United States, and subject to the
+jurisdiction thereof, are citizens of the United States and of the State
+wherein they reside. No State shall make or enforce any law which shall
+abridge the privileges or immunities of citizens of the United States; nor
+shall any State deprive any person of life, liberty, or property, without due
+process of law; nor deny to any person within its jurisdiction the equal
+protection of the laws.
+
+2. Representatives shall be apportioned among the several States according to
+their respective numbers, counting the whole number of persons in each State,
+excluding Indians not taxed. But when the right to vote at any election for
+the choice of electors for President and Vice-President of the United States,
+Representatives in Congress, the Executive and Judicial officers of a State,
+or the members of the Legislature thereof, is denied to any of the male
+inhabitants of such State, being twenty-one years of age, and citizens of the
+United States, or in any way abridged, except for participation in rebellion,
+or other crime, the basis of representation therein shall be reduced in the
+proportion which the number of such male citizens shall bear to the whole
+number of male citizens twenty-one years of age in such State.
+
+3. No person shall be a Senator or Representative in Congress, or elector of
+President and Vice-President, or hold any office, civil or military, under
+the United States, or under any State, who, having previously taken an oath,
+as a member of Congress, or as an officer of the United States, or as a
+member of any State legislature, or as an executive or judicial officer of
+any State, to support the Constitution of the United States, shall have
+engaged in insurrection or rebellion against the same, or given aid or
+comfort to the enemies thereof. But Congress may by a vote of two-thirds of
+each House, remove such disability.
+
+4. The validity of the public debt of the United States, authorized by law,
+including debts incurred for payment of pensions and bounties for services in
+suppressing insurrection or rebellion, shall not be questioned. But neither
+the United States nor any State shall assume or pay any debt or obligation
+incurred in aid of insurrection or rebellion against the United States, or
+any claim for the loss or emancipation of any slave; but all such debts,
+obligations and claims shall be held illegal and void.
+
+5. The Congress shall have power to enforce, by appropriate legislation, the
+provisions of this article.
+
diff --git a/perl/sample/us_constitution/amend15.txt b/perl/sample/us_constitution/amend15.txt
new file mode 100644
index 0000000..ebe8d81
--- /dev/null
+++ b/perl/sample/us_constitution/amend15.txt
@@ -0,0 +1,9 @@
+Amendment XV
+
+1. The right of citizens of the United States to vote shall not be denied or
+abridged by the United States or by any State on account of race, color, or
+previous condition of servitude.
+
+2. The Congress shall have power to enforce this article by appropriate
+legislation.
+
diff --git a/perl/sample/us_constitution/amend16.txt b/perl/sample/us_constitution/amend16.txt
new file mode 100644
index 0000000..ed6087c
--- /dev/null
+++ b/perl/sample/us_constitution/amend16.txt
@@ -0,0 +1,6 @@
+Amendment XVI
+
+ The Congress shall have power to lay and collect taxes on incomes, from
+whatever source derived, without apportionment among the several States, and
+without regard to any census or enumeration.
+
diff --git a/perl/sample/us_constitution/amend17.txt b/perl/sample/us_constitution/amend17.txt
new file mode 100644
index 0000000..18fbb15
--- /dev/null
+++ b/perl/sample/us_constitution/amend17.txt
@@ -0,0 +1,16 @@
+Amendment XVII
+
+The Senate of the United States shall be composed of two Senators from each
+State, elected by the people thereof, for six years; and each Senator shall
+have one vote. The electors in each State shall have the qualifications
+requisite for electors of the most numerous branch of the State legislatures.
+
+When vacancies happen in the representation of any State in the Senate, the
+executive authority of such State shall issue writs of election to fill such
+vacancies: Provided, That the legislature of any State may empower the
+executive thereof to make temporary appointments until the people fill the
+vacancies by election as the legislature may direct.
+
+This amendment shall not be so construed as to affect the election or term of
+any Senator chosen before it becomes valid as part of the Constitution.
+
diff --git a/perl/sample/us_constitution/amend18.txt b/perl/sample/us_constitution/amend18.txt
new file mode 100644
index 0000000..6e6fab5
--- /dev/null
+++ b/perl/sample/us_constitution/amend18.txt
@@ -0,0 +1,16 @@
+Amendment XVIII
+
+1. After one year from the ratification of this article the manufacture,
+sale, or transportation of intoxicating liquors within, the importation
+thereof into, or the exportation thereof from the United States and all
+territory subject to the jurisdiction thereof for beverage purposes is hereby
+prohibited.
+
+2. The Congress and the several States shall have concurrent power to enforce
+this article by appropriate legislation.
+
+3. This article shall be inoperative unless it shall have been ratified as an
+amendment to the Constitution by the legislatures of the several States, as
+provided in the Constitution, within seven years from the date of the
+submission hereof to the States by the Congress.
+
diff --git a/perl/sample/us_constitution/amend19.txt b/perl/sample/us_constitution/amend19.txt
new file mode 100644
index 0000000..ec3c054
--- /dev/null
+++ b/perl/sample/us_constitution/amend19.txt
@@ -0,0 +1,7 @@
+Amendment XIX
+
+The right of citizens of the United States to vote shall not be denied or
+abridged by the United States or by any State on account of sex.
+
+Congress shall have power to enforce this article by appropriate legislation.
+
diff --git a/perl/sample/us_constitution/amend2.txt b/perl/sample/us_constitution/amend2.txt
new file mode 100644
index 0000000..3389890
--- /dev/null
+++ b/perl/sample/us_constitution/amend2.txt
@@ -0,0 +1,5 @@
+Amendment II
+
+A well regulated Militia, being necessary to the security of a free State,
+the right of the people to keep and bear Arms, shall not be infringed.
+
diff --git a/perl/sample/us_constitution/amend20.txt b/perl/sample/us_constitution/amend20.txt
new file mode 100644
index 0000000..92eeaa0
--- /dev/null
+++ b/perl/sample/us_constitution/amend20.txt
@@ -0,0 +1,36 @@
+Amendment XX
+
+1. The terms of the President and Vice President shall end at noon on the
+20th day of January, and the terms of Senators and Representatives at noon on
+the 3d day of January, of the years in which such terms would have ended if
+this article had not been ratified; and the terms of their successors shall
+then begin.
+
+2. The Congress shall assemble at least once in every year, and such meeting
+shall begin at noon on the 3d day of January, unless they shall by law
+appoint a different day.
+
+3. If, at the time fixed for the beginning of the term of the President, the
+President elect shall have died, the Vice President elect shall become
+President. If a President shall not have been chosen before the time fixed
+for the beginning of his term, or if the President elect shall have failed to
+qualify, then the Vice President elect shall act as President until a
+President shall have qualified; and the Congress may by law provide for the
+case wherein neither a President elect nor a Vice President elect shall have
+qualified, declaring who shall then act as President, or the manner in which
+one who is to act shall be selected, and such person shall act accordingly
+until a President or Vice President shall have qualified.
+
+4. The Congress may by law provide for the case of the death of any of the
+persons from whom the House of Representatives may choose a President
+whenever the right of choice shall have devolved upon them, and for the case
+of the death of any of the persons from whom the Senate may choose a Vice
+President whenever the right of choice shall have devolved upon them.
+
+5. Sections 1 and 2 shall take effect on the 15th day of October following
+the ratification of this article.
+
+6. This article shall be inoperative unless it shall have been ratified as an
+amendment to the Constitution by the legislatures of three-fourths of the
+several States within seven years from the date of its submission.
+
diff --git a/perl/sample/us_constitution/amend21.txt b/perl/sample/us_constitution/amend21.txt
new file mode 100644
index 0000000..4b36e55
--- /dev/null
+++ b/perl/sample/us_constitution/amend21.txt
@@ -0,0 +1,14 @@
+Amendment XXI
+
+1. The eighteenth article of amendment to the Constitution of the United
+States is hereby repealed.
+
+2. The transportation or importation into any State, Territory, or possession
+of the United States for delivery or use therein of intoxicating liquors, in
+violation of the laws thereof, is hereby prohibited.
+
+3. The article shall be inoperative unless it shall have been ratified as an
+amendment to the Constitution by conventions in the several States, as
+provided in the Constitution, within seven years from the date of the
+submission hereof to the States by the Congress.
+
diff --git a/perl/sample/us_constitution/amend22.txt b/perl/sample/us_constitution/amend22.txt
new file mode 100644
index 0000000..782afa9
--- /dev/null
+++ b/perl/sample/us_constitution/amend22.txt
@@ -0,0 +1,17 @@
+Amendment XXII
+
+1. No person shall be elected to the office of the President more than twice,
+and no person who has held the office of President, or acted as President,
+for more than two years of a term to which some other person was elected
+President shall be elected to the office of the President more than once. But
+this Article shall not apply to any person holding the office of President,
+when this Article was proposed by the Congress, and shall not prevent any
+person who may be holding the office of President, or acting as President,
+during the term within which this Article becomes operative from holding the
+office of President or acting as President during the remainder of such term.
+
+2. This article shall be inoperative unless it shall have been ratified as an
+amendment to the Constitution by the legislatures of three-fourths of the
+several States within seven years from the date of its submission to the
+States by the Congress.
+
diff --git a/perl/sample/us_constitution/amend23.txt b/perl/sample/us_constitution/amend23.txt
new file mode 100644
index 0000000..ce2b31d
--- /dev/null
+++ b/perl/sample/us_constitution/amend23.txt
@@ -0,0 +1,15 @@
+Amendment XXIII
+
+1. The District constituting the seat of Government of the United States
+shall appoint in such manner as the Congress may direct: A number of electors
+of President and Vice President equal to the whole number of Senators and
+Representatives in Congress to which the District would be entitled if it
+were a State, but in no event more than the least populous State; they shall
+be in addition to those appointed by the States, but they shall be
+considered, for the purposes of the election of President and Vice President,
+to be electors appointed by a State; and they shall meet in the District and
+perform such duties as provided by the twelfth article of amendment.
+
+2. The Congress shall have power to enforce this article by appropriate
+legislation.
+
diff --git a/perl/sample/us_constitution/amend24.txt b/perl/sample/us_constitution/amend24.txt
new file mode 100644
index 0000000..809b582
--- /dev/null
+++ b/perl/sample/us_constitution/amend24.txt
@@ -0,0 +1,11 @@
+Amendment XXIV
+
+1. The right of citizens of the United States to vote in any primary or other
+election for President or Vice President, for electors for President or Vice
+President, or for Senator or Representative in Congress, shall not be denied
+or abridged by the United States or any State by reason of failure to pay any
+poll tax or other tax.
+
+2. The Congress shall have power to enforce this article by appropriate
+legislation.
+
diff --git a/perl/sample/us_constitution/amend25.txt b/perl/sample/us_constitution/amend25.txt
new file mode 100644
index 0000000..69b3472
--- /dev/null
+++ b/perl/sample/us_constitution/amend25.txt
@@ -0,0 +1,41 @@
+Amendment XXV
+
+1. In case of the removal of the President from office or of his death or
+resignation, the Vice President shall become President.
+
+2. Whenever there is a vacancy in the office of the Vice President, the
+President shall nominate a Vice President who shall take office upon
+confirmation by a majority vote of both Houses of Congress.
+
+3. Whenever the President transmits to the President pro tempore of the
+Senate and the Speaker of the House of Representatives his written
+declaration that he is unable to discharge the powers and duties of his
+office, and until he transmits to them a written declaration to the contrary,
+such powers and duties shall be discharged by the Vice President as Acting
+President.
+
+4. Whenever the Vice President and a majority of either the principal
+officers of the executive departments or of such other body as Congress may
+by law provide, transmit to the President pro tempore of the Senate and the
+Speaker of the House of Representatives their written declaration that the
+President is unable to discharge the powers and duties of his office, the
+Vice President shall immediately assume the powers and duties of the office
+as Acting President.
+
+Thereafter, when the President transmits to the President pro tempore of the
+Senate and the Speaker of the House of Representatives his written
+declaration that no inability exists, he shall resume the powers and duties
+of his office unless the Vice President and a majority of either the
+principal officers of the executive department or of such other body as
+Congress may by law provide, transmit within four days to the President pro
+tempore of the Senate and the Speaker of the House of Representatives their
+written declaration that the President is unable to discharge the powers and
+duties of his office. Thereupon Congress shall decide the issue, assembling
+within forty eight hours for that purpose if not in session. If the Congress,
+within twenty one days after receipt of the latter written declaration, or,
+if Congress is not in session, within twenty one days after Congress is
+required to assemble, determines by two thirds vote of both Houses that the
+President is unable to discharge the powers and duties of his office, the
+Vice President shall continue to discharge the same as Acting President;
+otherwise, the President shall resume the powers and duties of his office.
+
diff --git a/perl/sample/us_constitution/amend26.txt b/perl/sample/us_constitution/amend26.txt
new file mode 100644
index 0000000..2780a76
--- /dev/null
+++ b/perl/sample/us_constitution/amend26.txt
@@ -0,0 +1,9 @@
+Amendment XXVI
+
+1. The right of citizens of the United States, who are eighteen years of age
+or older, to vote shall not be denied or abridged by the United States or by
+any State on account of age.
+
+2. The Congress shall have power to enforce this article by appropriate
+legislation.
+
diff --git a/perl/sample/us_constitution/amend27.txt b/perl/sample/us_constitution/amend27.txt
new file mode 100644
index 0000000..6f10920
--- /dev/null
+++ b/perl/sample/us_constitution/amend27.txt
@@ -0,0 +1,6 @@
+Amendment XXVII
+
+No law, varying the compensation for the services of the Senators and
+Representatives, shall take effect, until an election of Representatives
+shall have intervened.
+
diff --git a/perl/sample/us_constitution/amend3.txt b/perl/sample/us_constitution/amend3.txt
new file mode 100644
index 0000000..edf39e0
--- /dev/null
+++ b/perl/sample/us_constitution/amend3.txt
@@ -0,0 +1,6 @@
+Amendment III
+
+No Soldier shall, in time of peace be quartered in any house, without the
+consent of the Owner, nor in time of war, but in a manner to be prescribed by
+law.
+
diff --git a/perl/sample/us_constitution/amend4.txt b/perl/sample/us_constitution/amend4.txt
new file mode 100644
index 0000000..eb55b4f
--- /dev/null
+++ b/perl/sample/us_constitution/amend4.txt
@@ -0,0 +1,8 @@
+Amendment IV
+
+The right of the people to be secure in their persons, houses, papers, and
+effects, against unreasonable searches and seizures, shall not be violated,
+and no Warrants shall issue, but upon probable cause, supported by Oath or
+affirmation, and particularly describing the place to be searched, and the
+persons or things to be seized.
+
diff --git a/perl/sample/us_constitution/amend5.txt b/perl/sample/us_constitution/amend5.txt
new file mode 100644
index 0000000..f975969
--- /dev/null
+++ b/perl/sample/us_constitution/amend5.txt
@@ -0,0 +1,11 @@
+Amendment V
+
+No person shall be held to answer for a capital, or otherwise infamous crime,
+unless on a presentment or indictment of a Grand Jury, except in cases
+arising in the land or naval forces, or in the Militia, when in actual
+service in time of War or public danger; nor shall any person be subject for
+the same offense to be twice put in jeopardy of life or limb; nor shall be
+compelled in any criminal case to be a witness against himself, nor be
+deprived of life, liberty, or property, without due process of law; nor shall
+private property be taken for public use, without just compensation.
+
diff --git a/perl/sample/us_constitution/amend6.txt b/perl/sample/us_constitution/amend6.txt
new file mode 100644
index 0000000..38ef75f
--- /dev/null
+++ b/perl/sample/us_constitution/amend6.txt
@@ -0,0 +1,10 @@
+Amendment VI
+
+In all criminal prosecutions, the accused shall enjoy the right to a speedy
+and public trial, by an impartial jury of the State and district wherein the
+crime shall have been committed, which district shall have been previously
+ascertained by law, and to be informed of the nature and cause of the
+accusation; to be confronted with the witnesses against him; to have
+compulsory process for obtaining witnesses in his favor, and to have the
+Assistance of Counsel for his defence.
+
diff --git a/perl/sample/us_constitution/amend7.txt b/perl/sample/us_constitution/amend7.txt
new file mode 100644
index 0000000..f1b9076
--- /dev/null
+++ b/perl/sample/us_constitution/amend7.txt
@@ -0,0 +1,7 @@
+Amendment VII
+
+In Suits at common law, where the value in controversy shall exceed twenty
+dollars, the right of trial by jury shall be preserved, and no fact tried by
+a jury, shall be otherwise re-examined in any Court of the United States,
+than according to the rules of the common law.
+
diff --git a/perl/sample/us_constitution/amend8.txt b/perl/sample/us_constitution/amend8.txt
new file mode 100644
index 0000000..d2abc91
--- /dev/null
+++ b/perl/sample/us_constitution/amend8.txt
@@ -0,0 +1,5 @@
+Amendment VIII
+
+Excessive bail shall not be required, nor excessive fines imposed, nor cruel
+and unusual punishments inflicted.
+
diff --git a/perl/sample/us_constitution/amend9.txt b/perl/sample/us_constitution/amend9.txt
new file mode 100644
index 0000000..4315578
--- /dev/null
+++ b/perl/sample/us_constitution/amend9.txt
@@ -0,0 +1,5 @@
+Amendment IX
+
+The enumeration in the Constitution, of certain rights, shall not be
+construed to deny or disparage others retained by the people.
+
diff --git a/perl/sample/us_constitution/art1sec1.txt b/perl/sample/us_constitution/art1sec1.txt
new file mode 100644
index 0000000..81e49fd
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec1.txt
@@ -0,0 +1,5 @@
+Article I Section 1
+
+All legislative Powers herein granted shall be vested in a Congress of the
+United States, which shall consist of a Senate and House of Representatives.
+
diff --git a/perl/sample/us_constitution/art1sec10.txt b/perl/sample/us_constitution/art1sec10.txt
new file mode 100644
index 0000000..e8362e4
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec10.txt
@@ -0,0 +1,20 @@
+Article I Section 10
+
+No State shall enter into any Treaty, Alliance, or Confederation; grant
+Letters of Marque and Reprisal; coin Money; emit Bills of Credit; make any
+Thing but gold and silver Coin a Tender in Payment of Debts; pass any Bill of
+Attainder, ex post facto Law, or Law impairing the Obligation of Contracts,
+or grant any Title of Nobility.
+
+No State shall, without the Consent of the Congress, lay any Imposts or
+Duties on Imports or Exports, except what may be absolutely necessary for
+executing it's inspection Laws: and the net Produce of all Duties and
+Imposts, laid by any State on Imports or Exports, shall be for the Use of the
+Treasury of the United States; and all such Laws shall be subject to the
+Revision and Controul of the Congress.
+
+No State shall, without the Consent of Congress, lay any duty of Tonnage,
+keep Troops, or Ships of War in time of Peace, enter into any Agreement or
+Compact with another State, or with a foreign Power, or engage in War, unless
+actually invaded, or in such imminent Danger as will not admit of delay.
+
diff --git a/perl/sample/us_constitution/art1sec2.txt b/perl/sample/us_constitution/art1sec2.txt
new file mode 100644
index 0000000..d3bf23c
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec2.txt
@@ -0,0 +1,35 @@
+Article I Section 2
+
+The House of Representatives shall be composed of Members chosen every second
+Year by the People of the several States, and the Electors in each State
+shall have the Qualifications requisite for Electors of the most numerous
+Branch of the State Legislature.
+
+No Person shall be a Representative who shall not have attained to the Age of
+twenty five Years, and been seven Years a Citizen of the United States, and
+who shall not, when elected, be an Inhabitant of that State in which he shall
+be chosen.
+
+Representatives and direct Taxes shall be apportioned among the several
+States which may be included within this Union, according to their respective
+Numbers, which shall be determined by adding to the whole Number of free
+Persons, including those bound to Service for a Term of Years, and excluding
+Indians not taxed, three fifths of all other Persons.
+
+The actual Enumeration shall be made within three Years after the first
+Meeting of the Congress of the United States, and within every subsequent
+Term of ten Years, in such Manner as they shall by Law direct. The Number of
+Representatives shall not exceed one for every thirty Thousand, but each
+State shall have at Least one Representative; and until such enumeration
+shall be made, the State of New Hampshire shall be entitled to chuse three,
+Massachusetts eight, Rhode Island and Providence Plantations one, Connecticut
+five, New York six, New Jersey four, Pennsylvania eight, Delaware one,
+Maryland six, Virginia ten, North Carolina five, South Carolina five and
+Georgia three.
+
+When vacancies happen in the Representation from any State, the Executive
+Authority thereof shall issue Writs of Election to fill such Vacancies.
+
+The House of Representatives shall chuse their Speaker and other Officers;
+and shall have the sole Power of Impeachment.
+
diff --git a/perl/sample/us_constitution/art1sec3.txt b/perl/sample/us_constitution/art1sec3.txt
new file mode 100644
index 0000000..3e2f4cc
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec3.txt
@@ -0,0 +1,39 @@
+Article I Section 3
+
+The Senate of the United States shall be composed of two Senators from each
+State, chosen by the Legislature thereof, for six Years; and each Senator
+shall have one Vote.
+
+Immediately after they shall be assembled in Consequence of the first
+Election, they shall be divided as equally as may be into three Classes. The
+Seats of the Senators of the first Class shall be vacated at the Expiration
+of the second Year, of the second Class at the Expiration of the fourth Year,
+and of the third Class at the Expiration of the sixth Year, so that one third
+may be chosen every second Year; and if Vacancies happen by Resignation, or
+otherwise, during the Recess of the Legislature of any State, the Executive
+thereof may make temporary Appointments until the next Meeting of the
+Legislature, which shall then fill such Vacancies.
+
+No person shall be a Senator who shall not have attained to the Age of thirty
+Years, and been nine Years a Citizen of the United States, and who shall not,
+when elected, be an Inhabitant of that State for which he shall be chosen.
+
+The Vice President of the United States shall be President of the Senate, but
+shall have no Vote, unless they be equally divided.
+
+The Senate shall chuse their other Officers, and also a President pro
+tempore, in the absence of the Vice President, or when he shall exercise the
+Office of President of the United States.
+
+The Senate shall have the sole Power to try all Impeachments. When sitting
+for that Purpose, they shall be on Oath or Affirmation. When the President of
+the United States is tried, the Chief Justice shall preside: And no Person
+shall be convicted without the Concurrence of two thirds of the Members
+present.
+
+Judgment in Cases of Impeachment shall not extend further than to removal
+from Office, and disqualification to hold and enjoy any Office of honor,
+Trust or Profit under the United States: but the Party convicted shall
+nevertheless be liable and subject to Indictment, Trial, Judgment and
+Punishment, according to Law.
+
diff --git a/perl/sample/us_constitution/art1sec4.txt b/perl/sample/us_constitution/art1sec4.txt
new file mode 100644
index 0000000..967f7ff
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec4.txt
@@ -0,0 +1,11 @@
+Article I Section 4
+
+The Times, Places and Manner of holding Elections for Senators and
+Representatives, shall be prescribed in each State by the Legislature
+thereof; but the Congress may at any time by Law make or alter such
+Regulations, except as to the Place of Chusing Senators.
+
+The Congress shall assemble at least once in every Year, and such Meeting
+shall be on the first Monday in December, unless they shall by Law appoint a
+different Day.
+
diff --git a/perl/sample/us_constitution/art1sec5.txt b/perl/sample/us_constitution/art1sec5.txt
new file mode 100644
index 0000000..5190f1e
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec5.txt
@@ -0,0 +1,21 @@
+Article I Section 5
+
+Each House shall be the Judge of the Elections, Returns and Qualifications of
+its own Members, and a Majority of each shall constitute a Quorum to do
+Business; but a smaller number may adjourn from day to day, and may be
+authorized to compel the Attendance of absent Members, in such Manner, and
+under such Penalties as each House may provide.
+
+Each House may determine the Rules of its Proceedings, punish its Members for
+disorderly Behavior, and, with the Concurrence of two-thirds, expel a Member.
+
+Each House shall keep a Journal of its Proceedings, and from time to time
+publish the same, excepting such Parts as may in their Judgment require
+Secrecy; and the Yeas and Nays of the Members of either House on any question
+shall, at the Desire of one fifth of those Present, be entered on the
+Journal.
+
+Neither House, during the Session of Congress, shall, without the Consent of
+the other, adjourn for more than three days, nor to any other Place than that
+in which the two Houses shall be sitting.
+
diff --git a/perl/sample/us_constitution/art1sec6.txt b/perl/sample/us_constitution/art1sec6.txt
new file mode 100644
index 0000000..2351866
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec6.txt
@@ -0,0 +1,16 @@
+Article I Section 6
+
+The Senators and Representatives shall receive a Compensation for their
+Services, to be ascertained by Law, and paid out of the Treasury of the
+United States. They shall in all Cases, except Treason, Felony and Breach of
+the Peace, be privileged from Arrest during their Attendance at the Session
+of their respective Houses, and in going to and returning from the same; and
+for any Speech or Debate in either House, they shall not be questioned in any
+other Place.
+
+No Senator or Representative shall, during the Time for which he was elected,
+be appointed to any civil Office under the Authority of the United States
+which shall have been created, or the Emoluments whereof shall have been
+increased during such time; and no Person holding any Office under the United
+States, shall be a Member of either House during his Continuance in Office.
+
diff --git a/perl/sample/us_constitution/art1sec7.txt b/perl/sample/us_constitution/art1sec7.txt
new file mode 100644
index 0000000..e7a9444
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec7.txt
@@ -0,0 +1,31 @@
+Article I Section 7
+
+All bills for raising Revenue shall originate in the House of
+Representatives; but the Senate may propose or concur with Amendments as on
+other Bills.
+
+Every Bill which shall have passed the House of Representatives and the
+Senate, shall, before it become a Law, be presented to the President of the
+United States; If he approve he shall sign it, but if not he shall return it,
+with his Objections to that House in which it shall have originated, who
+shall enter the Objections at large on their Journal, and proceed to
+reconsider it. If after such Reconsideration two thirds of that House shall
+agree to pass the Bill, it shall be sent, together with the Objections, to
+the other House, by which it shall likewise be reconsidered, and if approved
+by two thirds of that House, it shall become a Law. But in all such Cases the
+Votes of both Houses shall be determined by Yeas and Nays, and the Names of
+the Persons voting for and against the Bill shall be entered on the Journal
+of each House respectively. If any Bill shall not be returned by the
+President within ten Days (Sundays excepted) after it shall have been
+presented to him, the Same shall be a Law, in like Manner as if he had signed
+it, unless the Congress by their Adjournment prevent its Return, in which
+Case it shall not be a Law.
+
+Every Order, Resolution, or Vote to which the Concurrence of the Senate and
+House of Representatives may be necessary (except on a question of
+Adjournment) shall be presented to the President of the United States; and
+before the Same shall take Effect, shall be approved by him, or being
+disapproved by him, shall be repassed by two thirds of the Senate and House
+of Representatives, according to the Rules and Limitations prescribed in the
+Case of a Bill.
+
diff --git a/perl/sample/us_constitution/art1sec8.txt b/perl/sample/us_constitution/art1sec8.txt
new file mode 100644
index 0000000..d8dd374
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec8.txt
@@ -0,0 +1,64 @@
+Article I Section 8
+
+The Congress shall have Power To lay and collect Taxes, Duties, Imposts and
+Excises, to pay the Debts and provide for the common Defence and general
+Welfare of the United States; but all Duties, Imposts and Excises shall be
+uniform throughout the United States;
+
+To borrow money on the credit of the United States;
+
+To regulate Commerce with foreign Nations, and among the several States, and
+with the Indian Tribes;
+
+To establish an uniform Rule of Naturalization, and uniform Laws on the
+subject of Bankruptcies throughout the United States;
+
+To coin Money, regulate the Value thereof, and of foreign Coin, and fix the
+Standard of Weights and Measures;
+
+To provide for the Punishment of counterfeiting the Securities and current
+Coin of the United States;
+
+To establish Post Offices and Post Roads;
+
+To promote the Progress of Science and useful Arts, by securing for limited
+Times to Authors and Inventors the exclusive Right to their respective
+Writings and Discoveries;
+
+To constitute Tribunals inferior to the supreme Court;
+
+To define and punish Piracies and Felonies committed on the high Seas, and
+Offenses against the Law of Nations;
+
+To declare War, grant Letters of Marque and Reprisal, and make Rules
+concerning Captures on Land and Water;
+
+To raise and support Armies, but no Appropriation of Money to that Use shall
+be for a longer Term than two Years;
+
+To provide and maintain a Navy;
+
+To make Rules for the Government and Regulation of the land and naval Forces;
+
+To provide for calling forth the Militia to execute the Laws of the Union,
+suppress Insurrections and repel Invasions;
+
+To provide for organizing, arming, and disciplining the Militia, and for
+governing such Part of them as may be employed in the Service of the United
+States, reserving to the States respectively, the Appointment of the
+Officers, and the Authority of training the Militia according to the
+discipline prescribed by Congress;
+
+To exercise exclusive Legislation in all Cases whatsoever, over such District
+(not exceeding ten Miles square) as may, by Cession of particular States, and
+the acceptance of Congress, become the Seat of the Government of the United
+States, and to exercise like Authority over all Places purchased by the
+Consent of the Legislature of the State in which the Same shall be, for the
+Erection of Forts, Magazines, Arsenals, dock-Yards, and other needful
+Buildings; And
+
+To make all Laws which shall be necessary and proper for carrying into
+Execution the foregoing Powers, and all other Powers vested by this
+Constitution in the Government of the United States, or in any Department or
+Officer thereof.
+
diff --git a/perl/sample/us_constitution/art1sec9.txt b/perl/sample/us_constitution/art1sec9.txt
new file mode 100644
index 0000000..029cbb8
--- /dev/null
+++ b/perl/sample/us_constitution/art1sec9.txt
@@ -0,0 +1,30 @@
+Article I Section 9
+
+The Migration or Importation of such Persons as any of the States now
+existing shall think proper to admit, shall not be prohibited by the Congress
+prior to the Year one thousand eight hundred and eight, but a tax or duty may
+be imposed on such Importation, not exceeding ten dollars for each Person.
+
+The privilege of the Writ of Habeas Corpus shall not be suspended, unless
+when in Cases of Rebellion or Invasion the public Safety may require it.
+
+No Bill of Attainder or ex post facto Law shall be passed. No capitation, or
+other direct, Tax shall be laid, unless in Proportion to the Census or
+Enumeration herein before directed to be taken.
+
+No Tax or Duty shall be laid on Articles exported from any State.
+
+No Preference shall be given by any Regulation of Commerce or Revenue to the
+Ports of one State over those of another: nor shall Vessels bound to, or
+from, one State, be obliged to enter, clear, or pay Duties in another.
+
+No Money shall be drawn from the Treasury, but in Consequence of
+Appropriations made by Law; and a regular Statement and Account of the
+Receipts and Expenditures of all public Money shall be published from time to
+time.
+
+No Title of Nobility shall be granted by the United States: And no Person
+holding any Office of Profit or Trust under them, shall, without the Consent
+of the Congress, accept of any present, Emolument, Office, or Title, of any
+kind whatever, from any King, Prince or foreign State.
+
diff --git a/perl/sample/us_constitution/art2sec1.txt b/perl/sample/us_constitution/art2sec1.txt
new file mode 100644
index 0000000..66ee6ad
--- /dev/null
+++ b/perl/sample/us_constitution/art2sec1.txt
@@ -0,0 +1,65 @@
+Article II Section 1
+
+The executive Power shall be vested in a President of the United States of
+America. He shall hold his Office during the Term of four Years, and,
+together with the Vice-President chosen for the same Term, be elected, as
+follows:
+
+Each State shall appoint, in such Manner as the Legislature thereof may
+direct, a Number of Electors, equal to the whole Number of Senators and
+Representatives to which the State may be entitled in the Congress: but no
+Senator or Representative, or Person holding an Office of Trust or Profit
+under the United States, shall be appointed an Elector.
+
+The Electors shall meet in their respective States, and vote by Ballot for
+two persons, of whom one at least shall not lie an Inhabitant of the same
+State with themselves. And they shall make a List of all the Persons voted
+for, and of the Number of Votes for each; which List they shall sign and
+certify, and transmit sealed to the Seat of the Government of the United
+States, directed to the President of the Senate. The President of the Senate
+shall, in the Presence of the Senate and House of Representatives, open all
+the Certificates, and the Votes shall then be counted. The Person having the
+greatest Number of Votes shall be the President, if such Number be a Majority
+of the whole Number of Electors appointed; and if there be more than one who
+have such Majority, and have an equal Number of Votes, then the House of
+Representatives shall immediately chuse by Ballot one of them for President;
+and if no Person have a Majority, then from the five highest on the List the
+said House shall in like Manner chuse the President. But in chusing the
+President, the Votes shall be taken by States, the Representation from each
+State having one Vote; a quorum for this Purpose shall consist of a Member or
+Members from two-thirds of the States, and a Majority of all the States shall
+be necessary to a Choice. In every Case, after the Choice of the President,
+the Person having the greatest Number of Votes of the Electors shall be the
+Vice President. But if there should remain two or more who have equal Votes,
+the Senate shall chuse from them by Ballot the Vice-President.
+
+The Congress may determine the Time of chusing the Electors, and the Day on
+which they shall give their Votes; which Day shall be the same throughout the
+United States.
+
+No person except a natural born Citizen, or a Citizen of the United States,
+at the time of the Adoption of this Constitution, shall be eligible to the
+Office of President; neither shall any Person be eligible to that Office who
+shall not have attained to the Age of thirty-five Years, and been fourteen
+Years a Resident within the United States.
+
+In Case of the Removal of the President from Office, or of his Death,
+Resignation, or Inability to discharge the Powers and Duties of the said
+Office, the same shall devolve on the Vice President, and the Congress may by
+Law provide for the Case of Removal, Death, Resignation or Inability, both of
+the President and Vice President, declaring what Officer shall then act as
+President, and such Officer shall act accordingly, until the Disability be
+removed, or a President shall be elected.
+
+The President shall, at stated Times, receive for his Services, a
+Compensation, which shall neither be increased nor diminished during the
+Period for which he shall have been elected, and he shall not receive within
+that Period any other Emolument from the United States, or any of them.
+
+Before he enter on the Execution of his Office, he shall take the following
+Oath or Affirmation:
+
+"I do solemnly swear (or affirm) that I will faithfully execute the Office of
+President of the United States, and will to the best of my Ability, preserve,
+protect and defend the Constitution of the United States."
+
diff --git a/perl/sample/us_constitution/art2sec2.txt b/perl/sample/us_constitution/art2sec2.txt
new file mode 100644
index 0000000..753a370
--- /dev/null
+++ b/perl/sample/us_constitution/art2sec2.txt
@@ -0,0 +1,24 @@
+Article II Section 2
+
+The President shall be Commander in Chief of the Army and Navy of the United
+States, and of the Militia of the several States, when called into the actual
+Service of the United States; he may require the Opinion, in writing, of the
+principal Officer in each of the executive Departments, upon any subject
+relating to the Duties of their respective Offices, and he shall have Power
+to Grant Reprieves and Pardons for Offenses against the United States, except
+in Cases of Impeachment.
+
+He shall have Power, by and with the Advice and Consent of the Senate, to
+make Treaties, provided two thirds of the Senators present concur; and he
+shall nominate, and by and with the Advice and Consent of the Senate, shall
+appoint Ambassadors, other public Ministers and Consuls, Judges of the
+supreme Court, and all other Officers of the United States, whose
+Appointments are not herein otherwise provided for, and which shall be
+established by Law: but the Congress may by Law vest the Appointment of such
+inferior Officers, as they think proper, in the President alone, in the
+Courts of Law, or in the Heads of Departments.
+
+The President shall have Power to fill up all Vacancies that may happen
+during the Recess of the Senate, by granting Commissions which shall expire
+at the End of their next Session.
+
diff --git a/perl/sample/us_constitution/art2sec3.txt b/perl/sample/us_constitution/art2sec3.txt
new file mode 100644
index 0000000..c951a85
--- /dev/null
+++ b/perl/sample/us_constitution/art2sec3.txt
@@ -0,0 +1,11 @@
+Article II Section 3
+
+He shall from time to time give to the Congress Information of the State of
+the Union, and recommend to their Consideration such Measures as he shall
+judge necessary and expedient; he may, on extraordinary Occasions, convene
+both Houses, or either of them, and in Case of Disagreement between them,
+with Respect to the Time of Adjournment, he may adjourn them to such Time as
+he shall think proper; he shall receive Ambassadors and other public
+Ministers; he shall take Care that the Laws be faithfully executed, and shall
+Commission all the Officers of the United States.
+
diff --git a/perl/sample/us_constitution/art2sec4.txt b/perl/sample/us_constitution/art2sec4.txt
new file mode 100644
index 0000000..60f947f
--- /dev/null
+++ b/perl/sample/us_constitution/art2sec4.txt
@@ -0,0 +1,6 @@
+Article II Section 4
+
+The President, Vice President and all civil Officers of the United States,
+shall be removed from Office on Impeachment for, and Conviction of, Treason,
+Bribery, or other high Crimes and Misdemeanors.
+
diff --git a/perl/sample/us_constitution/art3sec1.txt b/perl/sample/us_constitution/art3sec1.txt
new file mode 100644
index 0000000..dead4fa
--- /dev/null
+++ b/perl/sample/us_constitution/art3sec1.txt
@@ -0,0 +1,9 @@
+Article III Section 1
+
+The judicial Power of the United States, shall be vested in one supreme
+Court, and in such inferior Courts as the Congress may from time to time
+ordain and establish. The Judges, both of the supreme and inferior Courts,
+shall hold their Offices during good Behavior, and shall, at stated Times,
+receive for their Services a Compensation which shall not be diminished
+during their Continuance in Office.
+
diff --git a/perl/sample/us_constitution/art3sec2.txt b/perl/sample/us_constitution/art3sec2.txt
new file mode 100644
index 0000000..3f4f942
--- /dev/null
+++ b/perl/sample/us_constitution/art3sec2.txt
@@ -0,0 +1,24 @@
+Article III Section 2
+
+The judicial Power shall extend to all Cases, in Law and Equity, arising
+under this Constitution, the Laws of the United States, and Treaties made, or
+which shall be made, under their Authority; to all Cases affecting
+Ambassadors, other public Ministers and Consuls; to all Cases of admiralty
+and maritime Jurisdiction; to Controversies to which the United States shall
+be a Party; to Controversies between two or more States; between a State and
+Citizens of another State; between Citizens of different States; between
+Citizens of the same State claiming Lands under Grants of different States,
+and between a State, or the Citizens thereof, and foreign States, Citizens or
+Subjects.
+
+In all Cases affecting Ambassadors, other public Ministers and Consuls, and
+those in which a State shall be Party, the supreme Court shall have original
+Jurisdiction. In all the other Cases before mentioned, the supreme Court
+shall have appellate Jurisdiction, both as to Law and Fact, with such
+Exceptions, and under such Regulations as the Congress shall make.
+
+Trial of all Crimes, except in Cases of Impeachment, shall be by Jury; and
+such Trial shall be held in the State where the said Crimes shall have been
+committed; but when not committed within any State, the Trial shall be at
+such Place or Places as the Congress may by Law have directed.
+
diff --git a/perl/sample/us_constitution/art3sec3.txt b/perl/sample/us_constitution/art3sec3.txt
new file mode 100644
index 0000000..2625150
--- /dev/null
+++ b/perl/sample/us_constitution/art3sec3.txt
@@ -0,0 +1,11 @@
+Article III Section 3
+
+Treason against the United States, shall consist only in levying War against
+them, or in adhering to their Enemies, giving them Aid and Comfort. No Person
+shall be convicted of Treason unless on the Testimony of two Witnesses to the
+same overt Act, or on Confession in open Court.
+
+The Congress shall have power to declare the Punishment of Treason, but no
+Attainder of Treason shall work Corruption of Blood, or Forfeiture except
+during the Life of the Person attainted.
+
diff --git a/perl/sample/us_constitution/art4sec1.txt b/perl/sample/us_constitution/art4sec1.txt
new file mode 100644
index 0000000..f50c27c
--- /dev/null
+++ b/perl/sample/us_constitution/art4sec1.txt
@@ -0,0 +1,7 @@
+Article IV Section 1
+
+Full Faith and Credit shall be given in each State to the public Acts,
+Records, and judicial Proceedings of every other State. And the Congress may
+by general Laws prescribe the Manner in which such Acts, Records and
+Proceedings shall be proved, and the Effect thereof.
+
diff --git a/perl/sample/us_constitution/art4sec2.txt b/perl/sample/us_constitution/art4sec2.txt
new file mode 100644
index 0000000..00af0da
--- /dev/null
+++ b/perl/sample/us_constitution/art4sec2.txt
@@ -0,0 +1,15 @@
+Article IV Section 2
+
+The Citizens of each State shall be entitled to all Privileges and Immunities
+of Citizens in the several States.
+
+A Person charged in any State with Treason, Felony, or other Crime, who shall
+flee from Justice, and be found in another State, shall on demand of the
+executive Authority of the State from which he fled, be delivered up, to be
+removed to the State having Jurisdiction of the Crime.
+
+No Person held to Service or Labour in one State, under the Laws thereof,
+escaping into another, shall, in Consequence of any Law or Regulation
+therein, be discharged from such Service or Labour, But shall be delivered up
+on Claim of the Party to whom such Service or Labour may be due.
+
diff --git a/perl/sample/us_constitution/art4sec3.txt b/perl/sample/us_constitution/art4sec3.txt
new file mode 100644
index 0000000..0c5bf9d
--- /dev/null
+++ b/perl/sample/us_constitution/art4sec3.txt
@@ -0,0 +1,13 @@
+Article IV Section 3
+
+New States may be admitted by the Congress into this Union; but no new States
+shall be formed or erected within the Jurisdiction of any other State; nor
+any State be formed by the Junction of two or more States, or parts of
+States, without the Consent of the Legislatures of the States concerned as
+well as of the Congress.
+
+The Congress shall have Power to dispose of and make all needful Rules and
+Regulations respecting the Territory or other Property belonging to the
+United States; and nothing in this Constitution shall be so construed as to
+Prejudice any Claims of the United States, or of any particular State.
+
diff --git a/perl/sample/us_constitution/art4sec4.txt b/perl/sample/us_constitution/art4sec4.txt
new file mode 100644
index 0000000..bd06b25
--- /dev/null
+++ b/perl/sample/us_constitution/art4sec4.txt
@@ -0,0 +1,7 @@
+Article IV Section 4
+
+The United States shall guarantee to every State in this Union a Republican
+Form of Government, and shall protect each of them against Invasion; and on
+Application of the Legislature, or of the Executive (when the Legislature
+cannot be convened) against domestic Violence.
+
diff --git a/perl/sample/us_constitution/art5.txt b/perl/sample/us_constitution/art5.txt
new file mode 100644
index 0000000..66cc4aa
--- /dev/null
+++ b/perl/sample/us_constitution/art5.txt
@@ -0,0 +1,14 @@
+Article V
+
+The Congress, whenever two thirds of both Houses shall deem it necessary,
+shall propose Amendments to this Constitution, or, on the Application of the
+Legislatures of two thirds of the several States, shall call a Convention for
+proposing Amendments, which, in either Case, shall be valid to all Intents
+and Purposes, as part of this Constitution, when ratified by the Legislatures
+of three fourths of the several States, or by Conventions in three fourths
+thereof, as the one or the other Mode of Ratification may be proposed by the
+Congress; Provided that no Amendment which may be made prior to the Year One
+thousand eight hundred and eight shall in any Manner affect the first and
+fourth Clauses in the Ninth Section of the first Article; and that no State,
+without its Consent, shall be deprived of its equal Suffrage in the Senate.
+
diff --git a/perl/sample/us_constitution/art6.txt b/perl/sample/us_constitution/art6.txt
new file mode 100644
index 0000000..463674a
--- /dev/null
+++ b/perl/sample/us_constitution/art6.txt
@@ -0,0 +1,19 @@
+Article VI
+
+All Debts contracted and Engagements entered into, before the Adoption of
+this Constitution, shall be as valid against the United States under this
+Constitution, as under the Confederation.
+
+This Constitution, and the Laws of the United States which shall be made in
+Pursuance thereof; and all Treaties made, or which shall be made, under the
+Authority of the United States, shall be the supreme Law of the Land; and the
+Judges in every State shall be bound thereby, any Thing in the Constitution
+or Laws of any State to the Contrary notwithstanding.
+
+The Senators and Representatives before mentioned, and the Members of the
+several State Legislatures, and all executive and judicial Officers, both of
+the United States and of the several States, shall be bound by Oath or
+Affirmation, to support this Constitution; but no religious Test shall ever
+be required as a Qualification to any Office or public Trust under the United
+States.
+
diff --git a/perl/sample/us_constitution/art7.txt b/perl/sample/us_constitution/art7.txt
new file mode 100644
index 0000000..1f53ef3
--- /dev/null
+++ b/perl/sample/us_constitution/art7.txt
@@ -0,0 +1,43 @@
+Article VII
+
+The Ratification of the Conventions of nine States, shall be sufficient for
+the Establishment of this Constitution between the States so ratifying the
+Same.
+
+Done in Convention by the Unanimous Consent of the States present the
+Seventeenth Day of September in the Year of our Lord one thousand seven
+hundred and Eighty seven and of the Independence of the United States of
+America the Twelfth. In Witness whereof We have hereunto subscribed our
+Names.
+
+Go Washington - President and deputy from Virginia
+
+New Hampshire - John Langdon, Nicholas Gilman
+
+Massachusetts - Nathaniel Gorham, Rufus King
+
+Connecticut - Wm Saml Johnson, Roger Sherman
+
+New York - Alexander Hamilton
+
+New Jersey - Wil Livingston, David Brearley, Wm Paterson, Jona. Dayton
+
+Pensylvania - B Franklin, Thomas Mifflin, Robt Morris, Geo. Clymer, Thos
+FitzSimons, Jared Ingersoll, James Wilson, Gouv Morris
+
+Delaware - Geo. Read, Gunning Bedford jun, John Dickinson, Richard Bassett,
+Jaco. Broom
+
+Maryland - James McHenry, Dan of St Tho Jenifer, Danl Carroll
+
+Virginia - John Blair, James Madison Jr.
+
+North Carolina - Wm Blount, Richd Dobbs Spaight, Hu Williamson
+
+South Carolina - J. Rutledge, Charles Cotesworth Pinckney, Charles Pinckney,
+Pierce Butler
+
+Georgia - William Few, Abr Baldwin
+
+Attest: William Jackson, Secretary
+
diff --git a/perl/sample/us_constitution/index.html b/perl/sample/us_constitution/index.html
new file mode 100644
index 0000000..16948b0
--- /dev/null
+++ b/perl/sample/us_constitution/index.html
@@ -0,0 +1,114 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
+
+<html>
+
+ <head>
+ <meta http-equiv="content-type" content="text/html;charset=UTF-8">
+ <link rel="stylesheet" type="text/css" href="uscon.css">
+ <title>US Constitution</title>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+ </head>
+
+ <body>
+ <div id="navigation">
+ US Constitution
+ </div><!--navigation-->
+
+ <div id="bodytext">
+ <ul>
+ <li><a href="preamble.txt">Preamble</a></li>
+
+ <li><a name="articles"></a>Article I</li>
+ <ul>
+ <li><a href="art1sec1.txt">Article I Section 1</a></li>
+ <li><a href="art1sec2.txt">Article I Section 2</a></li>
+ <li><a href="art1sec3.txt">Article I Section 3</a></li>
+ <li><a href="art1sec4.txt">Article I Section 4</a></li>
+ <li><a href="art1sec5.txt">Article I Section 5</a></li>
+ <li><a href="art1sec6.txt">Article I Section 6</a></li>
+ <li><a href="art1sec7.txt">Article I Section 7</a></li>
+ <li><a href="art1sec8.txt">Article I Section 8</a></li>
+ <li><a href="art1sec9.txt">Article I Section 9</a></li>
+ <li><a href="art1sec10.txt">Article I Section 10</a></li>
+ </ul>
+
+ <li>Article II</li>
+ <ul>
+ <li><a href="art2sec1.txt">Article II Section 1</a></li>
+ <li><a href="art2sec2.txt">Article II Section 2</a></li>
+ <li><a href="art2sec3.txt">Article II Section 3</a></li>
+ <li><a href="art2sec4.txt">Article II Section 4</a></li>
+ </ul>
+
+ <li>Article III</li>
+ <ul>
+ <li><a href="art3sec1.txt">Article III Section 1</a></li>
+ <li><a href="art3sec2.txt">Article III Section 2</a></li>
+ <li><a href="art3sec3.txt">Article III Section 3</a></li>
+ </ul>
+
+ <li>Article IV</li>
+ <ul>
+ <li><a href="art4sec1.txt">Article IV Section 1</a></li>
+ <li><a href="art4sec2.txt">Article IV Section 2</a></li>
+ <li><a href="art4sec3.txt">Article IV Section 3</a></li>
+ <li><a href="art4sec4.txt">Article IV Section 4</a></li>
+ </ul>
+
+ <li><a href="art5.txt">Article V</a></li>
+
+ <li><a href="art6.txt">Article VI</a></li>
+
+ <li><a href="art7.txt">Article VII</a></li>
+
+ <li><a name="amendments"></a>Amendments</li>
+ <ul>
+ <li><a href="amend1.txt">Amendment 1</a></li>
+ <li><a href="amend2.txt">Amendment 2</a></li>
+ <li><a href="amend3.txt">Amendment 3</a></li>
+ <li><a href="amend4.txt">Amendment 4</a></li>
+ <li><a href="amend5.txt">Amendment 5</a></li>
+ <li><a href="amend6.txt">Amendment 6</a></li>
+ <li><a href="amend7.txt">Amendment 7</a></li>
+ <li><a href="amend8.txt">Amendment 8</a></li>
+ <li><a href="amend9.txt">Amendment 9</a></li>
+ <li><a href="amend10.txt">Amendment 10</a></li>
+ <li><a href="amend11.txt">Amendment 11</a></li>
+ <li><a href="amend12.txt">Amendment 12</a></li>
+ <li><a href="amend13.txt">Amendment 13</a></li>
+ <li><a href="amend14.txt">Amendment 14</a></li>
+ <li><a href="amend15.txt">Amendment 15</a></li>
+ <li><a href="amend16.txt">Amendment 16</a></li>
+ <li><a href="amend17.txt">Amendment 17</a></li>
+ <li><a href="amend18.txt">Amendment 18</a></li>
+ <li><a href="amend19.txt">Amendment 19</a></li>
+ <li><a href="amend20.txt">Amendment 20</a></li>
+ <li><a href="amend21.txt">Amendment 21</a></li>
+ <li><a href="amend22.txt">Amendment 22</a></li>
+ <li><a href="amend23.txt">Amendment 23</a></li>
+ <li><a href="amend24.txt">Amendment 24</a></li>
+ <li><a href="amend25.txt">Amendment 25</a></li>
+ <li><a href="amend26.txt">Amendment 26</a></li>
+ <li><a href="amend27.txt">Amendment 27</a></li>
+ </ul>
+ </ul>
+ </div><!--bodytext-->
+ </body>
+
+</html>
+
diff --git a/perl/sample/us_constitution/preamble.txt b/perl/sample/us_constitution/preamble.txt
new file mode 100644
index 0000000..ee1bffe
--- /dev/null
+++ b/perl/sample/us_constitution/preamble.txt
@@ -0,0 +1,8 @@
+Preamble
+
+We the People of the United States, in Order to form a more perfect Union,
+establish Justice, insure domestic Tranquility, provide for the common
+defence, promote the general Welfare, and secure the Blessings of Liberty to
+ourselves and our Posterity, do ordain and establish this Constitution for
+the United States of America.
+
diff --git a/perl/sample/us_constitution/uscon.css b/perl/sample/us_constitution/uscon.css
new file mode 100644
index 0000000..c674edd
--- /dev/null
+++ b/perl/sample/us_constitution/uscon.css
@@ -0,0 +1,45 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+body,table,textarea {
+ font-family: Arial, Helvetica, sans-serif;
+}
+
+body {
+ font-size: 90%;
+ background: #fff;
+ margin: 0 0 0 0;
+}
+
+div#navigation {
+ background: #ddfff6;
+ padding: 10px;
+ border-bottom: 1px solid #555;
+}
+
+div#bodytext {
+ margin: 10px 10px 10px 10px;
+}
+
+form#usconSearch {
+ display: inline;
+ margin-bottom: 0px;
+}
+
+span.excerptURL {
+ color: green;
+}
+
diff --git a/perl/t/001-build_indexes.t b/perl/t/001-build_indexes.t
new file mode 100644
index 0000000..bcdc94e
--- /dev/null
+++ b/perl/t/001-build_indexes.t
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 4;
+use File::Spec::Functions qw( catfile );
+use File::Find qw( find );
+use Lucy::Test::TestUtils qw(
+ working_dir
+ create_working_dir
+ remove_working_dir
+ create_uscon_index
+ persistent_test_index_loc
+);
+
+remove_working_dir();
+ok( !-e working_dir(), "Working dir doesn't exist" );
+create_working_dir();
+ok( -e working_dir(), "Working dir successfully created" );
+
+create_uscon_index();
+
+my $path = persistent_test_index_loc();
+ok( -d $path, "created index directory" );
+my $num_cfmeta = 0;
+find(
+ { no_chdir => 1,
+ wanted => sub { $num_cfmeta++ if $File::Find::name =~ /cfmeta/ },
+ },
+ $path
+);
+cmp_ok( $num_cfmeta, '>', 0, "at least one .cf file exists" );
+
diff --git a/perl/t/002-lucy.t b/perl/t/002-lucy.t
new file mode 100644
index 0000000..1df85ec
--- /dev/null
+++ b/perl/t/002-lucy.t
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More;
+use File::Find 'find';
+
+my @modules;
+
+# None for now -- until we remove a module.
+my %excluded = map { ( $_ => 1 ) } qw();
+
+find(
+ { no_chdir => 1,
+ wanted => sub {
+ return unless $File::Find::name =~ /\.pm$/;
+ push @modules, $File::Find::name;
+ }
+ },
+ 'lib'
+);
+
+plan( tests => scalar @modules );
+
+for (@modules) {
+ s/^.*?Lucy/Lucy/;
+ s/^.*?LucyX/LucyX/;
+ s/\.pm$//;
+ s/\W+/::/g;
+ if ( $excluded{$_} ) {
+ eval qq|use $_;|;
+ like( $@, qr/removed|replaced|renamed/i,
+ "Removed module '$_' throws error on load" );
+ }
+ else {
+ use_ok($_);
+ }
+}
+
diff --git a/perl/t/015-sort_external.t b/perl/t/015-sort_external.t
new file mode 100644
index 0000000..d5c2822
--- /dev/null
+++ b/perl/t/015-sort_external.t
@@ -0,0 +1,165 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 15;
+use List::Util qw( shuffle );
+use Lucy::Test;
+use bytes qw();
+
+my ( $sortex, $cache, @orig, @sort_output );
+
+$sortex = Lucy::Test::Util::BBSortEx->new( mem_thresh => 4 );
+$sortex->feed( new_bytebuf('c') );
+is( $sortex->cache_count, 1, "feed elem into cache" );
+
+$sortex->feed( new_bytebuf('b') );
+$sortex->feed( new_bytebuf('d') );
+$sortex->sort_cache;
+SKIP: {
+ skip( "Restore when porting test to C", 1 );
+ $cache = $sortex->_peek_cache;
+ is_deeply( $cache, [qw( b c d )], "sort cache" );
+}
+
+$sortex->feed( new_bytebuf('a') );
+is( $sortex->cache_count, 0,
+ "cache flushed automatically when mem_thresh crossed" );
+#is( $sortex->get_num_runs, 1, "run added" );
+
+my @bytebufs = map { new_bytebuf($_) } qw( x y z );
+my $run = Lucy::Test::Util::BBSortEx->new( external => \@bytebufs );
+$sortex->add_run($run);
+$sortex->flip;
+@orig = qw( a b c d x y z );
+while ( my $result = $sortex->fetch ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig, "Add_Run" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new( mem_thresh => 4 );
+$sortex->feed( new_bytebuf('c') );
+$sortex->clear_cache;
+is( $sortex->cache_count, 0, "Clear_Cache" );
+$sortex->feed( new_bytebuf('b') );
+$sortex->feed( new_bytebuf('a') );
+$sortex->flush;
+$sortex->flip;
+@orig = qw( a b );
+is( $sortex->peek, 'a', "Peek" );
+
+while ( defined( my $result = $sortex->fetch ) ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig,
+ "elements cleared via Clear_Cache truly cleared" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new;
+@orig = ( 'a' .. 'z' );
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->flip;
+while ( defined( my $result = $sortex->fetch ) ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig, "sort letters" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new;
+@orig = qw( a a a b c d x x x x x x y y );
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->flip;
+while ( defined( my $result = $sortex->fetch ) ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig, "sort repeated letters" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new;
+@orig = ( '', '', 'a' .. 'z' );
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->flip;
+while ( defined( my $result = $sortex->fetch ) ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig, "sort letters and empty strings" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new( mem_thresh => 30 );
+@orig = 'a' .. 'z';
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->flip;
+while ( defined( my $result = $sortex->fetch ) ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig, "... with an absurdly low mem_thresh" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new( mem_thresh => 1 );
+@orig = 'a' .. 'z';
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->flip;
+while ( defined( my $result = $sortex->fetch ) ) {
+ push @sort_output, $result;
+}
+is_deeply( \@sort_output, \@orig, "... with an even lower mem_thresh" );
+@orig = ();
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new;
+$sortex->flip;
+@sort_output = $sortex->fetch;
+is_deeply( \@sort_output, [undef], "Sorting nothing returns undef" );
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new( mem_thresh => 5_000 );
+@orig = map { pack( 'N', $_ ) } ( 0 .. 11_000 );
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+$sortex->flip;
+while ( defined( my $item = $sortex->fetch ) ) {
+ push @sort_output, $item;
+}
+is_deeply( \@sort_output, \@orig, "Sorting packed integers..." );
+@sort_output = ();
+
+$sortex = Lucy::Test::Util::BBSortEx->new( mem_thresh => 15_000 );
+@orig = ();
+for my $iter ( 0 .. 1_000 ) {
+ my $string = '';
+ for my $string_len ( 0 .. int( rand(1200) ) ) {
+ $string .= pack( 'C', int( rand(256) ) );
+ }
+ push @orig, $string;
+}
+$sortex->feed( new_bytebuf($_) ) for shuffle(@orig);
+@orig = sort @orig;
+$sortex->flip;
+while ( defined( my $item = $sortex->fetch ) ) {
+ push @sort_output, $item;
+}
+is_deeply( \@sort_output, \@orig, "Random binary strings of random length" );
+@sort_output = ();
+
+sub new_bytebuf { Lucy::Object::ByteBuf->new(shift) }
+
diff --git a/perl/t/018-host.t b/perl/t/018-host.t
new file mode 100644
index 0000000..aae101a
--- /dev/null
+++ b/perl/t/018-host.t
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 8;
+use Lucy::Test;
+use Lucy qw( to_perl to_clownfish );
+
+my $object = Lucy::Object::Host->new();
+isa_ok( $object, "Lucy::Object::Host" );
+
+is( $object->_callback, undef, "void callback" );
+is( $object->_callback_f64, 5, "f64 callback" );
+is( $object->_callback_i64, 5, "integer callback" );
+
+my $test_obj = $object->_callback_obj;
+isa_ok( $test_obj, "Lucy::Object::ByteBuf" );
+
+my %complex_data_structure = (
+ a => [ 1, 2, 3, { ooga => 'booga' } ],
+ b => { foo => 'foofoo', bar => 'barbar' },
+);
+my $kobj = to_clownfish( \%complex_data_structure );
+isa_ok( $kobj, 'Lucy::Object::Obj' );
+my $transformed = to_perl($kobj);
+is_deeply( $transformed, \%complex_data_structure,
+ "transform from Perl to Clownfish data structures and back" );
+
+my $bread_and_butter = Lucy::Object::Hash->new;
+$bread_and_butter->store( 'bread', Lucy::Object::ByteBuf->new('butter') );
+my $salt_and_pepper = Lucy::Object::Hash->new;
+$salt_and_pepper->store( 'salt', Lucy::Object::ByteBuf->new('pepper') );
+$complex_data_structure{c} = $bread_and_butter;
+$complex_data_structure{d} = $salt_and_pepper;
+$transformed = to_perl( to_clownfish( \%complex_data_structure ) );
+$complex_data_structure{c} = { bread => 'butter' };
+$complex_data_structure{d} = { salt => 'pepper' };
+is_deeply( $transformed, \%complex_data_structure,
+ "handle mixed data structure correctly" );
diff --git a/perl/t/021-vtable.t b/perl/t/021-vtable.t
new file mode 100644
index 0000000..5244a85
--- /dev/null
+++ b/perl/t/021-vtable.t
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MyHash;
+use base qw( Lucy::Object::Hash );
+
+sub oodle { }
+
+package RAMFolderOfDeath;
+use base qw( Lucy::Store::RAMFolder );
+
+sub open_in {
+ my ( $self, $filename ) = @_;
+ die "Sweet, sweet death.";
+}
+
+package OnceRemoved;
+use base qw( Lucy::Object::Obj );
+
+our $serialize_was_called = 0;
+sub serialize {
+ my ( $self, $outstream ) = @_;
+ $serialize_was_called++;
+ $self->SUPER::serialize($outstream);
+}
+
+package TwiceRemoved;
+use base qw( OnceRemoved );
+
+package main;
+
+use Lucy::Test;
+use Test::More tests => 9;
+use Storable qw( nfreeze );
+
+{
+ my $twice_removed = TwiceRemoved->new;
+ # This triggers a call to Obj_Serialize() via the VTable dispatch.
+ my $frozen = nfreeze($twice_removed);
+ ok( $serialize_was_called,
+ "Overridden method in intermediate class recognized" );
+ my $vtable = $twice_removed->get_vtable;
+ is( $vtable->get_name, "TwiceRemoved", "correct class" );
+ my $parent_vtable = $vtable->get_parent;
+ is( $parent_vtable->get_name, "OnceRemoved", "correct parent class" )
+}
+
+my $stringified;
+my $storage = Lucy::Object::Hash->new;
+
+{
+ my $subclassed_hash = MyHash->new;
+ $stringified = $subclassed_hash->to_string;
+
+ isa_ok( $subclassed_hash, "MyHash", "Perl isa reports correct subclass" );
+
+ # Store the subclassed object. At the end of this block, the Perl object
+ # will go out of scope and DESTROY will be called, but the Clownfish object
+ # will persist.
+ $storage->store( "test", $subclassed_hash );
+}
+
+my $resurrected = $storage->_fetch("test");
+
+isa_ok( $resurrected, "MyHash", "subclass name survived Perl destruction" );
+is( $resurrected->to_string, $stringified,
+ "It's the same Hash from earlier (though a different Perl object)" );
+
+my $booga = Lucy::Object::CharBuf->new("booga");
+$resurrected->store( "ooga", $booga );
+
+is( $resurrected->fetch("ooga"),
+ "booga", "subclassed object still performs correctly at the C level" );
+
+my $methods = Lucy::Object::VTable->novel_host_methods('MyHash');
+is_deeply( $methods->to_perl, ['oodle'], "novel_host_methods" );
+
+my $folder = RAMFolderOfDeath->new;
+eval { $folder->slurp_file('foo') }; # calls open_in, which dies per above.
+like( $@, qr/sweet/i, "override vtable method with pure perl method" );
diff --git a/perl/t/023-stepper.t b/perl/t/023-stepper.t
new file mode 100644
index 0000000..d5055cd
--- /dev/null
+++ b/perl/t/023-stepper.t
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MyStepper;
+use base qw( Lucy::Util::Stepper );
+
+our %number;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ $number{$$self} = 0;
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $number{$$self};
+ $self->SUPER::DESTROY;
+}
+
+sub get_number { $number{ ${ +shift } } }
+
+sub read_record {
+ my ( $self, $instream ) = @_;
+ $number{$$self} += $instream->read_c32;
+}
+
+package main;
+use Test::More tests => 1;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $outstream = $folder->open_out("foo") or die Lucy->error;
+$outstream->write_c32(10) for 1 .. 5;
+$outstream->close;
+my $instream = $folder->open_in("foo") or die Lucy->error;
+my $stepper = MyStepper->new;
+
+my @got;
+while ( $instream->tell < $instream->length ) {
+ $stepper->read_record($instream);
+ push @got, $stepper->get_number;
+}
+is_deeply( \@got, [ 10, 20, 30, 40, 50 ], 'Read_Record' );
+
diff --git a/perl/t/025-debug.t b/perl/t/025-debug.t
new file mode 100644
index 0000000..f195ae9
--- /dev/null
+++ b/perl/t/025-debug.t
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More;
+use File::Spec::Functions qw( catfile );
+use Fcntl;
+use Lucy::Util::Debug qw(
+ DEBUG_ENABLED
+ DEBUG_PRINT
+ DEBUG
+ ASSERT
+ set_env_cache
+);
+use Lucy::Test::TestUtils qw( working_dir );
+
+BEGIN {
+ if ( !DEBUG_ENABLED() ) {
+ plan( skip_all => 'DEBUG not enabled' );
+ }
+ elsif ( $ENV{LUCY_VALGRIND} ) {
+ plan( skip_all => 'Tests disabled under valgrind' );
+ }
+ else {
+ plan( tests => 7 );
+ }
+}
+
+my $stderr_dumpfile = catfile( working_dir(), 'lucy_garbage' );
+unlink $stderr_dumpfile;
+sysopen( STDERR, $stderr_dumpfile, O_CREAT | O_WRONLY | O_EXCL )
+ or die "Failed to redirect STDERR";
+
+DEBUG_PRINT("Roach Motel");
+like( slurp_file($stderr_dumpfile), qr/Roach Motel/, "DEBUG_PRINT" );
+
+ASSERT(1);
+pass("ASSERT(true) didn't die");
+
+SKIP: {
+ skip( "Windows fork not supported by Lucy", 3 )
+ if $^O =~ /(mswin|cygwin)/i;
+
+ my $stderr_out = capture_debug( 'Lucy.xs', 'Borax' );
+ like( $stderr_out, qr/Borax/, "DEBUG - file name" );
+
+ $stderr_out = capture_debug( 'XS_Lucy__Util__Debug_DEBUG', "Strychnine" );
+ like( $stderr_out, qr/Strychnine/, "DEBUG - function name" );
+
+ $stderr_out = capture_debug( 'Lucy*', 'Raid' );
+ like( $stderr_out, qr/Raid/, "DEBUG - wildcard" );
+
+ my $pid = fork();
+ if ( $pid == 0 ) { # child
+ ASSERT(0);
+ exit;
+ }
+ else {
+ waitpid( $pid, 0 );
+ like(
+ slurp_file($stderr_dumpfile),
+ qr/ASSERT FAILED/,
+ "failing ASSERT"
+ );
+ }
+}
+
+set_env_cache("");
+DEBUG("Slug and Snail Death");
+unlike( slurp_file($stderr_dumpfile), qr/Slug/, "DEBUG disabled by default" );
+
+# Clean up.
+unlink $stderr_dumpfile;
+
+sub capture_debug {
+ my ( $fake_env_var, $debug_string ) = @_;
+ my $pid = fork();
+ if ( $pid == 0 ) { # child
+ set_env_cache($fake_env_var);
+ DEBUG($debug_string);
+ exit;
+ }
+ else {
+ waitpid( $pid, 0 );
+ }
+ return slurp_file($stderr_dumpfile);
+}
+
+sub slurp_file {
+ my $path = shift;
+ open( my $fh, '<', $path ) or die "Can't open '$path' for reading: $!";
+ my $content = do { local $/; <$fh> };
+ return $content;
+}
diff --git a/perl/t/026-serialization.t b/perl/t/026-serialization.t
new file mode 100644
index 0000000..f1032d5
--- /dev/null
+++ b/perl/t/026-serialization.t
@@ -0,0 +1,106 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 7;
+
+package BasicObj;
+use base qw( Lucy::Object::Obj );
+
+package MyObj;
+use base qw( Lucy::Object::Obj );
+
+my %extra;
+
+sub get_extra { my $self = shift; $extra{$$self} }
+
+sub new {
+ my ( $class, $extra ) = @_;
+ my $self = $class->SUPER::new();
+ $extra{$$self} = $extra;
+ return $self;
+}
+
+sub serialize {
+ my ( $self, $outstream ) = @_;
+ $self->SUPER::serialize($outstream);
+ $outstream->write_string( $self->get_extra );
+}
+
+sub deserialize {
+ my ( $thing, $instream ) = @_;
+ my $self = $thing->SUPER::deserialize($instream);
+ $extra{$$self} = $instream->read_string;
+ return $self;
+}
+
+sub DESTROY {
+ my $self = shift;
+ delete $extra{$$self};
+ $self->SUPER::DESTROY;
+}
+
+package BadObj;
+use base qw( MyObj );
+
+sub deserialize {
+ return __PACKAGE__->new("illegal");
+}
+
+package main;
+use Storable qw( freeze thaw );
+use Lucy::Test;
+use Carp;
+
+my $obj = BasicObj->new;
+run_test_cycle( $obj, sub { ref( $_[0] ) } );
+
+my $subclassed_obj = MyObj->new("bar");
+run_test_cycle( $subclassed_obj, sub { shift->get_extra } );
+
+my $bb = Lucy::Object::ByteBuf->new("foo");
+run_test_cycle( $bb, sub { shift->to_perl } );
+
+SKIP: {
+ skip( "Invalid deserialization causes leaks", 1 ) if $ENV{LUCY_VALGRIND};
+ my $bad_obj = BadObj->new("Royale With Cheese");
+ eval {
+ run_test_cycle( $bad_obj, sub { ref( $_[0] ) } );
+ };
+ like( $@, qr/BadObj/i, "throw error with bad deserialize" );
+}
+
+sub run_test_cycle {
+ my ( $orig, $transform ) = @_;
+ my $class = ref($orig);
+
+ my $frozen = freeze($orig);
+ my $thawed = thaw($frozen);
+ is( $transform->($thawed), $transform->($orig), "$class: freeze/thaw" );
+
+ my $ram_file = Lucy::Store::RAMFile->new;
+ my $outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or confess Lucy->error;
+ $orig->serialize($outstream);
+ $outstream->close;
+ my $instream = Lucy::Store::InStream->open( file => $ram_file )
+ or confess Lucy->error;
+ my $deserialized = $class->deserialize($instream);
+
+ is( $transform->($deserialized),
+ $transform->($orig), "$class: call deserialize via class name" );
+}
diff --git a/perl/t/028-sortexrun.t b/perl/t/028-sortexrun.t
new file mode 100644
index 0000000..6508593
--- /dev/null
+++ b/perl/t/028-sortexrun.t
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More skip_all => 'Disabled until test ported to C';
+#use Test::More tests => 5;
+use Lucy::Test;
+use Lucy qw( to_perl );
+
+my $letters = Lucy::Object::VArray->new( capacity => 26 );
+$letters->push( Lucy::Object::ByteBuf->new($_) ) for 'a' .. 'z';
+my $run = Lucy::Test::Util::BBSortEx->new( external => $letters );
+$run->set_mem_thresh(5);
+
+my $num_in_cache = $run->refill;
+is( $run->cache_count, 5, "Read_Elem puts the brakes on Refill" );
+my $endpost = $run->peek_last;
+is( $endpost, 'e', "Peek_Last" );
+$endpost = Lucy::Object::ByteBuf->new('b');
+my $slice = $run->pop_slice($endpost);
+is( scalar @$slice, 2, "Pop_Slice gets only less-than-or-equal elems" );
+@$slice = map { to_perl($_) } @$slice;
+is_deeply( $slice, [qw( a b )], "Pop_Slice picks highest elems" );
+
+my @got = qw( a b );
+while (1) {
+ $endpost = $run->peek_last;
+ $slice = $run->pop_slice( Lucy::Object::ByteBuf->new($endpost) );
+ push @got, map { to_perl($_) } @$slice;
+ last unless $run->refill;
+}
+is_deeply( \@got, [ 'a' .. 'z' ], "retrieve all elems" );
+
diff --git a/perl/t/050-ramfile.t b/perl/t/050-ramfile.t
new file mode 100644
index 0000000..65817bd
--- /dev/null
+++ b/perl/t/050-ramfile.t
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 8;
+use Lucy::Test;
+
+my ( $ram_file, $outstream, $instream, $foo );
+
+$ram_file = Lucy::Store::RAMFile->new;
+$outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or die Lucy->error;
+$outstream->print("foo");
+$outstream->flush;
+is( $ram_file->get_contents, "foo", '$ramfile->get_contents' );
+
+my $long_string = 'a' x 5000;
+$outstream->print($long_string);
+$outstream->flush;
+
+is( $ram_file->get_contents, "foo$long_string",
+ "store a string spread out over several buffers" );
+
+$instream = Lucy::Store::InStream->open( file => $ram_file )
+ or die Lucy->error;
+$instream->read( $foo, 3 );
+is( $foo, 'foo', "instream reads ramfile properly" );
+
+my $long_dupe;
+$instream->read( $long_dupe, 5000 );
+is( $long_dupe, $long_string, "read string spread out over several buffers" );
+
+eval { my $blah; $instream->read( $blah, 3 ); };
+like( $@, qr/EOF/, "reading past EOF throws an error" );
+
+$ram_file = Lucy::Store::RAMFile->new;
+$outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or die Lucy->error;
+my $BUF_SIZE = Lucy::Store::FileHandle::_BUF_SIZE();
+my $rep_count = $BUF_SIZE - 1;
+$outstream->print( 'a' x $rep_count );
+$outstream->print('foo');
+$outstream->close;
+$instream = Lucy::Store::InStream->open( file => $ram_file )
+ or die Lucy->error;
+$instream->read( $long_dupe, $rep_count );
+undef $foo;
+$instream->read( $foo, 3 );
+is( $foo, 'foo', "read across buffer boundary " );
+
+$ram_file = Lucy::Store::RAMFile->new;
+$outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or die Lucy->error;
+$outstream->print( 'a' x 1024 );
+$outstream->print('foo');
+$outstream->close;
+
+$instream = Lucy::Store::InStream->open( file => $ram_file )
+ or die Lucy->error;
+$instream->seek(1024);
+undef $foo;
+$instream->read( $foo, 3 );
+is( $foo, 'foo', "InStream seek" );
+
+my $dupe = $instream->reopen(
+ filename => 'foo',
+ offset => 1023,
+ len => 4
+);
+undef $foo;
+$dupe->read( $foo, 4 );
+is( $foo, 'afoo', "reopened instream" );
+
diff --git a/perl/t/051-fsfile.t b/perl/t/051-fsfile.t
new file mode 100644
index 0000000..3228490
--- /dev/null
+++ b/perl/t/051-fsfile.t
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 5;
+use Carp;
+use File::Spec::Functions qw( tmpdir catdir catfile );
+use Lucy::Test::TestUtils qw( init_test_index_loc );
+
+sub slurp_file {
+ my $path = shift;
+ open( my $fh, '<', $path ) or confess "Couldn't open '$path': $!";
+ local $/;
+ return <$fh>;
+}
+
+my $dir = init_test_index_loc();
+my $filename = 'hogus_bogus';
+my $filepath = catfile( $dir, $filename );
+my ( $outstream, $instream );
+my $folder = Lucy::Store::FSFolder->new( path => $dir );
+my $foo;
+
+sub new_outstream {
+ undef $outstream;
+ unlink $filepath;
+ my $fh = Lucy::Store::FSFileHandle->open(
+ path => $filepath,
+ create => 1,
+ write_only => 1,
+ exclusive => 1,
+ );
+ my $outstream = Lucy::Store::OutStream->open( file => $fh )
+ or confess Lucy->error;
+ return $outstream;
+}
+
+sub new_instream {
+ undef $instream;
+ return $folder->open_in($filename) || confess Lucy->error;
+}
+
+$outstream = new_outstream();
+$outstream->print("foo");
+$outstream->close;
+$instream = new_instream();
+undef $foo;
+$instream->read( $foo, 3 );
+is( $foo, "foo", "outstream writes, instream reads" );
+$instream->close;
+
+my $long_string = 'a' x 5000;
+$outstream = new_outstream();
+$outstream->print( 'foo', $long_string );
+$outstream->close;
+$instream = new_instream();
+undef $foo;
+$instream->read( $foo, 5003 );
+is( $foo, "foo$long_string", "long string" );
+
+eval { my $blah; $instream->read( $blah, 2 ) };
+like( $@, qr/EOF/, "reading past EOF throws an error" );
+undef $instream;
+
+$outstream = new_outstream();
+$outstream->print( 'a' x 1024 );
+$outstream->print('foo');
+$outstream->close;
+
+$instream = new_instream();
+$instream->seek(1024);
+undef $foo;
+$instream->read( $foo, 3 );
+is( $foo, 'foo', "InStream seek" );
+
+my $dupe = $instream->reopen(
+ filename => 'foo',
+ offset => 1023,
+ len => 4
+);
+undef $foo;
+$dupe->read( $foo, 4 );
+
+is( $foo, 'afoo', "reopened instream" );
+
+# Trigger destruction.
+undef $folder;
diff --git a/perl/t/102-strings_io.t b/perl/t/102-strings_io.t
new file mode 100644
index 0000000..b4df1ca
--- /dev/null
+++ b/perl/t/102-strings_io.t
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 3;
+use Lucy::Test;
+
+my ( @items, $packed, $template, $buf, $file, $out, $in, $correct );
+
+$file = Lucy::Store::RAMFile->new;
+$out = Lucy::Store::OutStream->open( file => $file )
+ or die Lucy->error;
+$out->write_c64(10000);
+$out->close;
+$in = Lucy::Store::InStream->open( file => $file )
+ or die Lucy->error;
+$in->read_raw_c64($buf);
+$correct = $file->get_contents;
+is( $buf, $correct, "read_raw_c64" );
+
+$file = Lucy::Store::RAMFile->new;
+$out = Lucy::Store::OutStream->open( file => $file )
+ or die Lucy->error;
+$out->print("mint");
+$out->close;
+$buf = "funny";
+$in = Lucy::Store::InStream->open( file => $file )
+ or die Lucy->error;
+$in->read( $buf, 1 );
+is( $buf, "munny", 'read' );
+
+$file = Lucy::Store::RAMFile->new;
+$out = Lucy::Store::OutStream->open( file => $file )
+ or die Lucy->error;
+$out->print("cute");
+$out->close;
+$in = Lucy::Store::InStream->open( file => $file )
+ or die Lucy->error;
+$buf = "buzz";
+$in->read( $buf, 3, 4 );
+is( $buf, "buzzcut", 'read with offset' );
diff --git a/perl/t/105-folder.t b/perl/t/105-folder.t
new file mode 100644
index 0000000..4e7e8c1
--- /dev/null
+++ b/perl/t/105-folder.t
@@ -0,0 +1,107 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 25;
+use File::Spec::Functions qw( catfile );
+use Fcntl;
+use Lucy::Test::TestUtils qw( init_test_index_loc );
+use Lucy::Util::StringHelper qw( to_base36 );
+
+my $fs_index_loc = init_test_index_loc();
+my $fs_folder = Lucy::Store::FSFolder->new( path => $fs_index_loc, );
+my $ram_folder = Lucy::Store::RAMFolder->new;
+
+my $king = "I'm the king of rock.";
+for my $folder ( $fs_folder, $ram_folder ) {
+ my $outstream = $folder->open_out('king_of_rock')
+ or die Lucy->error;
+ $outstream->print($king);
+ $outstream->close;
+}
+
+for my $folder ( $fs_folder, $ram_folder ) {
+
+ my $files = $folder->list_r;
+ is_deeply( $files, ['king_of_rock'], "list lists files" );
+
+ $folder->mkdir('queen');
+ ok( $folder->exists('queen'), "mkdir" );
+
+ my $slurped = $folder->slurp_file('king_of_rock');
+ is( $slurped, $king, "slurp_file works" );
+
+ my $lock = Lucy::Store::LockFileLock->new(
+ host => '',
+ folder => $folder,
+ name => 'lock_robster',
+ timeout => 0,
+ );
+ my $competing_lock = Lucy::Store::LockFileLock->new(
+ host => '',
+ folder => $folder,
+ name => 'lock_robster',
+ timeout => 0,
+ );
+
+ $lock->obtain;
+ ok( $lock->is_locked, "lock is locked" );
+ ok( !$competing_lock->obtain, "shouldn't get lock on existing resource" );
+ ok( $lock->is_locked, "lock still locked after competing attempt" );
+
+ $lock->release;
+ ok( !$lock->is_locked, "release works" );
+
+ $lock->obtain;
+ $folder->rename( from => 'king_of_rock', to => 'king_of_lock' );
+ $lock->release;
+
+ ok( !$folder->exists('king_of_rock'),
+ "file successfully removed while locked"
+ );
+ is( $folder->exists('king_of_lock'),
+ 1, "file successfully moved while locked" );
+
+ is( $folder->open_out("king_of_lock"),
+ undef, "open_out returns undef when file exists" );
+
+ isa_ok( $folder->open_out("lockit"),
+ "Lucy::Store::OutStream",
+ "open_out succeeds when file doesn't exist" );
+
+ $folder->delete('king_of_lock');
+ ok( !$folder->exists('king_of_lock'), "Delete()" );
+}
+
+my $foo_path = catfile( $fs_index_loc, 'foo' );
+my $cf_path = catfile( $fs_index_loc, '_1.cf' );
+
+for ( $foo_path, $cf_path ) {
+ unlink $_;
+ sysopen( my $fh, $_, O_CREAT | O_EXCL | O_WRONLY )
+ or die "Couldn't open '$_' for writing: $!";
+ print $fh 'stuff';
+}
+
+$fs_folder = Lucy::Store::FSFolder->new( path => $fs_index_loc, );
+ok( -e $foo_path, "creating an FSFolder shouldn't wipe an unrelated file" );
+
+for ( 0 .. 100 ) {
+ my $filename = '_1-' . to_base36($_) . '.stuff';
+ $ram_folder->open_out($filename) or die Lucy->error;
+}
diff --git a/perl/t/106-locking.t b/perl/t/106-locking.t
new file mode 100644
index 0000000..b002b0a
--- /dev/null
+++ b/perl/t/106-locking.t
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Time::HiRes qw( sleep );
+use Test::More;
+use File::Spec::Functions qw( catfile );
+use Lucy::Test::TestUtils qw( init_test_index_loc );
+
+BEGIN {
+ if ( $^O =~ /(mswin|cygwin)/i ) {
+ plan( 'skip_all', "fork on Windows not supported by Lucy" );
+ }
+ else {
+ plan( tests => 3 );
+ }
+}
+
+my $path = init_test_index_loc();
+
+Dead_locks_are_removed: {
+ my $lock_path = catfile( $path, 'locks', 'foo.lock' );
+
+ # Remove any existing lockfile
+ unlink $lock_path;
+ die "Can't unlink '$lock_path'" if -e $lock_path;
+
+ my $folder = Lucy::Store::FSFolder->new( path => $path );
+
+ sub make_lock {
+ my $lock = Lucy::Store::LockFileLock->new(
+ timeout => 0,
+ name => 'foo',
+ host => '',
+ @_
+ );
+ $lock->clear_stale;
+ $lock->obtain or die "no dice";
+ return $lock;
+ }
+
+ # Fork a process that will create a lock and then exit
+ my $pid = fork();
+ if ( $pid == 0 ) { # child
+ make_lock( folder => $folder );
+ exit;
+ }
+ else {
+ waitpid( $pid, 0 );
+ }
+
+ sleep .1;
+ ok( -e $lock_path, "child secured lock" );
+
+ # The locking attempt will fail if the pid from the process that made the
+ # lock is active, so do the best we can to see whether another process
+ # started up with the child's pid (which would be weird).
+ my $pid_active = kill( 0, $pid );
+
+ eval { make_lock( folder => $folder, host => 'somebody_else' ) };
+ like( $@, qr/no dice/, "different host fails to get lock" );
+
+ eval { make_lock( folder => $folder ) };
+ warn $@ if $@;
+ my $saved_err = $@;
+ $pid_active ||= kill( 0, $pid );
+SKIP: {
+ skip( "Child's pid is active", 1 ) if $pid_active;
+ ok( !$saved_err,
+ 'second lock attempt clobbered dead lock file and did not die' );
+ }
+
+ undef $folder;
+}
diff --git a/perl/t/109-read_locking.t b/perl/t/109-read_locking.t
new file mode 100644
index 0000000..9a139fe
--- /dev/null
+++ b/perl/t/109-read_locking.t
@@ -0,0 +1,188 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 15;
+
+package FastIndexManager;
+use base qw( Lucy::Index::IndexManager );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ $self->set_deletion_lock_timeout(100);
+ return $self;
+}
+
+package NonMergingIndexManager;
+use base qw( FastIndexManager );
+sub recycle { [] }
+
+package main;
+use Scalar::Util qw( blessed );
+
+use Lucy::Test::TestUtils qw( create_index );
+use Lucy::Util::IndexFileNames qw( latest_snapshot );
+
+my $folder = create_index(qw( a b c ));
+my $schema = Lucy::Test::TestSchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ manager => FastIndexManager->new,
+ create => 1,
+);
+$indexer->delete_by_term( field => 'content', term => $_ ) for qw( a b c );
+$indexer->add_doc( { content => 'x' } );
+
+# Artificially create deletion lock.
+my $outstream = $folder->open_out('locks/deletion.lock')
+ or die Lucy->error;
+$outstream->print("{}");
+$outstream->close;
+{
+ my $captured;
+ local $SIG{__WARN__} = sub { $captured = shift; };
+ $indexer->commit;
+ like( $captured, qr/obsolete/,
+ "Indexer warns if it can't get a deletion lock" );
+}
+
+ok( $folder->exists('locks/deletion.lock'),
+ "Indexer doesn't delete deletion lock when it can't get it" );
+my $num_ds_files = grep {m/documents\.dat$/} @{ $folder->list_r };
+cmp_ok( $num_ds_files, '>', 1,
+ "Indexer doesn't process deletions when it can't get deletion lock" );
+
+my $num_snap_files = grep {m/snapshot/} @{ $folder->list_r };
+is( $num_snap_files, 2, "didn't zap the old snap file" );
+
+my $reader;
+SKIP: {
+ skip( "IndexReader opening failure leaks", 1 )
+ if $ENV{LUCY_VALGRIND};
+ eval {
+ $reader = Lucy::Index::IndexReader->open(
+ index => $folder,
+ manager => FastIndexManager->new( host => 'me' ),
+ );
+ };
+ ok( blessed($@) && $@->isa("Lucy::Store::LockErr"),
+ "IndexReader dies if it can't get deletion lock" );
+}
+$folder->delete('locks/deletion.lock') or die "Can't delete 'deletion.lock'";
+
+Test_race_condition_1: {
+ my $latest_snapshot_file = latest_snapshot($folder);
+
+ # Artificially set up situation where the index was updated and files
+ # PolyReader was expecting to see were zapped after a snapshot file was
+ # picked.
+ $folder->rename( from => $latest_snapshot_file, to => 'temp' );
+ $folder->rename(
+ from => 'seg_1',
+ to => 'seg_1.hidden',
+ );
+ Lucy::Index::IndexReader::set_race_condition_debug1(
+ Lucy::Object::CharBuf->new($latest_snapshot_file) );
+
+ $reader = Lucy::Index::IndexReader->open(
+ index => $folder,
+ manager => FastIndexManager->new( host => 'me' ),
+ );
+ is( $reader->doc_count, 1,
+ "reader overcomes race condition of index update after read lock" );
+ is( Lucy::Index::IndexReader::debug1_num_passes(),
+ 2, "reader retried before succeeding" );
+
+ # Clean up our artificial mess.
+ $folder->rename(
+ from => 'seg_1.hidden',
+ to => 'seg_1',
+ );
+ Lucy::Index::IndexReader::set_race_condition_debug1(undef);
+
+ $reader->close;
+}
+
+# Start over with one segment.
+$folder = create_index(qw( a b c x ));
+
+{
+ # Add a second segment and delete one doc from existing segment.
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ manager => NonMergingIndexManager->new,
+ );
+ $indexer->add_doc( { content => 'foo' } );
+ $indexer->add_doc( { content => 'bar' } );
+ $indexer->delete_by_term( field => 'content', term => 'x' );
+ $indexer->commit;
+
+ # Delete a doc from the second seg and increase del gen on first seg.
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ manager => NonMergingIndexManager->new,
+ );
+ $indexer->delete_by_term( field => 'content', term => 'a' );
+ $indexer->delete_by_term( field => 'content', term => 'foo' );
+ $indexer->commit;
+}
+
+# Establish read lock.
+$reader = Lucy::Index::IndexReader->open(
+ index => $folder,
+ manager => FastIndexManager->new( host => 'me' ),
+);
+
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => 'a' );
+$indexer->optimize;
+$indexer->commit;
+
+my $files = $folder->list_r;
+$num_snap_files = scalar grep {m/snapshot_\w+\.json$/} @$files;
+is( $num_snap_files, 2, "lock preserved last snapshot file" );
+my $num_del_files = scalar grep {m/deletions-seg_1\.bv$/} @$files;
+is( $num_del_files, 2, "outdated but locked del files survive" );
+ok( $folder->exists('seg_3/deletions-seg_1.bv'),
+ "first correct old del file" );
+ok( $folder->exists('seg_3/deletions-seg_2.bv'),
+ "second correct old del file" );
+$num_ds_files = scalar grep {m/documents\.dat$/} @$files;
+cmp_ok( $num_ds_files, '>', 1, "segment data files preserved" );
+
+undef $reader;
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->optimize;
+$indexer->commit;
+
+$files = $folder->list_r;
+$num_del_files = scalar grep {m/deletions/} @$files;
+is( $num_del_files, 0, "lock freed, del files optimized away" );
+$num_snap_files = scalar grep {m/snapshot_\w+\.json$/} @$files;
+is( $num_snap_files, 1, "lock freed, now only one snapshot file" );
+$num_ds_files = scalar grep {m/documents\.dat$/} @$files;
+is( $num_ds_files, 1, "lock freed, now only one ds file" );
diff --git a/perl/t/110-shared_lock.t b/perl/t/110-shared_lock.t
new file mode 100644
index 0000000..6a38273
--- /dev/null
+++ b/perl/t/110-shared_lock.t
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 14;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+
+my $lock = Lucy::Store::SharedLock->new(
+ folder => $folder,
+ name => 'ness',
+ timeout => 0,
+ host => 'nessie',
+);
+
+ok( !$lock->is_locked, "not locked yet" );
+
+ok( $lock->obtain, "obtain" );
+ok( $lock->is_locked, "is_locked" );
+ok( $folder->exists('locks/ness-1.lock'), "lockfile exists" );
+
+my $another_lock = Lucy::Store::SharedLock->new(
+ folder => $folder,
+ name => 'ness',
+ timeout => 0,
+ host => 'nessie',
+);
+ok( $another_lock->obtain, "got a second lock on the same resource" );
+
+$lock->release;
+ok( $lock->is_locked,
+ "first lock released but still is_locked because of other lock" );
+
+my $ya_lock = Lucy::Store::SharedLock->new(
+ folder => $folder,
+ name => 'ness',
+ timeout => 0,
+ host => 'nessie',
+);
+ok( $ya_lock->obtain, "got yet another lock" );
+
+ok( $lock->obtain, "got first lock again" );
+is( $lock->get_lock_path, "locks/ness-3.lock",
+ "first lock uses a different lock_path now" );
+
+# Rewrite lock file to spec a different pid.
+my $content = $folder->slurp_file("locks/ness-3.lock");
+$content =~ s/$$/123456789/;
+$folder->delete('locks/ness-3.lock') or die "Can't delete 'ness-3.lock'";
+my $outstream = $folder->open_out('locks/ness-3.lock')
+ or die Lucy->error;
+$outstream->print($content);
+$outstream->close;
+
+$lock->release;
+$another_lock->release;
+$ya_lock->release;
+
+ok( $lock->is_locked, "failed to release a lock with a different pid" );
+$lock->clear_stale;
+ok( !$lock->is_locked, "clear_stale" );
+
+ok( $lock->obtain, "got lock again" );
+ok( $lock->is_locked, "it's locked" );
+
+# Rewrite lock file to spec a different host.
+$content = $folder->slurp_file("locks/ness-1.lock");
+$content =~ s/nessie/sting/;
+$folder->delete('locks/ness-1.lock') or die "Can't delete 'ness-1.lock'";
+$outstream = $folder->open_out('locks/ness-1.lock') or die Lucy->error;
+$outstream->print($content);
+$outstream->close;
+
+$lock->release;
+ok( $lock->is_locked, "don't delete lock belonging to another host" );
diff --git a/perl/t/111-index_manager.t b/perl/t/111-index_manager.t
new file mode 100644
index 0000000..3044b4b
--- /dev/null
+++ b/perl/t/111-index_manager.t
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package NonMergingIndexManager;
+use base qw( Lucy::Index::IndexManager );
+sub recycle { [] }
+
+package BogusManager;
+use base qw( Lucy::Index::IndexManager );
+
+# Adds a bogus dupe.
+sub recycle {
+ my $recyclables = shift->SUPER::recycle(@_);
+ if (@$recyclables) { push @$recyclables, $recyclables->[0] }
+ return $recyclables;
+}
+
+package main;
+
+use Test::More tests => 16;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+
+my $lock_factory = Lucy::Store::LockFactory->new(
+ folder => $folder,
+ host => 'me',
+);
+
+my $lock = $lock_factory->make_lock(
+ name => 'angie',
+ timeout => 1000,
+);
+isa_ok( $lock, 'Lucy::Store::Lock', "make_lock" );
+is( $lock->get_name, "angie", "correct lock name" );
+is( $lock->get_host, "me", "correct host" );
+
+$lock = $lock_factory->make_shared_lock(
+ name => 'fred',
+ timeout => 0,
+);
+is( ref($lock), 'Lucy::Store::SharedLock', "make_shared_lock" );
+is( $lock->get_name, "fred", "correct lock name" );
+is( $lock->get_host, "me", "correct host" );
+
+my $schema = Lucy::Test::TestSchema->new;
+$folder = Lucy::Store::RAMFolder->new;
+
+for ( 1 .. 20 ) {
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ manager => NonMergingIndexManager->new,
+ );
+
+ # Two big segs that shouldn't merge, then small, mergable segs.
+ my $reps = $_ <= 2 ? 100 : 1;
+ $indexer->add_doc( { content => $_ } ) for 1 .. $reps;
+ $indexer->commit;
+}
+my $num_segs = grep {m/segmeta.json/} @{ $folder->list_r };
+is( $num_segs, 20, "no merging" );
+
+my $manager = Lucy::Index::IndexManager->new;
+$manager->set_folder($folder);
+
+my $polyreader = Lucy::Index::PolyReader->open( index => $folder );
+my $segment = Lucy::Index::Segment->new( number => 22 );
+my $snapshot = Lucy::Index::Snapshot->new->read_file( folder => $folder );
+my $deletions_writer = Lucy::Index::DefaultDeletionsWriter->new(
+ schema => $schema,
+ segment => $segment,
+ snapshot => $snapshot,
+ polyreader => $polyreader,
+);
+my $seg_readers = $manager->recycle(
+ reader => $polyreader,
+ cutoff => 19,
+ del_writer => $deletions_writer,
+);
+is( scalar @$seg_readers, 1, "cutoff" );
+
+$seg_readers = $manager->recycle(
+ reader => $polyreader,
+ cutoff => 0,
+ del_writer => $deletions_writer,
+);
+is( scalar @$seg_readers,
+ 18, "recycle lots of small segs but leave big ones alone" );
+
+$manager->set_write_lock_timeout(1);
+is( $manager->get_write_lock_timeout, 1, "set/get write lock timeout" );
+$manager->set_write_lock_interval(2);
+is( $manager->get_write_lock_interval, 2, "set/get write lock interval" );
+$manager->set_merge_lock_timeout(3);
+is( $manager->get_merge_lock_timeout, 3, "set/get merge lock timeout" );
+$manager->set_merge_lock_interval(4);
+is( $manager->get_merge_lock_interval, 4, "set/get merge lock interval" );
+$manager->set_deletion_lock_timeout(5);
+is( $manager->get_deletion_lock_timeout, 5, "set/get deletion lock timeout" );
+$manager->set_deletion_lock_interval(6);
+is( $manager->get_deletion_lock_interval,
+ 6, "set/get deletion lock interval" );
+
+SKIP: {
+ skip( "Known leak", 1 ) if $ENV{LUCY_VALGRIND};
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ manager => BogusManager->new,
+ );
+ eval { $indexer->commit };
+ like( $@, qr/recycle/i, "duplicated segment via recycle triggers error" );
+}
diff --git a/perl/t/150-polyanalyzer.t b/perl/t/150-polyanalyzer.t
new file mode 100644
index 0000000..9c852b3
--- /dev/null
+++ b/perl/t/150-polyanalyzer.t
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 7;
+use Lucy::Test::TestUtils qw( test_analyzer );
+
+my $source_text = 'Eats, shoots and leaves.';
+my $case_folder = Lucy::Analysis::CaseFolder->new;
+my $polyanalyzer
+ = Lucy::Analysis::PolyAnalyzer->new( analyzers => [$case_folder], );
+test_analyzer(
+ $polyanalyzer, $source_text,
+ ['eats, shoots and leaves.'],
+ '"analyzers" constructor arg'
+);
+$polyanalyzer = Lucy::Analysis::PolyAnalyzer->new( language => 'en', );
+test_analyzer(
+ $polyanalyzer, $source_text,
+ [qw( eat shoot and leav )],
+ '"language" constructor arg'
+);
+
+ok( $polyanalyzer->get_analyzers(), "get_analyzers method" );
+
diff --git a/perl/t/151-analyzer.t b/perl/t/151-analyzer.t
new file mode 100644
index 0000000..08e2da4
--- /dev/null
+++ b/perl/t/151-analyzer.t
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 5;
+use Lucy::Test::TestUtils qw( utf8_test_strings test_analyzer );
+
+package TestAnalyzer;
+use base qw( Lucy::Analysis::Analyzer );
+sub transform { $_[1] } # satisfy mandatory override
+
+package main;
+my $analyzer = TestAnalyzer->new;
+
+my ( $smiley, $not_a_smiley, $frowny ) = utf8_test_strings();
+
+my $got = $analyzer->split($not_a_smiley)->[0];
+is( $got, $frowny, "split() upgrades non-UTF-8 correctly" );
+
+$got = $analyzer->split($smiley)->[0];
+is( $got, $smiley, "split() handles UTF-8 correctly" );
+
+test_analyzer( $analyzer, 'foo', ['foo'], "Analyzer (no-op)" );
diff --git a/perl/t/152-inversion.t b/perl/t/152-inversion.t
new file mode 100644
index 0000000..72a356a
--- /dev/null
+++ b/perl/t/152-inversion.t
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 4;
+use Lucy::Test::TestUtils qw( utf8_test_strings );
+
+my $inversion = Lucy::Analysis::Inversion->new;
+$inversion->append(
+ Lucy::Analysis::Token->new(
+ text => "car",
+ start_offset => 0,
+ end_offset => 3,
+ ),
+);
+$inversion->append(
+ Lucy::Analysis::Token->new(
+ text => "bike",
+ start_offset => 10,
+ end_offset => 14,
+ ),
+);
+$inversion->append(
+ Lucy::Analysis::Token->new(
+ text => "truck",
+ start_offset => 20,
+ end_offset => 25,
+ ),
+);
+
+my @texts;
+while ( my $token = $inversion->next ) {
+ push @texts, $token->get_text;
+}
+is_deeply( \@texts, [qw( car bike truck )], "return tokens in order" );
+
+$inversion = Lucy::Analysis::Inversion->new;
+$inversion->append(
+ Lucy::Analysis::Token->new(
+ text => "foo",
+ start_offset => 0,
+ end_offset => 3,
+ pos_inc => 10,
+ ),
+);
+$inversion->append(
+ Lucy::Analysis::Token->new(
+ text => "bar",
+ start_offset => 4,
+ end_offset => 7,
+ pos_inc => ( 2**31 - 2 ),
+ ),
+);
+eval { $inversion->invert; };
+like( $@, qr/position/, "catch overflow in token position calculation" );
+
+my ( $smiley, $not_a_smiley, $frowny ) = utf8_test_strings();
+
+$inversion = Lucy::Analysis::Inversion->new( text => $smiley );
+is( $inversion->next->get_text,
+ $smiley, "Inversion->new handles UTF-8 correctly" );
+$inversion = Lucy::Analysis::Inversion->new( text => $not_a_smiley );
+is( $inversion->next->get_text,
+ $frowny, "Inversion->new upgrades non-UTF-8 correctly" );
diff --git a/perl/t/153-case_folder.t b/perl/t/153-case_folder.t
new file mode 100644
index 0000000..8bf4c27
--- /dev/null
+++ b/perl/t/153-case_folder.t
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 3;
+use Lucy::Test::TestUtils qw( test_analyzer );
+
+my $case_folder = Lucy::Analysis::CaseFolder->new;
+
+test_analyzer( $case_folder, "caPiTal ofFensE",
+ ['capital offense'], 'lc plain text' );
diff --git a/perl/t/154-regex_tokenizer.t b/perl/t/154-regex_tokenizer.t
new file mode 100644
index 0000000..9ce06c9
--- /dev/null
+++ b/perl/t/154-regex_tokenizer.t
@@ -0,0 +1,102 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 15;
+use Lucy::Test;
+
+my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+my $other = Lucy::Analysis::RegexTokenizer->new( pattern => '\w+' );
+my $yet_another = Lucy::Analysis::RegexTokenizer->new( pattern => '\w+' );
+ok( $other->equals($yet_another), "Equals" );
+ok( !$tokenizer->equals($other), "different patterns foil Equals" );
+
+my $text = $tokenizer->split("o'malley's")->[0];
+is( $text, "o'malley's", "multiple apostrophes for default pattern" );
+
+my $inversion = Lucy::Analysis::Inversion->new( text => "a b c" );
+$inversion = $tokenizer->transform($inversion);
+
+my ( @token_texts, @start_offsets, @end_offsets );
+while ( my $token = $inversion->next ) {
+ push @token_texts, $token->get_text;
+ push @start_offsets, $token->get_start_offset;
+ push @end_offsets, $token->get_end_offset;
+}
+is_deeply( \@token_texts, [qw( a b c )], "correct texts" );
+is_deeply( \@start_offsets, [ 0, 2, 4, ], "correctstart offsets" );
+is_deeply( \@end_offsets, [ 1, 3, 5, ], "correct end offsets" );
+
+$tokenizer = Lucy::Analysis::RegexTokenizer->new( pattern => '.' );
+$inversion = Lucy::Analysis::Inversion->new( text => "a b c" );
+$inversion = $tokenizer->transform($inversion);
+
+@token_texts = ();
+@start_offsets = ();
+@end_offsets = ();
+while ( my $token = $inversion->next ) {
+ push @token_texts, $token->get_text;
+ push @start_offsets, $token->get_start_offset;
+ push @end_offsets, $token->get_end_offset;
+}
+is_deeply(
+ \@token_texts,
+ [ 'a', ' ', 'b', ' ', 'c' ],
+ "texts: custom pattern"
+);
+is_deeply( \@start_offsets, [ 0 .. 4 ], "starts: custom pattern" );
+is_deeply( \@end_offsets, [ 1 .. 5 ], "ends: custom pattern" );
+
+$inversion->reset;
+$inversion = $tokenizer->transform($inversion);
+@token_texts = ();
+while ( my $token = $inversion->next ) {
+ push @token_texts, $token->get_text;
+}
+is_deeply(
+ \@token_texts,
+ [ 'a', ' ', 'b', ' ', 'c' ],
+ "no freakout when fed multiple tokens"
+);
+
+$tokenizer = Lucy::Analysis::RegexTokenizer->new( token_re => qr/../ );
+is_deeply( $tokenizer->split('aabbcc'),
+ [qw( aa bb cc )], "back compat with token_re argument" );
+
+eval {
+ my $toke
+ = Lucy::Analysis::RegexTokenizer->new(
+ pattern => '\\p{Carp::confess}' );
+};
+like( $@, qr/\\p/, "\\p forbidden in pattern" );
+
+eval {
+ my $toke
+ = Lucy::Analysis::RegexTokenizer->new(
+ pattern => '\\P{Carp::confess}' );
+};
+like( $@, qr/\\P/, "\\P forbidden in pattern" );
+
+$tokenizer = Lucy::Analysis::RegexTokenizer->new( pattern => '\\w+' );
+my $dump = $tokenizer->dump;
+$dump->{pattern} = "\\p{Carp::confess}";
+eval { $tokenizer->load($dump) };
+like( $@, qr/\\p/, "\\p forbidden during load" );
+
+$dump->{pattern} = "\\P{Carp::confess}";
+eval { $tokenizer->load($dump) };
+like( $@, qr/\\P/, "\\P forbidden during load" );
diff --git a/perl/t/155-snowball_stop_filter.t b/perl/t/155-snowball_stop_filter.t
new file mode 100644
index 0000000..f5af6f5
--- /dev/null
+++ b/perl/t/155-snowball_stop_filter.t
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 6;
+use Lucy::Test::TestUtils qw( test_analyzer );
+
+my $stopfilter = Lucy::Analysis::SnowballStopFilter->new( language => 'en' );
+test_analyzer( $stopfilter, 'the', [], "single stopword stopalized" );
+
+my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $tokenizer, $stopfilter ], );
+test_analyzer( $polyanalyzer, 'i am the walrus',
+ ['walrus'], "multiple stopwords stopalized" );
diff --git a/perl/t/156-snowball_stemmer.t b/perl/t/156-snowball_stemmer.t
new file mode 100644
index 0000000..3bc1671
--- /dev/null
+++ b/perl/t/156-snowball_stemmer.t
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use Lucy::Test::TestUtils qw( test_analyzer );
+
+my $stemmer = Lucy::Analysis::SnowballStemmer->new( language => 'en' );
+test_analyzer( $stemmer, 'ponies', ['poni'], "single word stemmed" );
+test_analyzer( $stemmer, 'pony', ['poni'], "stem, not just truncate" );
+
+my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $tokenizer, $stemmer ], );
+test_analyzer(
+ $polyanalyzer,
+ 'peas porridge hot',
+ [ 'pea', 'porridg', 'hot' ],
+ "multiple words stemmed",
+);
diff --git a/perl/t/200-doc.t b/perl/t/200-doc.t
new file mode 100644
index 0000000..63a9564
--- /dev/null
+++ b/perl/t/200-doc.t
@@ -0,0 +1,57 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 11;
+use Storable qw( nfreeze thaw );
+use Lucy::Test;
+
+my $doc = Lucy::Document::Doc->new;
+is_deeply( $doc->get_fields, {}, "get_fields" );
+is( $doc->get_doc_id, 0, "default doc_id of 0" );
+
+$doc->set_fields( { foo => 'oink' } );
+is_deeply( $doc->get_fields, { foo => 'oink' }, "set_fields" );
+
+$doc->{foo} = "blah";
+is_deeply( $doc->get_fields, { foo => 'blah' }, "overloading" );
+
+my %hash = ( foo => 'foo' );
+$doc = Lucy::Document::Doc->new(
+ fields => \%hash,
+ doc_id => 30,
+);
+$hash{bar} = "blah";
+is_deeply(
+ $doc->get_fields,
+ { foo => 'foo', bar => 'blah' },
+ "using supplied hash"
+);
+is( $doc->get_doc_id, 30, "doc_id param" );
+$doc->set_doc_id(20);
+is( $doc->get_doc_id, 20, "doc_id param" );
+
+my $frozen = nfreeze($doc);
+my $thawed = thaw($frozen);
+is_deeply( $thawed->get_fields, $doc->get_fields,
+ "fields survive freeze/thaw" );
+is( $thawed->get_doc_id, $doc->get_doc_id, "doc_id survives freeze/thaw" );
+ok( $doc->equals($thawed), "equals" );
+
+my $dump = $doc->dump;
+my $loaded = $doc->load($dump);
+ok( $doc->equals($loaded), "dump => load round trip" );
diff --git a/perl/t/201-hit_doc.t b/perl/t/201-hit_doc.t
new file mode 100644
index 0000000..145fbd4
--- /dev/null
+++ b/perl/t/201-hit_doc.t
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 8;
+use Storable qw( nfreeze thaw );
+use Lucy::Test;
+
+my $doc = Lucy::Document::HitDoc->new;
+is( $doc->get_doc_id, 0, "default doc_id of 0" );
+is( $doc->get_score, 0.0, "default score of 0.0" );
+$doc->set_score(2);
+is( $doc->get_score, 2, "set_score" );
+
+$doc->{foo} = "foo foo";
+is( $doc->{foo}, "foo foo", "hash overloading" );
+
+my $frozen = nfreeze($doc);
+my $thawed = thaw($frozen);
+is( ref($thawed), ref($doc), "correct class after freeze/thaw" );
+is( $thawed->get_score, $doc->get_score, "score survives freeze/thaw" );
+ok( $doc->equals($thawed), "equals" );
+
+my $dump = $doc->dump;
+my $loaded = $doc->load($dump);
+ok( $doc->equals($loaded), "dump => load round trip" );
diff --git a/perl/t/204-doc_reader.t b/perl/t/204-doc_reader.t
new file mode 100644
index 0000000..df712a0
--- /dev/null
+++ b/perl/t/204-doc_reader.t
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 5;
+
+package TestAnalyzer;
+use base qw( Lucy::Analysis::Analyzer );
+sub transform { $_[1] }
+
+package main;
+use Encode qw( _utf8_on );
+use Lucy::Test;
+
+sub new_schema {
+ my $schema = Lucy::Plan::Schema->new;
+ my $analyzer = TestAnalyzer->new;
+ my $fulltext = Lucy::Plan::FullTextType->new( analyzer => $analyzer );
+ my $bin = Lucy::Plan::BlobType->new( stored => 1 );
+ my $not_stored = Lucy::Plan::FullTextType->new(
+ analyzer => $analyzer,
+ stored => 0,
+ );
+ my $float64 = Lucy::Plan::Float64Type->new( indexed => 0 );
+ $schema->spec_field( name => 'text', type => $fulltext );
+ $schema->spec_field( name => 'bin', type => $bin );
+ $schema->spec_field( name => 'unstored', type => $not_stored );
+ $schema->spec_field( name => 'float64', type => $float64 );
+ $schema->spec_field( name => 'empty', type => $fulltext );
+ return $schema;
+}
+
+# This valid UTF-8 string includes skull and crossbones, null byte -- however,
+# the binary value is not flagged as UTF-8.
+my $bin_val = my $val = "a b c \xe2\x98\xA0 \0a";
+_utf8_on($val);
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = new_schema();
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ create => 1,
+);
+$indexer->add_doc(
+ { text => $val,
+ bin => $bin_val,
+ unstored => $val,
+ empty => '',
+ float64 => 2.0,
+ }
+);
+$indexer->commit;
+
+my $snapshot = Lucy::Index::Snapshot->new->read_file( folder => $folder );
+my $segment = Lucy::Index::Segment->new( number => 1 );
+$segment->read_file($folder);
+my $doc_reader = Lucy::Index::DefaultDocReader->new(
+ schema => $schema,
+ folder => $folder,
+ snapshot => $snapshot,
+ segments => [$segment],
+ seg_tick => 0,
+);
+
+my $doc = $doc_reader->fetch_doc(0);
+
+is( $doc->{text}, $val, "text" );
+is( $doc->{bin}, $bin_val, "bin" );
+is( $doc->{unstored}, undef, "unstored" );
+is( $doc->{empty}, '', "empty" );
+is( $doc->{float64}, 2.0, "float64" );
diff --git a/perl/t/205-seg_reader.t b/perl/t/205-seg_reader.t
new file mode 100644
index 0000000..1927be9
--- /dev/null
+++ b/perl/t/205-seg_reader.t
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 8;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index(
+ "What's he building in there?",
+ "What's he building in there?",
+ "We have a right to know."
+);
+my $polyreader = Lucy::Index::IndexReader->open( index => $folder );
+my $reader = $polyreader->get_seg_readers->[0];
+
+isa_ok( $reader, 'Lucy::Index::SegReader' );
+
+is( $reader->doc_max, 3, "doc_max returns correct number" );
+
+my $lex_reader = $reader->fetch("Lucy::Index::LexiconReader");
+isa_ok( $lex_reader, 'Lucy::Index::LexiconReader', "fetch() a component" );
+ok( !defined( $reader->fetch("nope") ),
+ "fetch() returns undef when component can't be found" );
+$lex_reader = $reader->obtain("Lucy::Index::LexiconReader");
+isa_ok( $lex_reader, 'Lucy::Index::LexiconReader', "obtain() a component" );
+eval { $reader->obtain("boom."); };
+like( $@, qr/boom/, "obtain blows up when component can't be found" );
+
+is_deeply( $reader->seg_readers, [$reader], "seg_readers" );
+is_deeply( $reader->offsets, [0], "offsets" );
+
diff --git a/perl/t/207-seg_lexicon.t b/perl/t/207-seg_lexicon.t
new file mode 100644
index 0000000..f3e38e4
--- /dev/null
+++ b/perl/t/207-seg_lexicon.t
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use utf8;
+
+use Test::More tests => 5;
+use Lucy::Test;
+
+package TestAnalyzer;
+use base qw( Lucy::Analysis::Analyzer );
+sub transform { $_[1] }
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::FullTextType->new( analyzer => TestAnalyzer->new );
+ $self->spec_field( name => 'a', type => $type );
+ $self->spec_field( name => 'b', type => $type );
+ $self->spec_field( name => 'c', type => $type );
+ return $self;
+}
+
+package main;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ create => 1,
+ index => $folder,
+ schema => $schema,
+);
+
+# We need to test strings that exceed the Latin-1 range to make sure that
+# get_term treats them correctly. (See change 3103 in the svn repo.)
+my @animals = qw( cat dog sloth λεοντάρι змейка );
+for my $animal (@animals) {
+ $indexer->add_doc(
+ { a => $animal,
+ b => $animal,
+ c => $animal,
+ }
+ );
+}
+$indexer->commit;
+
+my $snapshot = Lucy::Index::Snapshot->new->read_file( folder => $folder );
+my $segment = Lucy::Index::Segment->new( number => 1 );
+$segment->read_file($folder);
+my $lex_reader = Lucy::Index::DefaultLexiconReader->new(
+ schema => $schema,
+ folder => $folder,
+ snapshot => $snapshot,
+ segments => [$segment],
+ seg_tick => 0,
+);
+my %lexicons;
+for (qw( a b c )) {
+ $lexicons{$_} = $lex_reader->lexicon( field => $_ );
+}
+
+my @fields;
+my @terms;
+for (qw( a b c )) {
+ my $lexicon = $lexicons{$_};
+ while ( $lexicon->next ) {
+ push @fields, $lexicon->get_field;
+ push @terms, $lexicon->get_term;
+ }
+}
+is_deeply( \@fields, [qw( a a a a a b b b b b c c c c c )],
+ "correct fields" );
+my @correct_texts = (@animals) x 3;
+is_deeply( \@terms, \@correct_texts, "correct terms" );
+
+my $lexicon = $lexicons{b};
+$lexicon->seek("dog");
+$lexicon->next;
+is( $lexicon->get_term, 'sloth', "lexicon seeks to correct term (ptr)" );
+is( $lexicon->get_field, 'b', "lexicon has correct field" );
+
+$lexicon->reset;
+$lexicon->next;
+is( $lexicon->get_term, 'cat', "reset" );
diff --git a/perl/t/208-terminfo.t b/perl/t/208-terminfo.t
new file mode 100644
index 0000000..7b8d326
--- /dev/null
+++ b/perl/t/208-terminfo.t
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 11;
+use Lucy::Test;
+
+my $tinfo = Lucy::Index::TermInfo->new( doc_freq => 10, );
+$tinfo->set_post_filepos(20);
+$tinfo->set_skip_filepos(40);
+$tinfo->set_lex_filepos(50);
+
+my $cloned_tinfo = $tinfo->clone;
+ok( !$tinfo->equals($cloned_tinfo),
+ "the clone should be a separate C struct" );
+
+is( $tinfo->get_doc_freq, 10, "new sets doc_freq correctly" );
+is( $tinfo->get_doc_freq, 10, "... doc_freq cloned" );
+is( $tinfo->get_post_filepos, 20, "... post_filepos cloned" );
+is( $tinfo->get_skip_filepos, 40, "... skip_filepos cloned" );
+is( $tinfo->get_lex_filepos, 50, "... lex_filepos cloned" );
+
+$tinfo->set_doc_freq(5);
+is( $tinfo->get_doc_freq, 5, "set/get doc_freq" );
+is( $cloned_tinfo->get_doc_freq, 10, "setting orig doesn't affect clone" );
+
+$tinfo->set_post_filepos(15);
+is( $tinfo->get_post_filepos, 15, "set/get post_filepos" );
+
+$tinfo->set_skip_filepos(35);
+is( $tinfo->get_skip_filepos, 35, "set/get skip_filepos" );
+
+$tinfo->set_lex_filepos(45);
+is( $tinfo->get_lex_filepos, 45, "set/get lex_filepos" );
diff --git a/perl/t/209-seg_lexicon_heavy.t b/perl/t/209-seg_lexicon_heavy.t
new file mode 100644
index 0000000..1b8a915
--- /dev/null
+++ b/perl/t/209-seg_lexicon_heavy.t
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 3;
+use Lucy::Test::TestUtils qw( create_index );
+
+my @docs;
+my @chars = ( 'a' .. 'z', 'B' .. 'E', 'G' .. 'Z' );
+for ( 0 .. 1000 ) {
+ my $content = '';
+ for my $num_words ( 0 .. int( rand(20) ) ) {
+ for my $num_chars ( 1 .. int( rand(10) ) ) {
+ $content .= @chars[ rand(@chars) ];
+ }
+ $content .= ' ';
+ }
+ push @docs, "$content\n";
+}
+my $folder = create_index(
+ ( 1 .. 1000 ),
+ ( ("a") x 100 ),
+ "Foo",
+ @docs,
+ "Foo",
+ "A MAN",
+ "A PLAN",
+ "A CANAL",
+ "PANAMA"
+);
+my $schema = Lucy::Test::TestSchema->new;
+
+my $snapshot = Lucy::Index::Snapshot->new->read_file( folder => $folder );
+my $segment = Lucy::Index::Segment->new( number => 1 );
+$segment->read_file($folder);
+my $lex_reader = Lucy::Index::DefaultLexiconReader->new(
+ schema => $schema,
+ folder => $folder,
+ snapshot => $snapshot,
+ segments => [$segment],
+ seg_tick => 0,
+);
+
+my $lexicon = $lex_reader->lexicon( field => 'content' );
+$lexicon->next;
+my $last_text = $lexicon->get_term;
+$lexicon->next;
+my $current_text;
+my $num_iters = 2;
+while (1) {
+ $current_text = $lexicon->get_term;
+ last unless $current_text gt $last_text;
+ last unless $lexicon->next;
+ $num_iters++;
+ $current_text = $last_text;
+}
+cmp_ok( $last_text, 'lt', $current_text, "term texts in sorted order" );
+
+$lexicon->seek('A');
+my $tinfo = $lexicon->get_term_info();
+is( $tinfo->get_doc_freq, 3, "correct retrieval #1" );
+
+$lexicon->seek('Foo');
+$tinfo = $lexicon->get_term_info();
+is( $tinfo->get_doc_freq, 2, "correct retrieval #2" );
diff --git a/perl/t/210-deldocs.t b/perl/t/210-deldocs.t
new file mode 100644
index 0000000..948b8f1
--- /dev/null
+++ b/perl/t/210-deldocs.t
@@ -0,0 +1,179 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 16;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a' .. 'e' );
+my $polyreader = Lucy::Index::PolyReader->open( index => $folder, );
+my $seg_reader = $polyreader->seg_readers->[0];
+my $snapshot = $polyreader->get_snapshot;
+
+my $del_writer = Lucy::Index::DefaultDeletionsWriter->new(
+ schema => $polyreader->get_schema,
+ polyreader => $polyreader,
+ segment => $seg_reader->get_segment,
+ snapshot => $snapshot,
+);
+$del_writer->delete_by_term( field => 'content', term => 'c' );
+my $doc_map = $del_writer->generate_doc_map(
+ deletions => $del_writer->seg_deletions($seg_reader),
+ doc_max => $seg_reader->doc_max,
+ offset => 0,
+);
+my @correct = ( 1, 2, 0, 3, 4 );
+my @got;
+push @got, $doc_map->get($_) for 1 .. 5;
+is_deeply( \@got, \@correct, "doc map maps around deleted docs" );
+
+$doc_map = $del_writer->generate_doc_map(
+ deletions => $del_writer->seg_deletions($seg_reader),
+ doc_max => $seg_reader->doc_max,
+ offset => 100,
+);
+is( $doc_map->get(4), 103, "doc map handles offset correctly" );
+ok( !$doc_map->get(3), "doc_map handled deletions correctly" );
+
+my $new_seg = Lucy::Index::Segment->new( number => 2 );
+$del_writer = Lucy::Index::DefaultDeletionsWriter->new(
+ schema => $polyreader->get_schema,
+ polyreader => $polyreader,
+ segment => $new_seg,
+ snapshot => $snapshot,
+);
+$del_writer->delete_by_term( field => 'content', term => 'a' );
+$del_writer->delete_by_doc_id(2);
+$folder->mkdir('seg_2'); # ordinarily done by Indexer
+$del_writer->finish;
+$new_seg->write_file($folder);
+$snapshot->add_entry( $new_seg->get_name );
+
+for my $entry ( values %{ $new_seg->fetch_metadata('deletions')->{files} } ) {
+ $snapshot->add_entry( $entry->{filename} );
+}
+$snapshot->write_file( folder => $folder );
+
+$polyreader = Lucy::Index::PolyReader->open( index => $folder );
+$seg_reader = $polyreader->seg_readers->[0];
+my $del_reader = $seg_reader->obtain("Lucy::Index::DeletionsReader");
+my $deldocs = $del_reader->read_deletions;
+
+ok( $deldocs->get(2), "Delete_By_Term" );
+ok( $deldocs->get(2), "Delete_By_Doc_ID" );
+
+my @deleted_or_not = map { $deldocs->get($_) } 0 .. 7;
+is_deeply(
+ \@deleted_or_not,
+ [ 0, 1, 1, 0, 0, 0, 0, 0 ],
+ "finish() and read_deldocs() save/recover deletions correctly"
+);
+
+is( $deldocs->count, 2,
+ "finish() and read_deldocs() save/recover num_deletions correctly" );
+is( $deldocs->get_capacity, 8, "finish() wrote correct number of bytes" );
+
+$folder = Lucy::Store::RAMFolder->new;
+my $schema = Lucy::Test::TestSchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { content => $_ } ) for 'a' .. 'c';
+$indexer->commit;
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_query( Lucy::Search::MatchAllQuery->new );
+$indexer->commit;
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { content => $_ } ) for 'a' .. 'c';
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'a' );
+is( $hits->total_hits, 1, "deleting then re-adding works" );
+
+my @expected;
+for ( 'a' .. 'e' ) {
+ $hits = $searcher->hits( query => $_ );
+ my @contents;
+ while ( my $hit = $hits->next ) {
+ push @contents, $hit->{content};
+ }
+ push @expected, \@contents;
+}
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->optimize;
+$indexer->commit;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+@got = ();
+for ( 'a' .. 'e' ) {
+ $hits = $searcher->hits( query => $_ );
+ my @contents;
+ while ( my $hit = $hits->next ) {
+ push @contents, $hit->{content};
+ }
+ push @got, \@contents;
+}
+is_deeply( \@got, \@expected, "segment merging handles deletions correctly" );
+
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $_ ) for 'a' .. 'c';
+$indexer->commit;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'a' );
+is( $hits->total_hits, 0, "adding and searching empty segments is ok" );
+
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ truncate => 1,
+);
+$indexer->add_doc( { content => 'foo' } );
+$indexer->add_doc( { content => 'bar' } );
+$indexer->commit;
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+is( $searcher->doc_max, 2, "correct number of docs in index" );
+$hits = $searcher->hits( query => 'foo' );
+is( $hits->total_hits, 1, "found term" );
+
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ truncate => 1,
+);
+$indexer->add_doc( { content => 'baz' } );
+$indexer->commit;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+is( $searcher->doc_max, 1, "correct doc_max after truncation" );
+$hits = $searcher->hits( query => 'foo' );
+is( $hits->total_hits, 0, "truncate succeeded" );
+$hits = $searcher->hits( query => 'baz' );
+is( $hits->total_hits, 1, "added doc during same session as truncation" );
diff --git a/perl/t/211-seg_posting_list.t b/perl/t/211-seg_posting_list.t
new file mode 100644
index 0000000..c755ad7
--- /dev/null
+++ b/perl/t/211-seg_posting_list.t
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 2004;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( qw( a b c ), 'c c d' );
+my $polyreader = Lucy::Index::IndexReader->open( index => $folder );
+my $reader = $polyreader->get_seg_readers->[0];
+my $plist_reader = $reader->fetch("Lucy::Index::PostingListReader");
+
+my $plist = $plist_reader->posting_list( field => 'content', term => 'c' );
+
+my ( @docs, @prox, @docs_from_next );
+while ( my $doc_id = $plist->next ) {
+ push @docs_from_next, $doc_id;
+ my $posting = $plist->get_posting;
+ push @docs, $posting->get_doc_id;
+ push @prox, $posting->get_prox;
+}
+is_deeply( \@docs, [ 3, 4 ], "correct docs after SegPList_Next" );
+is_deeply( \@docs_from_next, [ 3, 4 ], "correct docs via SegPList_Next" );
+is_deeply( \@prox, [ [0], [ 0, 1 ] ], "correct prox from SegPList_Next" );
+
+$plist->seek('c');
+$plist->next;
+is( $plist->get_posting->get_doc_id, 3, "seek" );
+
+$folder = Lucy::Store::RAMFolder->new;
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => Lucy::Test::TestSchema->new,
+);
+for ( 0 .. 100 ) {
+ my $content = 'a ';
+ $content .= 'b ' if ( $_ % 2 == 0 );
+ $content .= 'c ' if ( $_ % 3 == 0 );
+ $content .= 'd ' if ( $_ % 4 == 0 );
+ $content .= 'e ' if ( $_ % 5 == 0 );
+ $indexer->add_doc( { content => $content } );
+}
+$indexer->commit;
+$polyreader = Lucy::Index::IndexReader->open( index => $folder );
+$reader = $polyreader->get_seg_readers->[0];
+$plist_reader = $reader->fetch("Lucy::Index::PostingListReader");
+
+for my $letter (qw( a b c d e )) {
+ my $skipping_plist = $plist_reader->posting_list(
+ field => 'content',
+ term => $letter,
+ );
+ my $plodding_plist = $plist_reader->posting_list(
+ field => 'content',
+ term => $letter,
+ );
+
+ # Compare results of advance() to results of next().
+ for my $target ( 1 .. 100 ) {
+ $skipping_plist->seek($letter);
+ $plodding_plist->seek($letter);
+ my $skipping_doc_id = $skipping_plist->advance($target);
+ my $plodding_doc_id;
+ do {
+ $plodding_doc_id = $plodding_plist->next
+ or die "shouldn't happen: $target";
+ } while ( $plodding_plist->get_doc_id < $target );
+
+ # Verify that the plists have identical state.
+ is( $skipping_doc_id, $plodding_doc_id,
+ "$letter: doc_ids via advance, next are identical" );
+ is( $skipping_plist->get_doc_id, $plodding_plist->get_doc_id,
+ "$letter: skip to $target" );
+ is( $skipping_plist->get_post_stream->tell,
+ $plodding_plist->get_post_stream->tell,
+ "$letter: identical filepos for $target"
+ );
+ is( $skipping_plist->get_count, $plodding_plist->get_count,
+ "$letter: identical count for $target" );
+ }
+}
diff --git a/perl/t/213-segment_merging.t b/perl/t/213-segment_merging.t
new file mode 100644
index 0000000..fc5514b
--- /dev/null
+++ b/perl/t/213-segment_merging.t
@@ -0,0 +1,200 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+package NonMergingIndexManager;
+use base qw( Lucy::Index::IndexManager );
+
+sub recycle {
+ return Lucy::Object::VArray->new( capacity => 0 );
+}
+
+# BiggerSchema is like TestSchema, but it has an extra field named "aux".
+# Because "aux" sorts before "content", it forces a remapping of field numbers
+# when an index created under TestSchema is opened/modified under
+# BiggerSchema.
+package BiggerSchema;
+use base qw( Lucy::Test::TestSchema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new,
+ highlightable => 1,
+ );
+ $self->spec_field( name => 'content', type => $type );
+ $self->spec_field( name => 'aux', type => $type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 10;
+use Lucy::Test::TestUtils qw( create_index init_test_index_loc );
+use File::Find qw( find );
+
+my $index_loc = init_test_index_loc();
+
+my $num_reps;
+{
+ # Verify that optimization truly cuts down on the number of segments.
+ my $schema = Lucy::Test::TestSchema->new;
+ for ( $num_reps = 1;; $num_reps++ ) {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $index_loc,
+ schema => $schema,
+ );
+ my $num_segmeta = num_segmeta($index_loc);
+ if ( $num_reps > 2 and $num_segmeta > 1 ) {
+ $indexer->optimize;
+ $indexer->commit;
+ $num_segmeta = num_segmeta($index_loc);
+ is( $num_segmeta, 1, 'commit after optimize' );
+ last;
+ }
+ else {
+ $indexer->add_doc( { content => $_ } ) for 1 .. 5;
+ $indexer->commit;
+ }
+ }
+}
+
+my @correct;
+for my $num_letters ( reverse 1 .. 10 ) {
+ my $truncate = $num_letters == 10 ? 1 : 0;
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $index_loc,
+ truncate => $truncate,
+ );
+
+ for my $letter ( 'a' .. 'b' ) {
+ my $content = ( "$letter " x $num_letters ) . ( 'z ' x 50 );
+ $indexer->add_doc( { content => $content } );
+ push @correct, $content if $letter eq 'b';
+ }
+ $indexer->commit;
+}
+
+{
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $index_loc );
+ my $hits = $searcher->hits( query => 'b' );
+ is( $hits->total_hits, 10, "correct total_hits from merged index" );
+ my @got;
+ push @got, $hits->next->{content} for 1 .. $hits->total_hits;
+ is_deeply( \@got, \@correct,
+ "correct top scoring hit from merged index" );
+}
+
+{
+ # Reopen index under BiggerSchema and add some content.
+ my $schema = BiggerSchema->new;
+ my $folder = Lucy::Store::FSFolder->new( path => $index_loc );
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ manager => NonMergingIndexManager->new,
+ );
+ $indexer->add_doc( { aux => 'foo', content => 'bar' } );
+
+ # Now add some indexes.
+ my $another_folder = create_index( "atlantic ocean", "fresh fish" );
+ my $yet_another_folder = create_index("bonus");
+ $indexer->add_index($another_folder);
+ $indexer->add_index($yet_another_folder);
+ $indexer->commit;
+ cmp_ok( num_segmeta($index_loc), '>', 1,
+ "non-merging Indexer should produce multi-seg index" );
+}
+
+{
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $index_loc );
+ my $hits = $searcher->hits( query => 'fish' );
+ is( $hits->total_hits, 1, "correct total_hits after add_index" );
+ is( $hits->next->{content},
+ 'fresh fish', "other indexes successfully absorbed" );
+}
+
+{
+ # Open an IndexReader, to prevent the deletion of files on Windows and
+ # verify the file purging mechanism.
+ my $schema = BiggerSchema->new;
+ my $folder = Lucy::Store::FSFolder->new( path => $index_loc );
+ my $reader = Lucy::Index::IndexReader->open( index => $folder );
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->optimize;
+ $indexer->commit;
+ $reader->close;
+ undef $reader;
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->optimize;
+ $indexer->commit;
+}
+
+is( num_segmeta($index_loc), 1, "merged segment files successfully deleted" );
+
+{
+ my $folder = Lucy::Store::RAMFolder->new;
+ my $schema = Lucy::Test::TestSchema->new;
+ my $number = 1;
+ for ( 1 .. 3 ) {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ manager => NonMergingIndexManager->new,
+ );
+ $indexer->add_doc( { content => $number++ } ) for 1 .. 20;
+ $indexer->commit;
+ }
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ $indexer->delete_by_term( field => 'content', term => $_ )
+ for ( 3, 23, 24, 25 );
+ $indexer->commit;
+
+ ok( $folder->exists("seg_1/segmeta.json"),
+ "Segment with under 10% deletions preserved"
+ );
+ ok( !$folder->exists("seg_2/segmeta.json"),
+ "Segment with over 10% deletions merged away"
+ );
+}
+
+is( Lucy::Store::FileHandle::object_count(),
+ 0, "All FileHandle objects have been cleaned up" );
+
+sub num_segmeta {
+ my $dir = shift;
+ my $num_segmeta = 0;
+ find(
+ { no_chdir => 1,
+ wanted =>
+ sub { $num_segmeta++ if $File::Find::name =~ /segmeta.json/ },
+ },
+ $dir,
+ );
+ return $num_segmeta;
+}
diff --git a/perl/t/214-spec_field.t b/perl/t/214-spec_field.t
new file mode 100644
index 0000000..bd19898
--- /dev/null
+++ b/perl/t/214-spec_field.t
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+
+package PolyAnalyzerSpec;
+use base qw( Lucy::Plan::FullTextType );
+sub analyzer { Lucy::Analysis::PolyAnalyzer->new( language => 'en' ) }
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new( language => 'en' );
+ my $plain = Lucy::Plan::FullTextType->new( analyzer => $tokenizer, );
+ my $polyanalyzed
+ = Lucy::Plan::FullTextType->new( analyzer => $polyanalyzer );
+ my $string_spec = Lucy::Plan::StringType->new;
+ my $unindexedbutanalyzed = Lucy::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ indexed => 0,
+ );
+ my $unanalyzedunindexed = Lucy::Plan::StringType->new( indexed => 0, );
+ $self->spec_field( name => 'analyzed', type => $plain );
+ $self->spec_field( name => 'polyanalyzed', type => $polyanalyzed );
+ $self->spec_field( name => 'string', type => $string_spec );
+ $self->spec_field(
+ name => 'unindexedbutanalyzed',
+ type => $unindexedbutanalyzed
+ );
+ $self->spec_field(
+ name => 'unanalyzedunindexed',
+ type => $unanalyzedunindexed
+ );
+ return $self;
+}
+
+package main;
+use Test::More tests => 10;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+$indexer->add_doc( { $_ => 'United States' } ) for qw(
+ analyzed
+ polyanalyzed
+ string
+ unindexedbutanalyzed
+ unanalyzedunindexed
+);
+
+$indexer->commit;
+
+sub check {
+ my ( $field, $query_text, $expected_num_hits ) = @_;
+ my $query = Lucy::Search::TermQuery->new(
+ field => $field,
+ term => $query_text,
+ );
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+ my $hits = $searcher->hits( query => $query );
+
+ is( $hits->total_hits, $expected_num_hits, "$field correct num hits " );
+
+ # Don't check the contents of the hit if there aren't any.
+ return unless $expected_num_hits;
+
+ my $hit = $hits->next;
+ is( $hit->{$field}, 'United States', "$field correct doc returned" );
+}
+
+check( 'analyzed', 'States', 1 );
+check( 'polyanalyzed', 'state', 1 );
+check( 'string', 'United States', 1 );
+check( 'unindexedbutanalyzed', 'state', 0 );
+check( 'unindexedbutanalyzed', 'United States', 0 );
+check( 'unanalyzedunindexed', 'state', 0 );
+check( 'unanalyzedunindexed', 'United States', 0 );
diff --git a/perl/t/215-term_vectors.t b/perl/t/215-term_vectors.t
new file mode 100644
index 0000000..596d1ad
--- /dev/null
+++ b/perl/t/215-term_vectors.t
@@ -0,0 +1,104 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new,
+ highlightable => 1,
+ );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package main;
+use utf8;
+use Test::More tests => 5;
+use Storable qw( freeze thaw );
+
+my $schema = MySchema->new;
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+my $hasta = 'hasta la mañana';
+for ( 'a b c foo foo bar', $hasta ) {
+ $indexer->add_doc( { content => $_ } );
+}
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $doc_vec = $searcher->fetch_doc_vec(1);
+
+my $term_vector = $doc_vec->term_vector( field => "content", term => "foo" );
+ok( defined $term_vector, "successfully retrieved term vector" );
+
+$doc_vec = $searcher->fetch_doc_vec(2);
+$term_vector = $doc_vec->term_vector( field => 'content', term => 'mañana' );
+
+ok( defined $term_vector, "utf-8 term vector retrieved" );
+is( $term_vector->get_end_offsets->get(0),
+ length $hasta,
+ "end offset in utf8 characters, not bytes"
+);
+
+# Reopen the Folder under a new Schema with two fields. The new field ("aux")
+# sorts lexically before "content" so that "content" will have a new field
+# num. This tests the field num mapping during merging.
+my $alt_folder = Lucy::Store::RAMFolder->new;
+my $alt_schema = MySchema->new;
+my $type = $alt_schema->fetch_type('content');
+$alt_schema->spec_field( name => 'aux', type => $type );
+
+$indexer = Lucy::Index::Indexer->new(
+ schema => $alt_schema,
+ index => $alt_folder,
+);
+for ( 'blah blah blah ', 'yada yada yada ' ) {
+ $indexer->add_doc(
+ { content => $_,
+ aux => $_ . $_,
+ }
+ );
+}
+$indexer->commit;
+
+$indexer = Lucy::Index::Indexer->new(
+ schema => $alt_schema,
+ index => $alt_folder,
+);
+$indexer->add_index($folder);
+$indexer->commit;
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $alt_folder );
+my $hits = $searcher->hits( query => $hasta );
+my $hit_id = $hits->next->get_doc_id;
+$doc_vec = $searcher->fetch_doc_vec($hit_id);
+$term_vector = $doc_vec->term_vector( field => 'content', term => 'mañana' );
+ok( defined $term_vector, "utf-8 term vector retrieved after merge" );
+
+my $dupe = thaw( freeze($term_vector) );
+ok( $term_vector->equals($dupe), "freeze/thaw" );
diff --git a/perl/t/216-schema.t b/perl/t/216-schema.t
new file mode 100644
index 0000000..f18ead4
--- /dev/null
+++ b/perl/t/216-schema.t
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+package main;
+use Test::More tests => 2;
+
+my $schema;
+SKIP: {
+ skip( "constructor bailouts cause leaks", 1 ) if $ENV{LUCY_VALGRIND};
+
+ $schema = Lucy::Test::TestSchema->new;
+ eval { $schema->spec_field( name => 'foo', type => 'NotAType' ) };
+ Test::More::like( $@, qr/FieldType/, "bogus FieldType fails to load" );
+}
+
+$schema = Lucy::Test::TestSchema->new;
+my $type = $schema->fetch_type('content');
+$schema->spec_field( name => 'new_field', type => $type );
+my $got = grep { $_ eq 'new_field' } @{ $schema->all_fields };
+ok( $got, 'spec_field works' );
+
diff --git a/perl/t/217-poly_lexicon.t b/perl/t/217-poly_lexicon.t
new file mode 100644
index 0000000..92dec0a
--- /dev/null
+++ b/perl/t/217-poly_lexicon.t
@@ -0,0 +1,121 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+use strict;
+use warnings;
+
+package MyArchitecture;
+use base qw( Lucy::Plan::Architecture );
+
+sub index_interval { confess("should be displaced via local() below") }
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::StringType->new( sortable => 1 );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+sub architecture { MyArchitecture->new }
+
+package main;
+use Test::More tests => 35;
+use File::Spec::Functions qw( catfile );
+
+my @docs = ( ( ("a") x 100 ), 'j', "Foo" );
+my @chars = ( 'a' .. 'z' );
+for ( 0 .. 100 ) {
+ my $content = '';
+ for my $num_chars ( 1 .. int( rand(10) + 1 ) ) {
+ $content .= @chars[ rand(@chars) ];
+ }
+ push @docs, "$content";
+}
+
+# Accumulate unique sorted terms.
+my @correct = ();
+for ( sort @docs ) {
+ next if ( @correct and $_ eq $correct[-1] );
+ push @correct, $_;
+}
+
+# Remember where 'j' exists in our 'correct' list.
+my $correct_term_num = 0;
+for (@correct) {
+ last if $correct[$correct_term_num] eq 'j';
+ $correct_term_num++;
+}
+
+for my $index_interval ( 1, 2, 3, 4, 7, 128, 1024 ) {
+
+ no warnings 'redefine';
+ local *MyArchitecture::index_interval = sub {$index_interval};
+
+ my $folder = Lucy::Store::RAMFolder->new;
+ my $schema = MySchema->new;
+
+ my @docs_copy = @docs;
+ while (@docs_copy) {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ my $docs_this_seg = int( rand(@docs_copy) );
+ $docs_this_seg = 10 if $docs_this_seg < 10;
+
+ for ( 0 .. 10 ) {
+ last unless @docs_copy;
+ my $tick = int( rand(@docs_copy) );
+ $indexer->add_doc(
+ { content => splice( @docs_copy, $tick, 1 ) } );
+ }
+ $indexer->commit;
+ }
+
+ my $reader = Lucy::Index::IndexReader->open( index => $folder, );
+
+ my $lexicon = $reader->obtain("Lucy::Index::LexiconReader")
+ ->lexicon( field => 'content' );
+ isa_ok( $lexicon, "Lucy::Index::PolyLexicon" );
+
+ $lexicon->next;
+ is( $lexicon->get_term, $correct[0],
+ "calling lexicon() without term returns Lexicon with iterator reset"
+ );
+ $lexicon->reset;
+
+ my @got;
+ while ( $lexicon->next ) {
+ push @got, $lexicon->get_term;
+ }
+
+ is_deeply( \@got, \@correct,
+ "correct order for random strings (interval: $index_interval)" );
+
+ $lexicon->seek('j');
+ is( $lexicon->get_term, 'j', "seek (interval: $index_interval)" );
+
+ $lexicon->seek(undef);
+ ok( !defined $lexicon->get_term, "seek to undef resets" );
+}
diff --git a/perl/t/218-del_merging.t b/perl/t/218-del_merging.t
new file mode 100644
index 0000000..ff3ca56
--- /dev/null
+++ b/perl/t/218-del_merging.t
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+
+package NoMergeSeg1Manager;
+use base qw( Lucy::Index::IndexManager );
+sub recycle {
+ my $seg_readers = shift->SUPER::recycle(@_);
+ @$seg_readers = grep { $_->get_seg_num != 1 } @$seg_readers;
+ return $seg_readers;
+}
+
+package DelSchema;
+use base 'Lucy::Plan::Schema';
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new, );
+ $self->spec_field( name => 'foo', type => $type );
+ $self->spec_field( name => 'bar', type => $type );
+ return $self;
+}
+
+package main;
+
+use Test::More tests => 70;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = DelSchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { foo => 'foo', bar => $_ } ) for qw( x y z );
+$indexer->commit;
+
+for my $iter ( 1 .. 10 ) {
+ is( search_doc('foo'), 3, "match all docs prior to deletion $iter" );
+ is( search_doc('x'), 1, "match doc to be deleted $iter" );
+
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->delete_by_term( field => 'bar', term => 'x' );
+ $indexer->optimize;
+ $indexer->commit;
+
+ is( search_doc('x'), 0, "deletion successful $iter" );
+ is( search_doc('foo'), 2, "match all docs after deletion $iter" );
+
+ $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->add_doc( { foo => 'foo', bar => 'x' } );
+ $indexer->optimize;
+ $indexer->commit;
+}
+
+$folder = Lucy::Store::RAMFolder->new;
+$schema = DelSchema->new;
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+my @dox = ( 'a' .. 'z' );
+$indexer->add_doc( { foo => 'foo', bar => $_ } ) for @dox;
+$indexer->commit;
+
+for ( 1 .. 10 ) {
+ $indexer = Lucy::Index::Indexer->new(
+ manager => NoMergeSeg1Manager->new,
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->delete_by_term( field => 'bar', term => 'y' );
+ $indexer->commit;
+ my @num_seg_1_bv_files = grep {/deletions-seg_1.bv/} @{ $folder->list_r };
+ is( scalar @num_seg_1_bv_files,
+ 1, "seg_1 deletions file carried forward" );
+
+ $indexer = Lucy::Index::Indexer->new(
+ manager => NoMergeSeg1Manager->new,
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->delete_by_term( field => 'bar', term => 'new' );
+ $indexer->add_doc( { foo => 'foo', bar => 'new' } );
+ $indexer->commit;
+ @num_seg_1_bv_files = grep {/deletions-seg_1.bv/} @{ $folder->list_r };
+ is( scalar @num_seg_1_bv_files,
+ 1, "seg_1 deletions file carried forward" );
+
+ is( search_doc('foo'), scalar @dox, "deletions applied correctly" );
+}
+
+sub search_doc {
+ my $query_string = shift;
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+ my $hits = $searcher->hits( query => $query_string );
+ return $hits->total_hits;
+}
diff --git a/perl/t/219-byte_buf_doc.t b/perl/t/219-byte_buf_doc.t
new file mode 100644
index 0000000..bd95461
--- /dev/null
+++ b/perl/t/219-byte_buf_doc.t
@@ -0,0 +1,114 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MyArchitecture;
+use base qw( Lucy::Plan::Architecture );
+
+use LucyX::Index::ByteBufDocWriter;
+use LucyX::Index::ByteBufDocReader;
+
+sub register_doc_writer {
+ my ( $self, $seg_writer ) = @_;
+ my $doc_writer = LucyX::Index::ByteBufDocWriter->new(
+ width => 1,
+ field => 'value',
+ schema => $seg_writer->get_schema,
+ snapshot => $seg_writer->get_snapshot,
+ segment => $seg_writer->get_segment,
+ polyreader => $seg_writer->get_polyreader,
+ );
+ $seg_writer->register(
+ api => "Lucy::Index::DocReader",
+ component => $doc_writer,
+ );
+ $seg_writer->add_writer($doc_writer);
+}
+
+sub register_doc_reader {
+ my ( $self, $seg_reader ) = @_;
+ my $doc_reader = LucyX::Index::ByteBufDocReader->new(
+ width => 1,
+ field => 'value',
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ segments => $seg_reader->get_segments,
+ seg_tick => $seg_reader->get_seg_tick,
+ snapshot => $seg_reader->get_snapshot,
+ );
+ $seg_reader->register(
+ api => 'Lucy::Index::DocReader',
+ component => $doc_reader,
+ );
+}
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub architecture { MyArchitecture->new }
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $type = Lucy::Plan::FullTextType->new( analyzer => $tokenizer );
+ $self->spec_field( name => 'value', type => $type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 4;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+
+sub add_to_index {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ $indexer->add_doc( { value => $_ } ) for @_;
+ $indexer->commit;
+}
+
+add_to_index(qw( a b c ));
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'b' );
+is( $hits->next->{value}, 'b', "single segment, single hit" );
+
+add_to_index(qw( d e f g h ));
+add_to_index(qw( i j k l m ));
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'f' );
+is( $hits->next->{value}, 'f', "multiple segments, single hit" );
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'value', term => $_ ) for qw( b f l );
+$indexer->optimize;
+$indexer->commit;
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'b' );
+is( $hits->next, undef, "doc deleted" );
+
+$hits = $searcher->hits( query => 'c' );
+is( $hits->next->{value}, 'c', "map around deleted doc" );
diff --git a/perl/t/220-zlib_doc.t b/perl/t/220-zlib_doc.t
new file mode 100644
index 0000000..40c8fa7
--- /dev/null
+++ b/perl/t/220-zlib_doc.t
@@ -0,0 +1,133 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MyArchitecture;
+use base qw( Lucy::Plan::Architecture );
+
+use LucyX::Index::ZlibDocWriter;
+use LucyX::Index::ZlibDocReader;
+
+sub register_doc_writer {
+ my ( $self, $seg_writer ) = @_;
+ my $doc_writer = LucyX::Index::ZlibDocWriter->new(
+ schema => $seg_writer->get_schema,
+ snapshot => $seg_writer->get_snapshot,
+ segment => $seg_writer->get_segment,
+ polyreader => $seg_writer->get_polyreader,
+ );
+ $seg_writer->register(
+ api => "Lucy::Index::DocReader",
+ component => $doc_writer,
+ );
+ $seg_writer->add_writer($doc_writer);
+}
+
+sub register_doc_reader {
+ my ( $self, $seg_reader ) = @_;
+ my $doc_reader = LucyX::Index::ZlibDocReader->new(
+ schema => $seg_reader->get_schema,
+ folder => $seg_reader->get_folder,
+ segments => $seg_reader->get_segments,
+ seg_tick => $seg_reader->get_seg_tick,
+ snapshot => $seg_reader->get_snapshot,
+ );
+ $seg_reader->register(
+ api => 'Lucy::Index::DocReader',
+ component => $doc_reader,
+ );
+}
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub architecture { MyArchitecture->new }
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $main_type = Lucy::Plan::FullTextType->new( analyzer => $tokenizer );
+ my $unstored_type = Lucy::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ stored => 0,
+ );
+ my $blob_type = Lucy::Plan::BlobType->new( stored => 1 );
+ $self->spec_field( name => 'content', type => $main_type );
+ $self->spec_field( name => 'smiley', type => $main_type );
+ $self->spec_field( name => 'unstored', type => $unstored_type );
+ $self->spec_field( name => 'binary', type => $blob_type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 7;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+
+my $smiley = "\x{263a}";
+my $binary = pack( 'b4', 1, 2, 3, 4 );
+
+sub add_to_index {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+ for (@_) {
+ $indexer->add_doc(
+ { content => $_,
+ binary => $binary,
+ smiley => $smiley,
+ unstored => $_,
+ }
+ );
+ }
+ $indexer->commit;
+}
+
+add_to_index(qw( a b c ));
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'b' );
+my $hit = $hits->next;
+is( $hit->{content}, 'b', "single segment, single hit" );
+is( $hit->{smiley}, $smiley, "utf8 preserved" );
+is( $hit->{binary}, $binary, "blob field binary preserved" );
+ok( !defined( $hit->{unstored} ), "unstored" );
+
+add_to_index(qw( d e f g h ));
+add_to_index(qw( i j k l m ));
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'f' );
+is( $hits->next->{content}, 'f', "multiple segments, single hit" );
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $_ ) for qw( b f l );
+$indexer->optimize;
+$indexer->commit;
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'b' );
+is( $hits->next, undef, "doc deleted" );
+
+$hits = $searcher->hits( query => 'c' );
+is( $hits->next->{content}, 'c', "map around deleted doc" );
diff --git a/perl/t/221-sort_writer.t b/perl/t/221-sort_writer.t
new file mode 100644
index 0000000..59ee1b1
--- /dev/null
+++ b/perl/t/221-sort_writer.t
@@ -0,0 +1,174 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package NonMergingIndexManager;
+use base qw( Lucy::Index::IndexManager );
+
+sub recycle {
+ return Lucy::Object::VArray->new( capacity => 0 );
+}
+
+package SortSchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $fulltext_type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new,
+ sortable => 1,
+ );
+ my $string_type = Lucy::Plan::StringType->new( sortable => 1 );
+ my $unsortable = Lucy::Plan::StringType->new;
+ $self->spec_field( name => 'name', type => $fulltext_type );
+ $self->spec_field( name => 'speed', type => $string_type );
+ $self->spec_field( name => 'weight', type => $string_type );
+ $self->spec_field( name => 'home', type => $string_type );
+ $self->spec_field( name => 'cat', type => $string_type );
+ $self->spec_field( name => 'wheels', type => $string_type );
+ $self->spec_field( name => 'unused', type => $string_type );
+ $self->spec_field( name => 'nope', type => $unsortable );
+ return $self;
+}
+
+package main;
+use Lucy::Test;
+use Test::More tests => 57;
+
+# Force frequent flushes.
+Lucy::Index::SortWriter::set_default_mem_thresh(100);
+
+my $airplane = {
+ name => 'airplane',
+ speed => '0200',
+ weight => '8000',
+ home => 'air',
+ cat => 'vehicle',
+ wheels => 3,
+ nope => 'nyet',
+};
+my $bike = {
+ name => 'bike',
+ speed => '0015',
+ weight => '0025',
+ home => 'land',
+ cat => 'vehicle',
+ wheels => 2,
+};
+my $car = {
+ name => 'car',
+ speed => '0070',
+ weight => '3000',
+ home => 'land',
+ cat => 'vehicle',
+ wheels => 4,
+};
+my $dirigible = {
+ name => 'dirigible',
+ speed => '0040',
+ weight => '0000',
+ home => 'air',
+ cat => 'vehicle',
+ # no "wheels" field -- test NULL/undef
+};
+my $elephant = {
+ name => 'elephant',
+ speed => '0020',
+ weight => '6000',
+ home => 'land',
+ cat => 'vehicle',
+ # no "wheels" field -- test NULL/undef
+};
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = SortSchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+# Add vehicles.
+$indexer->add_doc($_) for ( $airplane, $bike, $car );
+
+$indexer->commit;
+
+my $polyreader = Lucy::Index::IndexReader->open( index => $folder );
+my $seg_reader = $polyreader->get_seg_readers->[0];
+my $sort_reader = $seg_reader->obtain("Lucy::Index::SortReader");
+my $doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+my $segment = $seg_reader->get_segment;
+
+for my $field (qw( name speed weight home cat wheels )) {
+ my $field_num = $segment->field_num($field);
+ ok( $folder->exists("seg_1/sort-$field_num.ord"),
+ "sort files written for $field" );
+ my $sort_cache = $sort_reader->fetch_sort_cache($field);
+ for ( 1 .. $seg_reader->doc_max ) {
+ is( $sort_cache->value( ord => $sort_cache->ordinal($_) ),
+ $doc_reader->fetch_doc($_)->{$field},
+ "correct cached value doc $_ "
+ );
+ }
+}
+
+for my $field (qw( unused nope )) {
+ my $field_num = $segment->field_num($field);
+ ok( !$folder->exists("seg_1/sort-$field_num.ord"),
+ "no sort files written for $field" );
+}
+
+# Add a second segment.
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ manager => NonMergingIndexManager->new,
+);
+$indexer->add_doc($dirigible);
+$indexer->commit;
+
+# Consolidate everything, to test merging.
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'name', term => 'bike' );
+$indexer->add_doc($elephant);
+$indexer->optimize;
+$indexer->commit;
+
+my $num_old_seg_files = scalar grep {m/seg_[12]/} @{ $folder->list_r };
+is( $num_old_seg_files, 0, "all files from earlier segments zapped" );
+
+$polyreader = Lucy::Index::IndexReader->open( index => $folder );
+$seg_reader = $polyreader->get_seg_readers->[0];
+$sort_reader = $seg_reader->obtain("Lucy::Index::SortReader");
+$doc_reader = $seg_reader->obtain("Lucy::Index::DocReader");
+$segment = $seg_reader->get_segment;
+
+for my $field (qw( name speed weight home cat wheels )) {
+ my $field_num = $segment->field_num($field);
+ ok( $folder->exists("seg_3/sort-$field_num.ord"),
+ "sort files written for $field" );
+ my $sort_cache = $sort_reader->fetch_sort_cache($field);
+ for ( 1 .. $seg_reader->doc_max ) {
+ is( $sort_cache->value( ord => $sort_cache->ordinal($_) ),
+ $doc_reader->fetch_doc($_)->{$field},
+ "correct cached value field $field doc $_ "
+ );
+ }
+}
diff --git a/perl/t/224-lex_reader.t b/perl/t/224-lex_reader.t
new file mode 100644
index 0000000..eb38131
--- /dev/null
+++ b/perl/t/224-lex_reader.t
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 3;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index(
+ "What's he building in there?",
+ "What's he building in there?",
+ "We have a right to know."
+);
+my $polyreader = Lucy::Index::IndexReader->open( index => $folder );
+my $seg_reader = $polyreader->get_seg_readers->[0];
+my $lex_reader = $seg_reader->obtain("Lucy::Index::LexiconReader");
+
+my $lexicon = $lex_reader->lexicon( field => 'content', term => 'building' );
+isa_ok( $lexicon, 'Lucy::Index::Lexicon',
+ "lexicon returns a Lucy::Index::Lexicon" );
+my $tinfo = $lexicon->get_term_info;
+is( $tinfo->get_doc_freq, 2, "correct place in lexicon" );
+
+$lexicon = $lex_reader->lexicon( field => 'content' );
+$lexicon->next;
+is( $lexicon->get_term, 'We',
+ 'calling lexicon without a term returns Lexicon with iterator reset' );
+
diff --git a/perl/t/233-background_merger.t b/perl/t/233-background_merger.t
new file mode 100644
index 0000000..f6e5da9
--- /dev/null
+++ b/perl/t/233-background_merger.t
@@ -0,0 +1,89 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package NoMergeManager;
+use base qw( Lucy::Index::IndexManager );
+sub recycle { [] }
+
+package main;
+use Test::More tests => 15;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = Lucy::Test::TestSchema->new;
+
+for my $letter (qw( a b c )) {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ manager => NoMergeManager->new,
+ );
+ $indexer->add_doc( { content => $letter } );
+ $indexer->commit;
+}
+my $bg_merger = Lucy::Index::BackgroundMerger->new( index => $folder );
+
+my $indexer = Lucy::Index::Indexer->new( index => $folder );
+$indexer->add_doc( { content => 'd' } );
+$indexer->commit;
+
+is( count_segs($folder), 4,
+ "BackgroundMerger prevents Indexer from merging claimed segments" );
+
+$indexer = Lucy::Index::Indexer->new( index => $folder );
+$indexer->add_doc( { content => 'e' } );
+$indexer->delete_by_term( field => 'content', term => 'b' );
+$indexer->commit;
+is( count_segs($folder), 4, "Indexer may still merge unclaimed segments" );
+
+$bg_merger->commit;
+is( count_segs($folder), 3, "Background merge completes" );
+ok( $folder->exists("seg_7/deletions-seg_4.bv"),
+ "deletions carried forward" );
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+is( $searcher->hits( query => 'b' )->total_hits,
+ 0, "deleted term still deleted" );
+is( $searcher->hits( query => $_ )->total_hits, 1, "term $_ still present" )
+ for qw( a c d e );
+
+# Simulate failed background merge.
+$bg_merger = Lucy::Index::BackgroundMerger->new( index => $folder );
+$bg_merger->prepare_commit;
+undef $bg_merger;
+$folder->delete("merge.lock");
+die "test set up failed" unless $folder->exists("merge.json");
+
+$indexer = Lucy::Index::Indexer->new( index => $folder );
+$indexer->optimize;
+$indexer->commit;
+
+ok( !$folder->exists("merge.json"), "Cleaned up after failed bg merge" );
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+is( $searcher->hits( query => 'b' )->total_hits,
+ 0, "deleted term still deleted after full optimize" );
+is( $searcher->hits( query => $_ )->total_hits,
+ 1, "term $_ still present after full optimize" )
+ for qw( a c d e );
+
+sub count_segs {
+ my $folder = shift;
+ return scalar grep {m/segmeta\.json/} @{ $folder->list_r };
+}
diff --git a/perl/t/302-many_fields.t b/perl/t/302-many_fields.t
new file mode 100644
index 0000000..a9e1a92
--- /dev/null
+++ b/perl/t/302-many_fields.t
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+our %fields = ();
+
+package main;
+
+use Test::More tests => 10;
+use Lucy::Test;
+
+my $schema = MySchema->new;
+my $type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new, );
+
+for my $num_fields ( 1 .. 10 ) {
+ # Build an index with $num_fields fields, and the same content in each.
+ $schema->spec_field( name => "field$num_fields", type => $type );
+ my $folder = Lucy::Store::RAMFolder->new;
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+
+ for my $content ( 'a' .. 'z', 'x x y' ) {
+ my %doc;
+ for ( 1 .. $num_fields ) {
+ $doc{"field$_"} = $content;
+ }
+ $indexer->add_doc( \%doc );
+ }
+ $indexer->commit;
+
+ # See if our search results match as expected.
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+ my $hits = $searcher->hits(
+ query => 'x',
+ num_wanted => 100,
+ );
+ is( $hits->total_hits, 2,
+ "correct number of hits for $num_fields fields" );
+ my $top_hit = $hits->next;
+}
diff --git a/perl/t/303-highlighter.t b/perl/t/303-highlighter.t
new file mode 100644
index 0000000..e7628f1
--- /dev/null
+++ b/perl/t/303-highlighter.t
@@ -0,0 +1,418 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $class = shift;
+ my $self = $class->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $plain_type = Lucy::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ highlightable => 1,
+ );
+ my $dunked_type = Lucy::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ highlightable => 1,
+ boost => 0.1,
+ );
+ $self->spec_field( name => 'content', type => $plain_type );
+ $self->spec_field( name => 'alt', type => $dunked_type );
+ return $self;
+}
+
+package MyHighlighter;
+use base qw( Lucy::Highlight::Highlighter );
+
+sub encode {
+ my ( $self, $text ) = @_;
+ $text =~ s/blind/wise/;
+ return $text;
+}
+
+sub highlight {
+ my ( $self, $text ) = @_;
+ return "*$text*";
+}
+
+package main;
+
+use Test::More tests => 35;
+use Lucy::Test;
+
+binmode( STDOUT, ":utf8" );
+
+my $phi = "\x{03a6}";
+my $encoded_phi = "Φ";
+
+my $string = '1 2 3 4 5 ' x 20; # 200 characters
+$string .= "$phi a b c d x y z h i j k ";
+$string .= '6 7 8 9 0 ' x 20;
+my $with_quotes = '"I see," said the blind man.';
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => MySchema->new,
+);
+
+$indexer->add_doc( { content => $_ } ) for ( $string, $with_quotes );
+$indexer->add_doc(
+ { content => "x but not why or 2ee",
+ alt => $string . " and extra stuff so it scores lower",
+ }
+);
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $q = qq|"x y z" AND $phi|;
+my $hits = $searcher->hits( query => $q );
+my $hl = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ excerpt_length => 3,
+);
+
+my $target = Lucy::Object::ViewCharBuf->_new("");
+
+my $field_val = make_cb("a $phi $phi b c");
+my $top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 2, 1, 1.0 ] ),
+);
+is( $target->to_perl, "$phi $phi b", "Find_Best_Fragment" );
+is( $top, 2, "correct offset returned by Find_Best_Fragment" );
+
+$field_val = make_cb("aa$phi");
+$top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 2, 1, 1.0 ] ),
+);
+is( $target->to_perl, $field_val->to_perl,
+ "Find_Best_Fragment returns whole field when field is short" );
+is( $top, 0, "correct offset" );
+
+$field_val = make_cb("aaaab$phi$phi");
+$top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 6, 2, 1.0 ] ),
+);
+is( $target->to_perl, "b$phi$phi",
+ "Find_Best_Fragment shifts left to deal with overrun" );
+is( $top, 4, "correct offset" );
+
+$field_val = make_cb( "a$phi" . "bcde" );
+$top = $hl->_find_best_fragment(
+ fragment => $target,
+ field_val => $field_val,
+ heat_map => make_heat_map( [ 0, 1, 1.0 ] ),
+);
+is( $target->to_perl,
+ "a$phi" . "bcd",
+ "Find_Best_Fragment start at field beginning"
+);
+is( $top, 0, "correct offset" );
+undef $target;
+
+$hl = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ excerpt_length => 6,
+);
+
+$target = make_cb("");
+$field_val = "Ook. Urk. Ick. ";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => "Ook. Urk.",
+ raw_excerpt => $target,
+ top => 0,
+ sentences => make_spans( [ 0, 4, 0 ], [ 6, 4, 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "Ook.", "Raw_Excerpt at top" );
+is( $top, 0, "top still 0" );
+
+$target = make_cb("");
+$field_val = "Ook. Urk. Ick. ";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => ". Urk. I",
+ raw_excerpt => $target,
+ top => 3,
+ sentences => make_spans( [ 6, 4, 0 ], [ 12, 4, 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "Urk.", "Raw_Excerpt in middle, with 2 bounds" );
+is( $top, 6, "top in the middle modified by Raw_Excerpt" );
+
+$target = make_cb("");
+$field_val = "Ook urk ick i.";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => "ick i.",
+ raw_excerpt => $target,
+ top => 8,
+ sentences => make_spans( [ 0, length($field_val), 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "\x{2026} i.", "Ellipsis at top" );
+is( $top, 10, "top correct when leading ellipsis inserted" );
+
+$target = make_cb("");
+$field_val = "Urk. Iz no good.";
+$top = $hl->_raw_excerpt(
+ field_val => $field_val,
+ fragment => " Iz no go",
+ raw_excerpt => $target,
+ top => 4,
+ sentences => make_spans( [ 6, length($field_val) - 6, 0 ] ),
+ heat_map => make_heat_map( [ 0, length($field_val), 1.0 ] ),
+);
+is( $target->to_perl, "Iz no\x{2026}", "Ellipsis at end" );
+is( $top, 6, "top trimmed" );
+
+$hl = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ excerpt_length => 3,
+);
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => 'a b c',
+ spans => make_spans( [ 2, 1 ] ),
+ top => 0,
+ highlighted => $target,
+);
+is( $target->to_perl, "a <strong>b</strong> c", "basic Highlight_Excerpt" );
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => "$phi",
+ spans => make_spans( [ 0, 1, 1.0 ], [ 10, 10, 1.0 ] ),
+ top => 0,
+ highlighted => $target,
+);
+unlike( $target->to_perl, qr#<strong></strong>#,
+ "don't surround spans off end of raw excerpt." );
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => "$phi $phi $phi",
+ spans => make_spans( [ 3, 1, 1.0 ] ),
+ top => 1,
+ highlighted => $target,
+);
+like(
+ $target->to_perl,
+ qr#^$encoded_phi <strong>$encoded_phi</strong> $encoded_phi$#i,
+ "Highlight_Excerpt pays attention to offset"
+);
+
+$target = make_cb("");
+$hl->_highlight_excerpt(
+ raw_excerpt => "$phi $phi $phi",
+ spans => make_spans( [ 3, 1, 1.0 ] ),
+ top => 1,
+ highlighted => $target,
+);
+like(
+ $target->to_perl,
+ qr#^$encoded_phi <strong>$encoded_phi</strong> $encoded_phi$#i,
+ "Highlight_Excerpt pays attention to offset"
+);
+
+$hl = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+);
+
+my $hit = $hits->next;
+my $excerpt = $hl->create_excerpt($hit);
+like( $excerpt, qr/$encoded_phi.*?z/i,
+ "excerpt contains all relevant terms" );
+like( $excerpt, qr#<strong>x y z</strong>#, "highlighter tagged the phrase" );
+like(
+ $excerpt,
+ qr#<strong>$encoded_phi</strong>#i,
+ "highlighter tagged the single term"
+);
+
+$hl->set_pre_tag("\e[1m");
+$hl->set_post_tag("\e[0m");
+like(
+ $hl->create_excerpt($hit),
+ qr#\e\[1m$encoded_phi\e\[0m#i, "set_pre_tag and set_post_tag",
+);
+
+like( $hl->create_excerpt( $hits->next() ),
+ qr/x/,
+ "excerpt field with partial hit doesn't cause highlighter freakout" );
+
+$hits = $searcher->hits( query => $q = 'x "x y z" AND b' );
+$hl = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+);
+$excerpt = $hl->create_excerpt( $hits->next() );
+$excerpt =~ s#</?strong>##g;
+like( $excerpt, qr/x y z/,
+ "query with same word in both phrase and term doesn't cause freakout" );
+
+$hits = $searcher->hits( query => $q = 'blind' );
+like(
+ Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ )->create_excerpt( $hits->next() ),
+ qr/quot/,
+ "HTML entity encoded properly"
+);
+
+$hits = $searcher->hits( query => $q = 'why' );
+unlike(
+ Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+ )->create_excerpt( $hits->next() ),
+ qr/\.\.\./,
+ "no ellipsis for short excerpt"
+);
+
+my $term_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'x',
+);
+$hits = $searcher->hits( query => $term_query );
+$hit = $hits->next();
+like(
+ Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $term_query,
+ field => 'content',
+ )->create_excerpt($hit),
+ qr/strong/,
+ "specify field highlights correct field..."
+);
+unlike(
+ Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $term_query,
+ field => 'alt',
+ )->create_excerpt($hit),
+ qr/strong/,
+ "... but not another field"
+);
+
+my $sentence_text = 'This is a sentence. ' x 15;
+$hl = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $q,
+ field => 'content',
+);
+my $sentences = $hl->find_sentences(
+ text => $sentence_text,
+ offset => 101,
+ length => 50,
+);
+is_deeply(
+ spans_to_arg_array($sentences),
+ [ [ 120, 19, 0 ], [ 140, 19, 0 ] ],
+ 'find_sentences with explicit args'
+);
+
+$sentences = $hl->find_sentences(
+ text => $sentence_text,
+ offset => 101,
+ length => 4,
+);
+is_deeply( spans_to_arg_array($sentences),
+ [], 'find_sentences with explicit args, finding nothing' );
+
+my @expected;
+for my $i ( 0 .. 14 ) {
+ push @expected, [ $i * 20, 19, 0 ];
+}
+$sentences = $hl->find_sentences( text => $sentence_text );
+is_deeply( spans_to_arg_array($sentences),
+ \@expected, 'find_sentences with default offset and length' );
+
+$sentences = $hl->find_sentences( text => ' Foo' );
+is_deeply(
+ spans_to_arg_array($sentences),
+ [ [ 1, 3, 0 ] ],
+ "Skip leading whitespace but get first sentence"
+);
+
+$hl = MyHighlighter->new(
+ searcher => $searcher,
+ query => "blind",
+ field => 'content',
+);
+$hits = $searcher->hits( query => 'blind' );
+$hit = $hits->next;
+like( $hl->create_excerpt($hit),
+ qr/\*wise\*/, "override both Encode() and Highlight()" );
+
+sub make_cb {
+ return Lucy::Object::CharBuf->new(shift);
+}
+
+sub make_heat_map {
+ return Lucy::Highlight::HeatMap->new( spans => make_spans(@_) );
+}
+
+sub make_span {
+ return Lucy::Search::Span->new(
+ offset => $_[0],
+ length => $_[1],
+ weight => $_[2],
+ );
+}
+
+sub make_spans {
+ my $spans = Lucy::Object::VArray->new( capacity => scalar @_ );
+ for my $span_spec (@_) {
+ $spans->push( make_span( @{$span_spec}[ 0 .. 2 ] ) );
+ }
+ return $spans;
+}
+
+sub spans_to_arg_array {
+ my $spans = shift;
+ my @out;
+ for (@$spans) {
+ push @out, [ $_->get_offset, $_->get_length, $_->get_weight ];
+ }
+ return \@out;
+}
diff --git a/perl/t/304-verify_utf8.t b/perl/t/304-verify_utf8.t
new file mode 100644
index 0000000..7d3b8e5
--- /dev/null
+++ b/perl/t/304-verify_utf8.t
@@ -0,0 +1,117 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $analyzer = Lucy::Analysis::RegexTokenizer->new( pattern => '\S+' );
+ my $type = Lucy::Plan::FullTextType->new( analyzer => $analyzer, );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 14;
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( utf8_test_strings );
+
+my ( $smiley, $not_a_smiley, $frowny ) = utf8_test_strings();
+
+my $turd = pack( 'C*', 254, 254 );
+my $polished_turd = $turd;
+utf8::upgrade($polished_turd);
+
+is( $turd, $polished_turd, "verify encoding acrobatics" );
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+$indexer->add_doc( { content => $smiley } );
+$indexer->add_doc( { content => $not_a_smiley } );
+$indexer->add_doc( { content => $turd } );
+$indexer->commit;
+
+my $qparser = Lucy::Search::QueryParser->new( schema => MySchema->new );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits( query => $qparser->parse($smiley) );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content},
+ $smiley, "Indexer and QueryParser handle UTF-8 source correctly" );
+
+$hits = $searcher->hits( query => $qparser->parse($frowny) );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content}, $frowny, "Indexer upgrades non-UTF-8 correctly" );
+
+$hits = $searcher->hits( query => $qparser->parse($not_a_smiley) );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content},
+ $not_a_smiley, "QueryParser upgrades non-UTF-8 correctly" );
+
+my $term_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => $not_a_smiley,
+);
+$hits = $searcher->hits( query => $term_query );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content},
+ $not_a_smiley, "TermQuery upgrades non-UTF-8 correctly" );
+
+$term_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => $smiley,
+);
+
+$hits = $searcher->hits( query => $term_query );
+is( $hits->total_hits, 1 );
+is( $hits->next->{content}, $smiley, "TermQuery handles UTF-8 correctly" );
+
+undef $indexer;
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $smiley );
+$indexer->commit;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+$hits = $searcher->hits( query => $smiley );
+is( $hits->total_hits, 0, "delete_by_term handles UTF-8 correctly" );
+
+$hits = $searcher->hits( query => $frowny );
+is( $hits->total_hits, 1, "delete_by_term handles UTF-8 correctly" );
+
+undef $indexer;
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->delete_by_term( field => 'content', term => $not_a_smiley );
+$indexer->commit;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+$hits = $searcher->hits( query => $frowny );
+is( $hits->total_hits, 0, "delete_by_term upgrades non-UTF-8 correctly" );
diff --git a/perl/t/305-indexer.t b/perl/t/305-indexer.t
new file mode 100644
index 0000000..8eed70e
--- /dev/null
+++ b/perl/t/305-indexer.t
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 4;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = Lucy::Test::TestSchema->new;
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+$indexer->add_doc( { content => 'foo' } );
+undef $indexer;
+
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { content => 'foo' } );
+pass("Indexer ignores garbage from interrupted session");
+
+SKIP: {
+ skip( "Known leak, though might be fixable", 2 ) if $ENV{LUCY_VALGRIND};
+ eval {
+ my $manager
+ = Lucy::Index::IndexManager->new( host => 'somebody_else' );
+ my $inv = Lucy::Index::Indexer->new(
+ manager => $manager,
+ index => $folder,
+ schema => $schema,
+ );
+ };
+ like( $@, qr/write.lock/, "failed to get lock with competing host" );
+ isa_ok( $@, "Lucy::Store::LockErr", "Indexer throws a LockErr" );
+}
+
+my $pid = 12345678;
+do {
+ # Fake a write lock.
+ $folder->delete("locks/write.lock") or die "Couldn't delete 'write.lock'";
+ my $outstream = $folder->open_out('locks/write.lock')
+ or die Lucy->error;
+ while ( kill( 0, $pid ) ) {
+ $pid++;
+ }
+ $outstream->print(
+ qq|
+ {
+ "host": "somebody_else",
+ "pid": $pid,
+ "name": "write"
+ }|
+ );
+ $outstream->close;
+
+ eval {
+ my $manager
+ = Lucy::Index::IndexManager->new( host => 'somebody_else' );
+ my $inv = Lucy::Index::Indexer->new(
+ manager => $manager,
+ schema => $schema,
+ index => $folder,
+ );
+ };
+
+ # Mitigate (though not eliminate) race condition false failure.
+} while ( kill( 0, $pid ) );
+
+ok( !$@, "clobbered lock from same host with inactive pid" );
diff --git a/perl/t/306-dynamic_schema.t b/perl/t/306-dynamic_schema.t
new file mode 100644
index 0000000..76ac178
--- /dev/null
+++ b/perl/t/306-dynamic_schema.t
@@ -0,0 +1,99 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use Lucy::Test;
+
+my $schema = Lucy::Test::TestSchema->new;
+my $type = $schema->fetch_type('content');
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+
+my %one_field = ( content => 'x 1' );
+my %two_fields = ( content => 'x x 2', a => 'a' );
+my %three_fields = ( content => 'x x x 3', a => 'a', b => 'b' );
+my %four_fields = ( content => 'x x x 3', a => 'a', b => 'b', c => 'c', );
+my %foo_doc = ( content => 'foo', foo => 'foo' );
+my %five_fields = (
+ content => 'x x x x x 5',
+ a => 'a',
+ b => 'b',
+ c => 'c',
+ foo => 'stuff'
+);
+
+$indexer->add_doc( \%one_field );
+$schema->spec_field( name => 'a', type => $type );
+$indexer->add_doc( \%two_fields );
+pass('Add a field in the middle of indexing');
+
+$schema->spec_field( name => 'a', type => $type );
+pass('Add same field again');
+
+$schema->spec_field( name => 'b', type => $type );
+$indexer->add_doc( \%three_fields );
+pass('Add another field');
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $hits = $searcher->hits( query => 'x', num_wanted => 100 );
+is( $hits->total_hits, 3,
+ "disparate docs successfully indexed and retrieved" );
+my $top_hit = $hits->next;
+is_deeply( $top_hit->get_fields, \%three_fields,
+ "all fields stored successfully" );
+
+my $schema2 = Lucy::Test::TestSchema->new;
+my $folder2 = Lucy::Store::RAMFolder->new;
+$schema2->spec_field( name => 'foo', type => $type );
+my $indexer2 = Lucy::Index::Indexer->new(
+ schema => $schema2,
+ index => $folder2,
+);
+$indexer2->add_doc( \%foo_doc );
+$indexer2->commit;
+
+undef $indexer;
+$indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+
+$schema->spec_field( name => 'c', type => $type );
+$indexer->add_doc( \%four_fields );
+
+$indexer->add_index($folder2);
+$indexer->add_doc( \%five_fields );
+pass('successfully absorbed new field def during add_index');
+$indexer->commit;
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => 'stuff', num_wanted => 100 );
+is( $hits->total_hits, 1,
+ "successfully aborbed unknown field during add_index" );
+$top_hit = $hits->next;
+delete $top_hit->{score};
+is_deeply( $top_hit->get_fields, \%five_fields,
+ "all fields stored successfully" );
+
+$hits = $searcher->hits( query => 'x', num_wanted => 100 );
+is( $hits->total_hits, 5, "indexes successfully merged" );
diff --git a/perl/t/308-simple.t b/perl/t/308-simple.t
new file mode 100644
index 0000000..b5ab9da
--- /dev/null
+++ b/perl/t/308-simple.t
@@ -0,0 +1,88 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 8;
+use Lucy::Simple;
+use Lucy::Test::TestUtils qw( init_test_index_loc );
+
+my $test_index_loc = init_test_index_loc();
+
+my $lucy = Lucy::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+);
+
+$lucy->add_doc( { food => 'creamed corn' } );
+is( $lucy->search( query => 'creamed' ), 1, "search warks right after add" );
+
+$lucy->add_doc( { food => 'creamed spinach' } );
+is( $lucy->search( query => 'creamed' ), 2, "search returns total hits" );
+
+$lucy->add_doc( { food => 'creamed broccoli' } );
+undef $lucy;
+$lucy = Lucy::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+);
+is( $lucy->search( query => 'cream' ), 3, "commit upon destroy" );
+
+while ( my $hit = $lucy->next ) {
+ like( $hit->{food}, qr/cream/, 'next' );
+}
+
+$lucy->add_doc( { band => 'Cream' } );
+is( $lucy->search( query => 'cream' ), 4,
+ "search uses correct PolyAnalyzer" );
+
+SKIP: {
+ skip( "fork on Windows not supported by Lucy", 1 )
+ if $^O =~ /(mswin|cygwin)/i;
+
+ # We need another one:
+ my $test_index_loc = init_test_index_loc();
+
+ # Fork a process that will create an index without explicitly finishing
+ # it, and then exit, with the Simple object still in existence at
+ # global destruction time.
+ my $pid = fork();
+ if ( $pid == 0 ) { # child
+ our # This *has* to be 'our' for the test to work
+ $lucy = Lucy::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+ );
+
+ $lucy->add_doc( { food => 'creamed corn' } );
+ exit;
+ }
+ else {
+ waitpid( $pid, 0 );
+ }
+
+ my $lucy = Lucy::Simple->new(
+ language => 'en',
+ path => $test_index_loc,
+ );
+
+ ok eval { # This should die if the index wasn't finished.
+ $lucy->search( query => 'creamed' );
+ 1;
+ }, 'Simple finishes indexing during END block (apparently)';
+
+}
diff --git a/perl/t/309-span.t b/perl/t/309-span.t
new file mode 100644
index 0000000..6bdc38c
--- /dev/null
+++ b/perl/t/309-span.t
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 6;
+
+use Lucy::Test;
+use Lucy::Search::Span;
+
+my $span = Lucy::Search::Span->new(
+ offset => 2,
+ length => 3,
+ weight => 7,
+);
+
+is( $span->get_offset, 2, "get_offset" );
+is( $span->get_length, 3, "get_length" );
+is( $span->get_weight, 7, "get_weight" );
+
+$span->set_offset(10);
+$span->set_length(1);
+$span->set_weight(4);
+
+is( $span->get_offset, 10, "set_offset" );
+is( $span->get_length, 1, "set_length" );
+is( $span->get_weight, 4, "set_weight" );
+
diff --git a/perl/t/310-heat_map.t b/perl/t/310-heat_map.t
new file mode 100644
index 0000000..6bcf830
--- /dev/null
+++ b/perl/t/310-heat_map.t
@@ -0,0 +1,143 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 13;
+use Lucy::Test;
+
+my $heat_map = Lucy::Highlight::HeatMap->new( spans => [], );
+
+my $big_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 10, 10, 1.0 )
+);
+my $equally_big_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 5, 4, 1.0 )
+);
+my $smaller_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 100, 10, 1.0 )
+);
+my $zero_boost = $heat_map->calc_proximity_boost(
+ span1 => make_span( 0, 10, 1.0 ),
+ span2 => make_span( 150, 10, 1.0 )
+);
+is( $big_boost, $equally_big_boost,
+ "overlapping and abutting produce the same proximity boost" );
+cmp_ok( $big_boost, '>', $smaller_boost, "closer is better" );
+is( $zero_boost, 0, "distance outside of window yields no prox boost" );
+
+my $spans = make_spans( [ 10, 10, 1.0 ], [ 16, 14, 2.0 ] );
+my $flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 3.0 ], [ 20, 10, 2.0 ] ],
+ "flatten two overlapping spans"
+);
+my $boosts = $heat_map->generate_proximity_boosts($spans);
+is_deeply(
+ spans_to_arg_array($boosts),
+ [ [ 10, 20, 3.0 ] ],
+ "prox boosts for overlap"
+);
+
+$spans = make_spans( [ 10, 10, 1.0 ], [ 16, 14, 2.0 ], [ 50, 1, 1.0 ] );
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 3.0 ], [ 20, 10, 2.0 ], [ 50, 1, 1.0 ] ],
+ "flatten two overlapping spans, leave hole, then third span"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts,
+ 2 + 1, "boosts generated for each unique pair, since all were in range" );
+
+$spans = make_spans( [ 10, 10, 1.0 ], [ 14, 4, 4.0 ], [ 16, 14, 2.0 ] );
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 4, 1.0 ],
+ [ 14, 2, 5.0 ],
+ [ 16, 2, 7.0 ],
+ [ 18, 2, 3.0 ],
+ [ 20, 10, 2.0 ]
+ ],
+ "flatten three overlapping spans"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts,
+ 2 + 1, "boosts generated for each unique pair, since all were in range" );
+
+$spans = make_spans(
+ [ 10, 10, 1.0 ],
+ [ 16, 14, 4.0 ],
+ [ 16, 14, 2.0 ],
+ [ 30, 10, 10.0 ]
+);
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 7.0 ], [ 20, 10, 6.0 ], [ 30, 10, 10.0 ] ],
+ "flatten 4 spans, middle two have identical range"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts,
+ 3 + 2 + 1,
+ "boosts generated for each unique pair, since all were in range"
+);
+
+$spans = make_spans(
+ [ 10, 10, 1.0 ],
+ [ 16, 4, 4.0 ],
+ [ 16, 14, 2.0 ],
+ [ 230, 10, 10.0 ]
+);
+$flattened = $heat_map->flatten_spans($spans);
+is_deeply(
+ spans_to_arg_array($flattened),
+ [ [ 10, 6, 1.0 ], [ 16, 4, 7.0 ], [ 20, 10, 2.0 ], [ 230, 10, 10.0 ] ],
+ "flatten 4 spans, middle two have identical starts but different ends"
+);
+$boosts = $heat_map->generate_proximity_boosts($spans);
+is( scalar @$boosts, 2 + 1, "boosts not generated for out of range span" );
+
+sub make_span {
+ return Lucy::Search::Span->new(
+ offset => $_[0],
+ length => $_[1],
+ weight => $_[2],
+ );
+}
+
+sub make_spans {
+ my @spans;
+ for my $arg_ref (@_) {
+ push @spans, make_span( @{$arg_ref}[ 0 .. 2 ] );
+ }
+ return \@spans;
+}
+
+sub spans_to_arg_array {
+ my $spans = shift;
+ my @out;
+ for (@$spans) {
+ push @out, [ $_->get_offset, $_->get_length, $_->get_weight ];
+ }
+ return \@out;
+}
+
diff --git a/perl/t/311-hl_selection.t b/perl/t/311-hl_selection.t
new file mode 100644
index 0000000..80bba65
--- /dev/null
+++ b/perl/t/311-hl_selection.t
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+use Test::More tests => 1;
+use Lucy::Test;
+
+my $schema = Lucy::Plan::Schema->new;
+$schema->spec_field(
+ name => 'content',
+ type => Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new,
+ highlightable => 1,
+ ),
+);
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ create => 1,
+);
+$indexer->add_doc(
+ doc => {
+ content => <<'EOF',
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla NNN bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla MMM bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla bla.
+EOF
+ },
+);
+$indexer->commit();
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder, );
+my $query = 'NNN MMM';
+my $highlighter = Lucy::Highlight::Highlighter->new(
+ searcher => $searcher,
+ query => $query,
+ field => 'content'
+);
+my $hits = $searcher->hits( query => $query, );
+my $hit = $hits->next();
+my $excerpt = $highlighter->create_excerpt($hit);
+like( $excerpt, qr/(NNN|MMM)/, "Sentence boundary algo doesn't chop terms" );
+
diff --git a/perl/t/400-match_posting.t b/perl/t/400-match_posting.t
new file mode 100644
index 0000000..edfa28f
--- /dev/null
+++ b/perl/t/400-match_posting.t
@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MatchOnlySim;
+use base qw( Lucy::Index::Similarity );
+
+sub make_posting {
+ my $self = shift;
+ return Lucy::Index::Posting::MatchPosting->new( similarity => $self );
+}
+
+package MatchSchema::MatchOnly;
+use base qw( Lucy::Plan::FullTextType );
+use Lucy::Index::Posting::MatchPosting;
+
+sub make_similarity { MatchOnlySim->new }
+
+package MatchSchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = MatchSchema::MatchOnly->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package main;
+
+use Lucy::Test::TestUtils qw( get_uscon_docs );
+use Test::More tests => 6;
+
+my $uscon_docs = get_uscon_docs();
+my $match_folder = make_index( MatchSchema->new, $uscon_docs );
+my $score_folder = make_index( Lucy::Test::TestSchema->new, $uscon_docs );
+
+my $match_searcher
+ = Lucy::Search::IndexSearcher->new( index => $match_folder );
+my $score_searcher
+ = Lucy::Search::IndexSearcher->new( index => $score_folder );
+
+for (qw( land of the free )) {
+ my $match_got = hit_ids_array( $match_searcher, $_ );
+ my $score_got = hit_ids_array( $score_searcher, $_ );
+ is_deeply( $match_got, $score_got, "same hits for '$_'" );
+}
+
+my $qstring = '"the legislature"';
+my $should_have_hits = hit_ids_array( $score_searcher, $qstring );
+my $should_be_empty = hit_ids_array( $match_searcher, $qstring );
+ok( scalar @$should_have_hits, "successfully scored phrase $qstring" );
+ok( !scalar @$should_be_empty, "no hits matched for phrase $qstring" );
+
+sub make_index {
+ my ( $schema, $docs ) = @_;
+ my $folder = Lucy::Store::RAMFolder->new;
+ my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+ );
+ $indexer->add_doc( { content => $_->{bodytext} } ) for values %$docs;
+ $indexer->commit;
+ return $folder;
+}
+
+sub hit_ids_array {
+ my ( $searcher, $query_string ) = @_;
+ my $query = $searcher->glean_query($query_string);
+
+ my $bit_vec
+ = Lucy::Object::BitVector->new( capacity => $searcher->doc_max + 1 );
+ my $bit_collector
+ = Lucy::Search::Collector::BitCollector->new( bit_vector => $bit_vec,
+ );
+ $searcher->collect( query => $query, collector => $bit_collector );
+ return $bit_vec->to_array->to_arrayref;
+}
diff --git a/perl/t/501-termquery.t b/perl/t/501-termquery.t
new file mode 100644
index 0000000..d18f90f
--- /dev/null
+++ b/perl/t/501-termquery.t
@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 12;
+use Storable qw( freeze thaw );
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a', 'b', 'c c c d', 'c d', 'd' .. 'z', );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $term_query
+ = Lucy::Search::TermQuery->new( field => 'content', term => 'c' );
+is( $term_query->to_string, "content:c", "to_string" );
+
+my $hits = $searcher->hits( query => $term_query );
+is( $hits->total_hits, 2, "correct number of hits returned" );
+
+my $hit = $hits->next;
+is( $hit->{content}, 'c c c d', "most relevant doc is highest" );
+
+$hit = $hits->next;
+is( $hit->{content}, 'c d', "second most relevant" );
+
+my $frozen = freeze($term_query);
+my $thawed = thaw($frozen);
+is( $thawed->get_field, 'content', "field survives freeze/thaw" );
+is( $thawed->get_term, 'c', "term survives freeze/thaw" );
+is( $thawed->get_boost, $term_query->get_boost,
+ "boost survives freeze/thaw" );
+ok( $thawed->equals($term_query), "equals" );
+$thawed->set_boost(10);
+ok( !$thawed->equals($term_query), "!equals (boost)" );
+my $different_term = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'd'
+);
+my $different_field = Lucy::Search::TermQuery->new(
+ field => 'title',
+ term => 'c'
+);
+ok( !$term_query->equals($different_term), "!equals (term)" );
+ok( !$term_query->equals($different_field), "!equals (field)" );
+
+my $term_compiler = $term_query->make_compiler( searcher => $searcher );
+$frozen = freeze($term_compiler);
+$thawed = thaw($frozen);
+ok( $term_compiler->equals($thawed), "freeze/thaw compiler" );
diff --git a/perl/t/502-phrasequery.t b/perl/t/502-phrasequery.t
new file mode 100644
index 0000000..03f5f46
--- /dev/null
+++ b/perl/t/502-phrasequery.t
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use Storable qw( freeze thaw );
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $best_match = 'x a b c d a b c d';
+
+my @docs = (
+ 1 .. 20,
+ 'a b c a b c a b c d',
+ 'a b c d x x a',
+ 'a c b d', 'a x x x b x x x c x x x x x x d x',
+ $best_match, 'a' .. 'z',
+);
+
+my $folder = create_index(@docs);
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $phrase_query = Lucy::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [],
+);
+is( $phrase_query->to_string, 'content:""', "empty PhraseQuery to_string" );
+$phrase_query = Lucy::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( a b c d )],
+);
+is( $phrase_query->to_string, 'content:"a b c d"', "to_string" );
+
+my $hits = $searcher->hits( query => $phrase_query );
+is( $hits->total_hits, 3, "correct number of hits" );
+my $first_hit = $hits->next;
+is( $first_hit->{content}, $best_match, 'best match appears first' );
+
+my $second_hit = $hits->next;
+ok( $first_hit->get_score > $second_hit->get_score,
+ "best match scores higher: "
+ . $first_hit->get_score . " > "
+ . $second_hit->get_score
+);
+
+$phrase_query = Lucy::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( c a )],
+);
+$hits = $searcher->hits( query => $phrase_query );
+is( $hits->total_hits, 1, 'avoid underflow when subtracting offset' );
+
+# "a b c"
+$phrase_query = Lucy::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( a b c )],
+);
+$hits = $searcher->hits( query => $phrase_query );
+is( $hits->total_hits, 3, 'offset starting from zero' );
+
+my $frozen = freeze($phrase_query);
+my $thawed = thaw($frozen);
+$hits = $searcher->hits( query => $thawed );
+is( $hits->total_hits, 3, 'freeze/thaw' );
+
+my $phrase_compiler = $phrase_query->make_compiler( searcher => $searcher );
+$frozen = freeze($phrase_compiler);
+$thawed = thaw($frozen);
+ok( $phrase_compiler->equals($thawed), "freeze/thaw compiler" );
diff --git a/perl/t/504-similarity.t b/perl/t/504-similarity.t
new file mode 100644
index 0000000..3383419
--- /dev/null
+++ b/perl/t/504-similarity.t
@@ -0,0 +1,120 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MySchema::LongTextField;
+use base qw( Lucy::Plan::FullTextType );
+use LucyX::Index::LongFieldSim;
+
+sub make_similarity { LucyX::Index::LongFieldSim->new }
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $analyzer = Lucy::Analysis::RegexTokenizer->new;
+ my $plain_type = Lucy::Plan::FullTextType->new( analyzer => $analyzer, );
+ my $long_field_type
+ = MySchema::LongTextField->new( analyzer => $analyzer, );
+ $self->spec_field( name => 'title', type => $plain_type );
+ $self->spec_field( name => 'body', type => $long_field_type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 9;
+use Lucy::Test;
+use bytes;
+no bytes;
+
+my $sim = Lucy::Index::Similarity->new;
+my $twin = $sim->load( $sim->dump );
+ok( $sim->equals($twin), "Dump/Load" );
+
+cmp_ok( $sim->tf(10) - $sim->tf(9), '<', 1, "TF is damped" );
+
+my $rare_idf = $sim->idf( doc_freq => 3, total_docs => 100 );
+my $common_idf = $sim->idf( doc_freq => 50, total_docs => 100 );
+cmp_ok( $rare_idf, '>', $common_idf, 'Rarer terms have higher IDF' );
+
+my $less_coordinated = $sim->coord( overlap => 2, max_overlap => 5 );
+my $more_coordinated = $sim->coord( overlap => 3, max_overlap => 5 );
+cmp_ok( $less_coordinated, '<', $more_coordinated,
+ "greater overlap means bigger coord bonus" );
+
+my @bytes = ( 100, 110, 120, 130, 140 );
+my @floats = ( 0.015625, 0.09375, 0.5, 3.0, 16.0 );
+my @transformed = map { $sim->decode_norm($_) } @bytes;
+is_deeply( \@floats, \@transformed,
+ "decode_norm more or less matches Java Lucene behavior" );
+
+@bytes = 0 .. 255;
+@floats = map { $sim->decode_norm($_) } @bytes;
+@transformed = map { $sim->encode_norm($_) } @floats;
+is_deeply( \@transformed, \@bytes,
+ "encode_norm and decode_norm are complementary" );
+
+my $norm_decoder = $sim->get_norm_decoder;
+@transformed = ();
+for ( 0 .. 255 ) {
+ push @transformed,
+ unpack( 'f', bytes::substr( $norm_decoder, $_ * 4, 4 ) );
+}
+is_deeply( \@transformed, \@floats,
+ "using the norm_decoder produces desired results" );
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => MySchema->new,
+);
+
+my %source_docs = (
+ 'spam' => 'spam spam',
+ 'not spam' => 'not spam not even close to spam no spam here',
+);
+while ( my ( $title, $body ) = each %source_docs ) {
+ $indexer->add_doc(
+ { title => $title,
+ body => $body,
+ }
+ );
+}
+$indexer->commit;
+undef $indexer;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits(
+ query => Lucy::Search::TermQuery->new(
+ field => 'title',
+ term => 'spam',
+ )
+);
+is( $hits->next->{'title'},
+ 'spam', "Default Similarity biased towards short fields" );
+
+$hits = $searcher->hits(
+ query => Lucy::Search::TermQuery->new(
+ field => 'body',
+ term => 'spam',
+ )
+);
+is( $hits->next->{'title'},
+ 'not spam', "LongFieldSim cancels short-field bias" );
diff --git a/perl/t/505-hit_queue.t b/perl/t/505-hit_queue.t
new file mode 100644
index 0000000..070e51a
--- /dev/null
+++ b/perl/t/505-hit_queue.t
@@ -0,0 +1,213 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 36;
+use Lucy::Test;
+use List::Util qw( shuffle );
+
+my $hit_q = Lucy::Search::HitQueue->new( wanted => 10 );
+
+my @docs_and_scores = (
+ [ 0, 1.0 ],
+ [ 5, 0.1 ],
+ [ 10, 0.1 ],
+ [ 1000, 0.9 ],
+ [ 2000, 1.0 ],
+ [ 3000, 1.0 ],
+);
+
+my @match_docs = map {
+ Lucy::Search::MatchDoc->new(
+ doc_id => $_->[0],
+ score => $_->[1],
+ )
+} @docs_and_scores;
+
+my @correct_order = sort {
+ $b->get_score <=> $a->get_score
+ or $a->get_doc_id <=> $b->get_doc_id
+} @match_docs;
+my @correct_docs = map { $_->get_doc_id } @correct_order;
+my @correct_scores = map { $_->get_score } @correct_order;
+
+$hit_q->insert($_) for @match_docs;
+my $got = $hit_q->pop_all;
+
+my @scores = map { $_->get_score } @$got;
+is_deeply( \@scores, \@correct_scores, "rank by scores first" );
+
+my @doc_ids = map { $_->get_doc_id } @$got;
+is_deeply( \@doc_ids, \@correct_docs, "rank by doc_id after score" );
+
+my $schema = Lucy::Plan::Schema->new;
+my $type = Lucy::Plan::StringType->new( sortable => 1 );
+$schema->spec_field( name => 'letter', type => $type );
+$schema->spec_field( name => 'number', type => $type );
+$schema->spec_field( name => 'id', type => $type );
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+my @letters = 'a' .. 'z';
+my @numbers = 1 .. 5;
+my @docs;
+for my $id ( 1 .. 100 ) {
+ my $doc = {
+ letter => $letters[ rand @letters ],
+ number => $numbers[ rand @numbers ],
+ id => $id,
+ };
+ push @docs, $doc;
+ $indexer->add_doc($doc);
+}
+$indexer->commit;
+my $seg_reader = Lucy::Index::IndexReader->open( index => $folder );
+
+my $by_letter = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'letter' ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+$hit_q = Lucy::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_letter,
+);
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $match_doc = Lucy::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => 1.0,
+ values => [ $docs[ $doc_id - 1 ]{letter} ],
+ );
+ $hit_q->insert($match_doc);
+}
+my @wanted = map { $_->{id} }
+ sort { $a->{letter} cmp $b->{letter} || $a->{id} <=> $b->{id} } @docs;
+my @got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by letter then doc id" );
+
+my $by_num_then_letter = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'number' ),
+ Lucy::Search::SortRule->new( field => 'letter', reverse => 1 ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+$hit_q = Lucy::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_num_then_letter,
+);
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $doc = $docs[ $doc_id - 1 ];
+ my $match_doc = Lucy::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => 1.0,
+ values => [ $doc->{number}, $doc->{letter} ],
+ );
+ $hit_q->insert($match_doc);
+}
+@wanted = map { $_->{id} }
+ sort {
+ $a->{number} <=> $b->{number}
+ || $b->{letter} cmp $a->{letter}
+ || $a->{id} <=> $b->{id}
+ } @docs;
+@got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by num then letter reversed then doc id" );
+
+my $by_num_then_score = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'number' ),
+ Lucy::Search::SortRule->new( type => 'score' ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+$hit_q = Lucy::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_num_then_score,
+);
+
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $match_doc = Lucy::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => $doc_id,
+ values => [ $docs[ $doc_id - 1 ]{number} ],
+ );
+ $hit_q->insert($match_doc);
+}
+@wanted = map { $_->{id} }
+ sort { $a->{number} <=> $b->{number} || $b->{id} <=> $a->{id} } @docs;
+@got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by num then score then doc id" );
+
+$hit_q = Lucy::Search::HitQueue->new(
+ wanted => 100,
+ schema => $schema,
+ sort_spec => $by_num_then_score,
+);
+
+for my $doc_id ( shuffle( 1 .. 100 ) ) {
+ my $fields = $docs[ $doc_id - 1 ];
+ my $values = Lucy::Object::VArray->new( capacity => 1 );
+ $values->push( Lucy::Object::CharBuf->new( $fields->{number} ) );
+ my $match_doc = Lucy::Search::MatchDoc->new(
+ doc_id => $doc_id,
+ score => $doc_id,
+ values => $values,
+ );
+ $hit_q->insert($match_doc);
+}
+@wanted = map { $_->{id} }
+ sort { $a->{number} <=> $b->{number} || $b->{id} <=> $a->{id} } @docs;
+@got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+is_deeply( \@got, \@wanted, "sort by value when no reader set" );
+
+@docs_and_scores = ();
+for ( 1 .. 30 ) {
+ push @docs_and_scores, [ int( rand(10000) ) + 1, rand(10) ];
+}
+@docs_and_scores = sort { $b->[1] <=> $a->[1] } @docs_and_scores;
+@doc_ids = map { $_->[0] } @docs_and_scores;
+
+@match_docs = map {
+ Lucy::Search::MatchDoc->new(
+ doc_id => $_->[0],
+ score => $_->[1],
+ )
+} sort { $a->[0] <=> $b->[0] } @docs_and_scores;
+
+for my $size ( 0 .. $#match_docs ) {
+ $hit_q = Lucy::Search::HitQueue->new( wanted => $size, );
+ $hit_q->insert($_) for @match_docs;
+
+ if ($size) {
+ @wanted = @doc_ids[ 0 .. $size - 1 ];
+ }
+ else {
+ @wanted = ();
+ }
+ @got = map { $_->get_doc_id } @{ $hit_q->pop_all };
+ is_deeply( \@got, \@wanted, "random docs and scores, size $size" );
+}
+
diff --git a/perl/t/506-collector.t b/perl/t/506-collector.t
new file mode 100644
index 0000000..9fa0714
--- /dev/null
+++ b/perl/t/506-collector.t
@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 5;
+use Lucy::Test;
+use LucyX::Search::MockMatcher;
+
+my @docs = ( 1, 5, 10, 1000 );
+my @scores = ( 2, 0, 0, 1 );
+
+my $collector = Lucy::Search::Collector::SortCollector->new( wanted => 3 );
+test_collect($collector);
+
+my @got = map { $_->get_score } @{ $collector->pop_match_docs };
+is_deeply( \@got, [ 2, 1, 0 ], "collect into HitQueue" );
+
+$collector = Lucy::Search::Collector::SortCollector->new( wanted => 0 );
+test_collect($collector);
+is( $collector->get_total_hits, 4,
+ "get_total_hits is accurate when no hits are requested" );
+my $match_docs = $collector->pop_match_docs;
+is( scalar @$match_docs, 0, "no hits wanted, so no hits returned" );
+
+my $bit_vec = Lucy::Object::BitVector->new;
+$collector
+ = Lucy::Search::Collector::BitCollector->new( bit_vector => $bit_vec );
+test_collect($collector);
+is_deeply(
+ $bit_vec->to_arrayref,
+ [ 1, 5, 10, 1000 ],
+ "BitCollector collects the right doc nums"
+);
+
+$bit_vec = Lucy::Object::BitVector->new;
+my $inner_coll
+ = Lucy::Search::Collector::BitCollector->new( bit_vector => $bit_vec );
+my $offset_coll = Lucy::Search::Collector::OffsetCollector->new(
+ collector => $inner_coll,
+ offset => 10,
+);
+test_collect($offset_coll);
+is_deeply( $bit_vec->to_arrayref, [ 11, 15, 20, 1010 ], "Offset collector" );
+
+sub test_collect {
+ my $collector = shift;
+ my $matcher = LucyX::Search::MockMatcher->new(
+ doc_ids => \@docs,
+ scores => \@scores,
+ );
+ $collector->set_matcher($matcher);
+ while ( my $doc_id = $matcher->next ) {
+ $collector->collect($doc_id);
+ }
+}
diff --git a/perl/t/507-filter.t b/perl/t/507-filter.t
new file mode 100644
index 0000000..741bc1a
--- /dev/null
+++ b/perl/t/507-filter.t
@@ -0,0 +1,128 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 24;
+use Storable qw( nfreeze thaw );
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( create_index );
+use LucyX::Search::Filter;
+
+my $query_parser
+ = Lucy::Search::QueryParser->new( schema => Lucy::Test::TestSchema->new );
+
+## Set up main objects.
+my ( $filter_1, $filter_2 );
+{
+ my $folder_1 = create_index( 'a x', 'b x', 'c x', 'a y', 'b y', 'c y' );
+ my $folder_2 = create_index( 'a w', 'b w', 'c w', 'a z', 'b z', 'c z' );
+
+ my $searcher_1 = Lucy::Search::IndexSearcher->new( index => $folder_1 );
+ my $searcher_2 = Lucy::Search::IndexSearcher->new( index => $folder_2 );
+
+ my $reader_1 = $searcher_1->get_reader->get_seg_readers->[0];
+ my $reader_2 = $searcher_2->get_reader->get_seg_readers->[0];
+
+ my $only_a_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'a',
+ );
+ $filter_1 = LucyX::Search::Filter->new( query => $only_a_query );
+ $filter_2 = LucyX::Search::Filter->new( query => $only_a_query );
+
+ ## Test index 1, filter 1.
+ my $hits = $searcher_1->hits( query => filt_query( $filter_1, 'x y z' ) );
+ is( $hits->total_hits, 2, 'filtering a query works' );
+
+ ## Test index 2, filter 2.
+ $hits = $searcher_2->hits( query => filt_query( $filter_2, 'x y z' ) );
+ is( $hits->total_hits, 1, 'filtering a query works' );
+
+ ## Compare 1-1 to 2-2.
+ my $cached_bits_1 = $filter_1->_bits($reader_1);
+ my $cached_bits_2 = $filter_2->_bits($reader_2);
+ ok( !$cached_bits_1->equals($cached_bits_2),
+ 'cached bits are unique (1-1 != 2-2)'
+ );
+
+ ## Test copy of index 1, filter 1.
+ $hits = $searcher_1->hits( query => filt_query( $filter_1, 'w y z' ) );
+ my $bits = $filter_1->_bits($reader_1);
+ is( $hits->total_hits, 1, 'filtering a query works' );
+ ok( $cached_bits_1->equals($bits),
+ 'cached bits are cached (1-1 == 1-1)' );
+ ok( !$cached_bits_2->equals($bits),
+ 'cached bits are unique (2-2 != 1-1)'
+ );
+
+ ## Test index 1, filter 2.
+ $hits = $searcher_1->hits( query => filt_query( $filter_2, 'w y z' ) );
+ my $cached_bits_3 = $bits = $filter_2->_bits($reader_1);
+ is( $hits->total_hits, 1, 'filtering a query works' );
+ ok( !$cached_bits_1->equals($bits),
+ 'cached bits are unique (1-1 != 1-2)'
+ );
+ ok( !$cached_bits_2->equals($bits),
+ 'cached bits are unique (2-2 != 1-2)'
+ );
+
+ ## Test copy of index 2, filter 2.
+ $hits = $searcher_2->hits( query => filt_query( $filter_2, 'x y z' ) );
+ $bits = $filter_2->_bits($reader_2);
+ is( $hits->total_hits, 1, 'filtering a query works' );
+ ok( !$cached_bits_1->equals($bits),
+ 'cached bits are unique (1-1 != 2-2)'
+ );
+ ok( $cached_bits_2->equals($bits),
+ 'cached bits are cached (2-2 != 2-2)' );
+ ok( !$cached_bits_3->equals($bits),
+ 'cached bits are unique (1-2 != 2-2)'
+ );
+
+ ## Test copy of index 1, filter 2.
+ $hits = $searcher_1->hits( query => filt_query( $filter_2, 'x y z' ) );
+ $bits = $filter_2->_bits($reader_1);
+ is( $hits->total_hits, 2, 'filtering a query works' );
+ ok( !$cached_bits_1->equals($bits),
+ 'cached bits are unique (1-1 != 1-2)'
+ );
+ ok( !$cached_bits_2->equals($bits),
+ 'cached bits are unique (2-2 != 1-2)'
+ );
+ ok( $cached_bits_3->equals($bits),
+ 'cached bits are cached (1-2 != 1-2)' );
+
+ is( $filter_1->_cached_count, 1, 'cache count check correct' );
+ is( $filter_2->_cached_count, 2, 'cache count check correct' );
+}
+
+sub filt_query {
+ my ( $filter, $query_string ) = @_;
+ return Lucy::Search::ANDQuery->new(
+ children => [ $filter, $query_parser->parse($query_string) ], );
+}
+
+# Readers should be automatically undef'd, refcnt == 0.
+is( $filter_1->_cached_count, 0, 'cache count check correct' );
+is( $filter_2->_cached_count, 0, 'cache count check correct' );
+
+ok( $filter_1->equals($filter_2), 'equals' );
+my $frozen = nfreeze($filter_1);
+my $thawed = thaw($frozen);
+ok( $thawed->equals($filter_1), 'freeze/thaw' );
+is( $filter_1->to_string, "Filter(content:a)", "to_string" );
diff --git a/perl/t/508-hits.t b/perl/t/508-hits.t
new file mode 100644
index 0000000..30a0ef0
--- /dev/null
+++ b/perl/t/508-hits.t
@@ -0,0 +1,65 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( create_index );
+
+my @docs = ( 'a b', 'a a b', 'a a a b', 'x' );
+my $folder = create_index(@docs);
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits(
+ query => 'a',
+ offset => 0,
+ num_wanted => 1,
+);
+is( $hits->total_hits, 3, "total_hits" );
+my $hit = $hits->next;
+cmp_ok( $hit->get_score, '>', 0.0, "score field added" );
+is( $hits->next, undef, "hits exhausted" );
+
+$hits->next;
+is( $hits->next, undef, "hits exhausted" );
+
+my @retrieved;
+@retrieved = ();
+$hits = $searcher->hits(
+ query => 'a',
+ offset => 0,
+ num_wanted => 100,
+);
+is( $hits->total_hits, 3, "total_hits still correct" );
+while ( my $hit = $hits->next ) {
+ push @retrieved, $hit->{content};
+}
+is_deeply( \@retrieved, [ @docs[ 2, 1, 0 ] ], "correct content via next()" );
+
+@retrieved = ();
+$hits = $searcher->hits(
+ query => 'a',
+ offset => 1,
+ num_wanted => 100,
+);
+is( $hits->total_hits, 3, "total_hits correct with offset" );
+while ( my $hit = $hits->next ) {
+ push @retrieved, $hit->{content};
+}
+is( scalar @retrieved, 2, "number retrieved with offset" );
+is_deeply( \@retrieved, [ @docs[ 1, 0 ] ], "correct content with offset" );
diff --git a/perl/t/509-poly_searcher.t b/perl/t/509-poly_searcher.t
new file mode 100644
index 0000000..13dee4d
--- /dev/null
+++ b/perl/t/509-poly_searcher.t
@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 8;
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder_a = create_index( 'x a', 'x b', 'x c' );
+my $folder_b = create_index( 'y b', 'y c', 'y d' );
+my $searcher_a = Lucy::Search::IndexSearcher->new( index => $folder_a );
+my $searcher_b = Lucy::Search::IndexSearcher->new( index => $folder_b );
+
+my $poly_searcher = Lucy::Search::PolySearcher->new(
+ schema => Lucy::Test::TestSchema->new,
+ searchers => [ $searcher_a, $searcher_b ],
+);
+
+is( $poly_searcher->doc_freq( field => 'content', term => 'b' ),
+ 2, 'doc_freq' );
+is( $poly_searcher->doc_max, 6, 'doc_max' );
+is( $poly_searcher->fetch_doc(1)->{content}, 'x a', "fetch_doc" );
+isa_ok( $poly_searcher->fetch_doc_vec(1), 'Lucy::Index::DocVector' );
+
+my $hits = $poly_searcher->hits( query => 'a' );
+is( $hits->total_hits, 1, "Find hit in first searcher" );
+
+$hits = $poly_searcher->hits( query => 'd' );
+is( $hits->total_hits, 1, "Find hit in second searcher" );
+
+$hits = $poly_searcher->hits( query => 'c' );
+is( $hits->total_hits, 2, "Find hits in both searchers" );
+
+my $bit_vec
+ = Lucy::Object::BitVector->new( capacity => $poly_searcher->doc_max );
+my $bitcoll
+ = Lucy::Search::Collector::BitCollector->new( bit_vector => $bit_vec );
+my $query = $poly_searcher->glean_query('b');
+$poly_searcher->collect( query => $query, collector => $bitcoll );
+is_deeply( $bit_vec->to_arrayref, [ 2, 4 ], "collect" );
diff --git a/perl/t/510-remote_search.t b/perl/t/510-remote_search.t
new file mode 100644
index 0000000..810f540
--- /dev/null
+++ b/perl/t/510-remote_search.t
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More;
+use Time::HiRes qw( sleep );
+use IO::Socket::INET;
+
+my $PORT_NUM = 7890;
+BEGIN {
+ if ( $^O =~ /(mswin|cygwin)/i ) {
+ plan( 'skip_all', "fork on Windows not supported by Lucy" );
+ }
+ elsif ( $ENV{LUCY_VALGRIND} ) {
+ plan( 'skip_all', "time outs cause probs under valgrind" );
+ }
+}
+
+package SortSchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $plain_type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new );
+ my $string_type = Lucy::Plan::StringType->new( sortable => 1 );
+ $self->spec_field( name => 'content', type => $plain_type );
+ $self->spec_field( name => 'number', type => $string_type );
+ return $self;
+}
+
+package main;
+
+use Lucy::Test;
+use LucyX::Remote::SearchServer;
+use LucyX::Remote::SearchClient;
+
+my $kid;
+$kid = fork;
+if ($kid) {
+ sleep .25; # allow time for the server to set up the socket
+ die "Failed fork: $!" unless defined $kid;
+}
+else {
+ my $folder = Lucy::Store::RAMFolder->new;
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => SortSchema->new,
+ );
+ my $number = 5;
+ for (qw( a b c )) {
+ $indexer->add_doc( { content => "x $_", number => $number } );
+ $number -= 2;
+ }
+ $indexer->commit;
+
+ my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+ my $server = LucyX::Remote::SearchServer->new(
+ port => $PORT_NUM,
+ searcher => $searcher,
+ password => 'foo',
+ );
+ $server->serve;
+ exit(0);
+}
+
+my $test_client_sock = IO::Socket::INET->new(
+ PeerAddr => "localhost:$PORT_NUM",
+ Proto => 'tcp',
+);
+if ($test_client_sock) {
+ plan( tests => 10 );
+ undef $test_client_sock;
+}
+else {
+ plan( 'skip_all', "Can't get a socket: $!" );
+}
+
+my $searchclient = LucyX::Remote::SearchClient->new(
+ schema => SortSchema->new,
+ peer_address => "localhost:$PORT_NUM",
+ password => 'foo',
+);
+
+is( $searchclient->doc_freq( field => 'content', term => 'x' ),
+ 3, "doc_freq" );
+is( $searchclient->doc_max, 3, "doc_max" );
+isa_ok( $searchclient->fetch_doc(1), "Lucy::Document::HitDoc", "fetch_doc" );
+isa_ok( $searchclient->fetch_doc_vec(1),
+ "Lucy::Index::DocVector", "fetch_doc_vec" );
+
+my $hits = $searchclient->hits( query => 'x' );
+is( $hits->total_hits, 3, "retrieved hits from search server" );
+
+$hits = $searchclient->hits( query => 'a' );
+is( $hits->total_hits, 1, "retrieved hit from search server" );
+
+my $folder_b = Lucy::Store::RAMFolder->new;
+my $number = 6;
+for (qw( a b c )) {
+ my $indexer = Lucy::Index::Indexer->new(
+ index => $folder_b,
+ schema => SortSchema->new,
+ );
+ $indexer->add_doc( { content => "y $_", number => $number } );
+ $number -= 2;
+ $indexer->add_doc( { content => 'blah blah blah' } ) for 1 .. 3;
+ $indexer->commit;
+}
+
+my $searcher_b = Lucy::Search::IndexSearcher->new( index => $folder_b, );
+is( ref( $searcher_b->get_reader ), 'Lucy::Index::PolyReader', );
+
+my $poly_searcher = Lucy::Search::PolySearcher->new(
+ schema => SortSchema->new,
+ searchers => [ $searcher_b, $searchclient ],
+);
+
+$hits = $poly_searcher->hits( query => 'b' );
+is( $hits->total_hits, 2, "retrieved hits from PolySearcher" );
+
+my %results;
+$results{ $hits->next()->{content} } = 1;
+$results{ $hits->next()->{content} } = 1;
+my %expected = ( 'x b' => 1, 'y b' => 1, );
+
+is_deeply( \%results, \%expected, "docs fetched from both local and remote" );
+
+my $sort_spec = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'number' ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ],
+);
+$hits = $poly_searcher->hits(
+ query => 'b',
+ sort_spec => $sort_spec,
+);
+my @got;
+
+while ( my $hit = $hits->next ) {
+ push @got, $hit->{content};
+}
+$sort_spec = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'number', reverse => 1 ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ],
+);
+$hits = $poly_searcher->hits(
+ query => 'b',
+ sort_spec => $sort_spec,
+);
+my @reversed;
+while ( my $hit = $hits->next ) {
+ push @reversed, $hit->{content};
+}
+is_deeply(
+ \@got,
+ [ reverse @reversed ],
+ "Sort combination of remote and local"
+);
+
+END {
+ $searchclient->terminate if defined $searchclient;
+ kill( TERM => $kid ) if $kid;
+}
diff --git a/perl/t/511-sort_spec.t b/perl/t/511-sort_spec.t
new file mode 100644
index 0000000..902b91c
--- /dev/null
+++ b/perl/t/511-sort_spec.t
@@ -0,0 +1,328 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 18;
+use List::Util qw( shuffle );
+
+package ReverseType;
+use base qw( Lucy::Plan::Int32Type );
+
+sub new {
+ return shift->SUPER::new( indexed => 0, sortable => 1, @_ );
+}
+
+sub compare_values {
+ my ( $self, %args ) = @_;
+ return $args{b} <=> $args{a};
+}
+
+package SortSchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $unsortable = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new, );
+ my $string_type = Lucy::Plan::StringType->new( sortable => 1 );
+ my $int32_type = Lucy::Plan::Int32Type->new(
+ indexed => 0,
+ sortable => 1,
+ );
+ my $int64_type = Lucy::Plan::Int64Type->new(
+ indexed => 0,
+ sortable => 1,
+ );
+ my $float32_type = Lucy::Plan::Float32Type->new(
+ indexed => 0,
+ sortable => 1,
+ );
+ my $float64_type = Lucy::Plan::Float64Type->new(
+ indexed => 0,
+ sortable => 1,
+ );
+ $self->spec_field( name => 'name', type => $string_type );
+ $self->spec_field( name => 'speed', type => $int32_type );
+ $self->spec_field( name => 'sloth', type => ReverseType->new );
+ $self->spec_field( name => 'weight', type => $int32_type );
+ $self->spec_field( name => 'int32', type => $int32_type );
+ $self->spec_field( name => 'int64', type => $int64_type );
+ $self->spec_field( name => 'float32', type => $float32_type );
+ $self->spec_field( name => 'float64', type => $float64_type );
+ $self->spec_field( name => 'home', type => $string_type );
+ $self->spec_field( name => 'cat', type => $string_type );
+ $self->spec_field( name => 'unused', type => $string_type );
+ $self->spec_field( name => 'nope', type => $unsortable );
+ return $self;
+}
+
+package main;
+use Lucy::Test;
+
+my $airplane = {
+ name => 'airplane',
+ speed => 200,
+ sloth => 200,
+ weight => 8000,
+ home => 'air',
+ cat => 'vehicle',
+};
+my $bike = {
+ name => 'bike',
+ speed => 15,
+ sloth => 15,
+ weight => 25,
+ home => 'land',
+ cat => 'vehicle',
+};
+my $car = {
+ name => 'car',
+ speed => 70,
+ sloth => 70,
+ weight => 3000,
+ home => 'land',
+ cat => 'vehicle',
+};
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = SortSchema->new;
+my $indexer;
+
+sub refresh_indexer {
+ $indexer->commit if $indexer;
+ $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+ );
+}
+
+# First, add vehicles.
+refresh_indexer();
+$indexer->add_doc($_) for ( $airplane, $bike, $car );
+
+# Add random strings.
+my @random_strings;
+my @letters = 'a' .. 'z';
+for ( 0 .. 99 ) {
+ my $string = "";
+ for ( 0 .. int( rand(10) ) ) {
+ $string .= $letters[ rand @letters ];
+ }
+ $indexer->add_doc(
+ { cat => 'random',
+ name => $string,
+ }
+ );
+ push @random_strings, $string;
+ refresh_indexer() if $_ % 10 == 0;
+}
+@random_strings = sort @random_strings;
+
+# Add random int32s.
+my @random_int32s;
+my $i32_max = 2**31 - 1;
+for ( 0 .. 99 ) {
+ my $random_num = int( rand($i32_max) );
+ $indexer->add_doc(
+ { cat => 'random_int32s',
+ name => $random_num,
+ int32 => $random_num,
+ }
+ );
+ push @random_int32s, $random_num;
+ refresh_indexer() if $_ % 10 == 0;
+}
+@random_int32s = sort { $a <=> $b } @random_int32s;
+
+# Add random int64s. On 32-bit Perls, precision errors may occur since we SVs
+# only store numbers in doubles above U32_MAX, but that's fine because the
+# errors precede the indexing stage.
+my @random_int64s;
+my $i64_max = 2**63 - 1;
+for ( 0 .. 99 ) {
+ my $random_num = int( rand($i64_max) );
+ $indexer->add_doc(
+ { cat => 'random_int64s',
+ name => $random_num,
+ int64 => $random_num,
+ }
+ );
+ push @random_int64s, $random_num;
+ refresh_indexer() if $_ % 10 == 0;
+}
+@random_int64s = sort { $a <=> $b } @random_int64s;
+
+# Add random float32s.
+my @random_float32s;
+for ( 0 .. 99 ) {
+ my $random_num = rand(10);
+ $random_num = unpack( "f", pack( "f", $random_num ) ); # strip precision
+ $indexer->add_doc(
+ { cat => 'random_float32s',
+ name => $random_num,
+ float32 => $random_num,
+ }
+ );
+ push @random_float32s, $random_num;
+ refresh_indexer() if $_ % 10 == 0;
+}
+@random_float32s = sort { $a <=> $b } @random_float32s;
+
+# Add random float64s.
+my @random_float64s;
+for ( 0 .. 99 ) {
+ my $random_num = rand(10);
+ $indexer->add_doc(
+ { cat => 'random_float64s',
+ name => $random_num,
+ float64 => $random_num,
+ }
+ );
+ push @random_float64s, $random_num;
+ refresh_indexer() if $_ % 10 == 0;
+}
+@random_float64s = sort { $a <=> $b } @random_float64s;
+
+# Add numbers to verify consistent ordering.
+for ( shuffle( 0 .. 99 ) ) {
+ $indexer->add_doc(
+ { cat => 'num',
+ name => sprintf( '%02d', $_ ),
+ }
+ );
+ refresh_indexer() if $_ % 10 == 0;
+}
+
+$indexer->commit;
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $results = test_sorted_search( 'vehicle', 100, name => 0 );
+is_deeply( $results, [qw( airplane bike car )], "sort by one criteria" );
+
+SKIP: {
+ skip( "known leaks", 2 ) if $ENV{LUCY_VALGRIND};
+ eval { $results = test_sorted_search( 'vehicle', 100, nope => 0 ) };
+ like( $@, qr/sortable/,
+ "sorting on a non-sortable field throws an error" );
+
+ eval { $results = test_sorted_search( 'vehicle', 100, unknown => 0 ) };
+ like( $@, qr/sortable/, "sorting on an unknown field throws an error" );
+}
+
+$results = test_sorted_search( 'vehicle', 100, weight => 0 );
+is_deeply( $results, [qw( bike car airplane )], "sort by one criteria" );
+
+$results = test_sorted_search( 'vehicle', 100, name => 1 );
+is_deeply( $results, [qw( car bike airplane )], "reverse sort" );
+
+$results = test_sorted_search( 'vehicle', 100, home => 0, name => 0 );
+is_deeply( $results, [qw( airplane bike car )], "multiple criteria" );
+
+$results = test_sorted_search( 'vehicle', 100, home => 0, name => 1 );
+is_deeply( $results, [qw( airplane car bike )],
+ "multiple criteria with reverse" );
+
+$results = test_sorted_search( 'vehicle', 100, speed => 1 );
+my $reversed = test_sorted_search( 'vehicle', 100, sloth => 0 );
+is_deeply( $results, $reversed, "FieldType_Compare_Values" );
+
+$results = test_sorted_search( 'random', 100, name => 0, );
+is_deeply( $results, \@random_strings, "random strings" );
+
+$results = test_sorted_search( 'random_int32s', 100, int32 => 0, );
+is_deeply( $results, \@random_int32s, "int32" );
+
+$results = test_sorted_search( 'random_int64s', 100, int64 => 0, );
+is_deeply( $results, \@random_int64s, "int64" );
+
+$results = test_sorted_search( 'random_float32s', 100, float32 => 0, );
+is_deeply( $results, \@random_float32s, "float32" );
+
+$results = test_sorted_search( 'random_float64s', 100, float64 => 0, );
+is_deeply( $results, \@random_float64s, "float64" );
+
+$results
+ = test_sorted_search( 'bike bike bike car car airplane', 100, unused => 0,
+ );
+is_deeply( $results, [qw( airplane bike car )],
+ "sorting on field with no values sorts by doc id" );
+
+$results = test_sorted_search( '99 OR car', 10, speed => 0 );
+is_deeply( $results, [qw( car 99 )], "doc with NULL value sorts last" );
+
+my $ten_results = test_sorted_search( 'num', 10, name => 0 );
+my $thirty_results = test_sorted_search( 'num', 30, name => 0 );
+my @first_ten_of_thirty = @{$thirty_results}[ 0 .. 9 ];
+is_deeply( $ten_results, \@first_ten_of_thirty,
+ "same order regardless of queue size" );
+
+$ten_results = test_sorted_search( 'num', 10, name => 1 );
+$thirty_results = test_sorted_search( 'num', 30, name => 1 );
+@first_ten_of_thirty = @{$thirty_results}[ 0 .. 9 ];
+is_deeply( $ten_results, \@first_ten_of_thirty,
+ "same order regardless of queue size (reverse sort)" );
+
+# Add another seg to index.
+undef $indexer;
+$indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+$indexer->add_doc(
+ { name => 'carrot',
+ speed => 0,
+ weight => 1,
+ home => 'land',
+ cat => 'food',
+ }
+);
+$indexer->commit;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+$results = test_sorted_search( 'vehicle', 100, name => 0 );
+is_deeply( $results, [qw( airplane bike car )], "Multi-segment sort" );
+
+# Take a list of criteria, create a SortSpec, perform a search, and return an
+# Array of 'name' values for the sorted results.
+sub test_sorted_search {
+ my ( $query, $num_wanted, @criteria ) = @_;
+ my @rules;
+
+ while (@criteria) {
+ my $field = shift @criteria;
+ my $rev = shift @criteria;
+ push @rules,
+ Lucy::Search::SortRule->new(
+ field => $field,
+ reverse => $rev,
+ );
+ }
+ push @rules, Lucy::Search::SortRule->new( type => 'doc_id' );
+ my $sort_spec = Lucy::Search::SortSpec->new( rules => \@rules );
+ my $hits = $searcher->hits(
+ query => $query,
+ sort_spec => $sort_spec,
+ num_wanted => $num_wanted,
+ );
+ my @results;
+ while ( my $hit = $hits->next ) {
+ push @results, $hit->{name};
+ }
+
+ return \@results;
+}
diff --git a/perl/t/513-matcher.t b/perl/t/513-matcher.t
new file mode 100644
index 0000000..5554a19
--- /dev/null
+++ b/perl/t/513-matcher.t
@@ -0,0 +1,127 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MyMatcher;
+use base qw( Lucy::Search::Matcher );
+
+package main;
+
+use Test::More tests => 22;
+
+use LucyX::Search::MockMatcher;
+use Lucy::Test;
+
+my $matcher = MyMatcher->new;
+for (qw( score get_doc_id next )) {
+ eval { $matcher->$_; };
+ like( $@, qr/abstract/i, "$_ is abstract" );
+}
+
+my $got = test_search( docs => [ 1 .. 10 ] );
+is_deeply( $got, [ 1 .. 10 ], "defaults" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [4] );
+is_deeply( $got, [ 1 .. 3, 5 .. 10 ], "deletion between hits" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [5] );
+is_deeply( $got, [ 1 .. 3, 6 .. 10 ], "deletion after gap" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [1] );
+is_deeply( $got, [ 2 .. 3, 5 .. 10 ], "first doc deleted" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [ 1, 2 ] );
+is_deeply( $got, [ 3, 5 .. 10 ], "first two docs deleted" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [10] );
+is_deeply( $got, [ 1 .. 3, 5 .. 9 ], "last doc deleted" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [ 9, 10 ] );
+is_deeply( $got, [ 1 .. 3, 5 .. 8 ], "last two docs deleted" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [ 3, 4 ] );
+is_deeply( $got, [ 1 .. 2, 5 .. 10 ], "deletions continuing into gap" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [ 4, 5 ] );
+is_deeply( $got, [ 1 .. 3, 6 .. 10 ], "deletions continuing from gap" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [ 3, 4, 5 ] );
+is_deeply( $got, [ 1 .. 2, 6 .. 10 ], "deletions spanning gap" );
+
+$got = test_search( docs => [ 1 .. 3, 5 .. 10 ], dels => [ 3, 5 ] );
+is_deeply( $got, [ 1 .. 2, 6 .. 10 ], "deletions surrounding gap" );
+
+$got = test_search( docs => [ 1 .. 3, 5, 7 .. 10 ], dels => [5] );
+is_deeply( $got, [ 1 .. 3, 7 .. 10 ], "gaps surrounding deletion" );
+
+$got = test_search( docs => [ 1, 3, 5, 7, 9 ], dels => [ 2, 4, 6, 8, 10 ] );
+is_deeply( $got, [ 1, 3, 5, 7, 9 ], "synchronized gaps and deletions" );
+
+$got = test_search( docs => [ 1, 3, 5, 7, 9 ], dels => [ 1, 3, 5, 7, 9 ] );
+is_deeply( $got, [], "alternating gaps and deletions" );
+
+$got = test_search( docs => [ 1 .. 3, 6 .. 10 ], dels => [ 4, 5 ] );
+is_deeply( $got, [ 1 .. 3, 6 .. 10 ], "two deletions between hits" );
+
+$got = test_search( docs => [ 1 .. 3, 6 .. 10 ], dels => [3] );
+is_deeply( $got, [ 1 .. 2, 6 .. 10 ], "deletion before double gap" );
+
+$got = test_search( docs => [ 1 .. 3, 6 .. 10 ], dels => [6] );
+is_deeply( $got, [ 1 .. 3, 7 .. 10 ], "deletion after double gap" );
+
+$got = test_search( docs => [ 1 .. 3, 6 .. 10 ], dels => [ 3, 4, 5 ] );
+is_deeply( $got, [ 1 .. 2, 6 .. 10 ],
+ "deletions continuing into double gap" );
+
+$got = test_search( docs => [ 1 .. 3, 6 .. 10 ], dels => [ 4, 5, 6 ] );
+is_deeply(
+ $got,
+ [ 1 .. 3, 7 .. 10 ],
+ "deletions continuing out of double gap"
+);
+
+sub test_search {
+ my %args = @_;
+ my $docs = delete $args{docs} || [];
+ my $dels = delete $args{dels} || [];
+ my $del_enum;
+
+ my $matcher = LucyX::Search::MockMatcher->new(
+ doc_ids => $docs,
+ scores => [ (0) x scalar @$docs ],
+ );
+ if (@$dels) {
+ my $bit_vec
+ = Lucy::Object::BitVector->new( capacity => $dels->[-1] + 1 );
+ $bit_vec->set($_) for @$dels;
+ $del_enum
+ = Lucy::Search::BitVecMatcher->new( bit_vector => $bit_vec );
+ }
+
+ my $collector
+ = Lucy::Search::Collector::SortCollector->new( wanted => 100 );
+ $matcher->collect(
+ %Lucy::Search::Matcher::collect_PARAMS,
+ collector => $collector,
+ deletions => $del_enum,
+ %args,
+ );
+ my $match_docs = $collector->pop_match_docs;
+ my @doc_ids = map { $_->get_doc_id } @$match_docs;
+ return \@doc_ids;
+}
diff --git a/perl/t/514-and_matcher.t b/perl/t/514-and_matcher.t
new file mode 100644
index 0000000..1d9c09b
--- /dev/null
+++ b/perl/t/514-and_matcher.t
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 1362;
+use Lucy::Test;
+use LucyX::Search::MockMatcher;
+use Lucy::Test::TestUtils qw( modulo_set doc_ids_from_td_coll );
+
+my $sim = Lucy::Index::Similarity->new;
+
+for my $interval_a ( reverse 1 .. 17 ) {
+ for my $interval_b ( reverse 10 .. 17 ) {
+ check_matcher( $interval_a, $interval_b );
+ for my $interval_c ( 30, 75 ) {
+ check_matcher( $interval_a, $interval_b, $interval_c );
+ check_matcher( $interval_c, $interval_b, $interval_a );
+ }
+ }
+}
+check_matcher(1000);
+
+sub check_matcher {
+ my @intervals = @_;
+ my @doc_id_arrays = map { modulo_set( $_, 100 ) } @intervals;
+ my @children = map {
+ LucyX::Search::MockMatcher->new(
+ doc_ids => $_,
+ scores => [ (0) x scalar @$_ ],
+ )
+ } @doc_id_arrays;
+ my $and_matcher = Lucy::Search::ANDMatcher->new(
+ children => \@children,
+ similarity => $sim,
+ );
+ my @expected = intersect(@doc_id_arrays);
+ my $collector
+ = Lucy::Search::Collector::SortCollector->new( wanted => 1000 );
+ $and_matcher->collect( collector => $collector );
+ is( $collector->get_total_hits,
+ scalar @expected,
+ "correct num hits @intervals"
+ );
+ my ( $by_score, $by_id ) = doc_ids_from_td_coll($collector);
+ is_deeply( $by_id, \@expected, "correct doc nums @intervals" );
+}
+
+sub intersect {
+ my @arrays = @_;
+ my @out = @{ $arrays[0] };
+ for my $array (@arrays) {
+ my %hash;
+ @hash{@$array} = (1) x @$array;
+ @out = grep { exists $hash{$_} } @out;
+ }
+ return @out;
+}
+
+# Trigger destruction.
+undef $sim;
diff --git a/perl/t/515-range_query.t b/perl/t/515-range_query.t
new file mode 100644
index 0000000..a810b12
--- /dev/null
+++ b/perl/t/515-range_query.t
@@ -0,0 +1,313 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 63;
+use List::Util qw( shuffle );
+use Storable qw( nfreeze thaw );
+use Lucy::Test;
+
+package RangeSchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::StringType->new( sortable => 1 );
+ $self->spec_field( name => 'name', type => $type );
+ $self->spec_field( name => 'cat', type => $type );
+ $self->spec_field( name => 'unused', type => $type );
+ return $self;
+}
+
+package main;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = RangeSchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+my @letters = 'f' .. 't';
+my %letters;
+my $count = 0;
+for my $letter ( shuffle @letters ) {
+ $indexer->add_doc(
+ { name => $letter,
+ cat => 'letter',
+ }
+ );
+ $letters{$letter} = ++$count;
+}
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ upper_term => 'm',
+ include_upper => 1,
+ include_lower => 1,
+ string => 'name:[h TO m]',
+);
+test_results( $results, [ 'h' .. 'm' ], "include lower and upper" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ upper_term => 'm',
+ include_lower => 1,
+);
+test_results(
+ $results,
+ [ 'h' .. 'm' ],
+ "include lower and upper (upper not defined)"
+);
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ upper_term => 'm',
+ include_upper => 1,
+);
+test_results(
+ $results,
+ [ 'h' .. 'm' ],
+ "include lower and upper (lower not defined)"
+);
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ upper_term => 'm',
+);
+test_results(
+ $results,
+ [ 'h' .. 'm' ],
+ "include lower and upper (neither defined)"
+);
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ upper_term => 'm',
+ include_upper => 0,
+ include_lower => 1,
+ string => 'name:[h TO m}',
+);
+test_results( $results, [ 'h' .. 'l' ], "include lower but not upper" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ upper_term => 'm',
+ include_upper => 1,
+ include_lower => 0,
+ string => 'name:{h TO m]',
+);
+test_results( $results, [ 'i' .. 'm' ], "include upper but not lower" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'm',
+ upper_term => 'h',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results( $results, [], "no results when bounds exclude set" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'hh',
+ upper_term => 'm',
+ include_upper => 1,
+ include_lower => 1,
+);
+test_results(
+ $results,
+ [ 'i' .. 'm' ],
+ "included bounds not present in index"
+);
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'hh',
+ upper_term => 'mm',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results(
+ $results,
+ [ 'i' .. 'm' ],
+ "non-included bounds not present in index"
+);
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'e',
+ upper_term => 'tt',
+ include_upper => 1,
+ include_lower => 1,
+);
+test_results(
+ $results,
+ [ 'f' .. 't' ],
+ "included bounds off the end of the lexicon"
+);
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'e',
+ upper_term => 'tt',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results(
+ $results,
+ [ 'f' .. 't' ],
+ "non-included bounds off the end of the lexicon"
+);
+
+$results = test_range_search(
+ field => 'unused',
+ lower_term => 'ff',
+ upper_term => 'tt',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results( $results, [],
+ "range query on field without values produces empty result set" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'a',
+ upper_term => 'e',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results( $results, [],
+ "range query expecting no results returns no results" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'a',
+ upper_term => 'e',
+ include_upper => 1,
+ include_lower => 1,
+);
+test_results( $results, [],
+ "range query expecting no results returns no results" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'u',
+ upper_term => 'z',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results( $results, [],
+ "range query expecting no results returns no results" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'u',
+ upper_term => 'z',
+ include_upper => 1,
+ include_lower => 1,
+);
+test_results( $results, [],
+ "range query expecting no results returns no results" );
+
+$results = test_range_search(
+ field => 'name',
+ upper_term => 'm',
+ string => 'name:[* TO m]',
+);
+test_results( $results, [ 'f' .. 'm' ], "lower term unspecified" );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'h',
+ string => 'name:[h TO *]',
+);
+test_results( $results, [ 'h' .. 't' ], "upper term unspecified" );
+
+eval { $results = test_range_search( field => 'name' ); };
+like( $@, qr/lower_term/,
+ "Failing to supply either lower_term or upper_term throws an exception" );
+
+# Add more docs, test multi-segment searches.
+$indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc(
+ { name => 'mh',
+ cat => 'letter',
+ }
+);
+$indexer->commit;
+$letters{'mh'} = ++$count;
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+$results = test_range_search(
+ field => 'name',
+ lower_term => 'hh',
+ upper_term => 'mm',
+ include_upper => 0,
+ include_lower => 0,
+);
+test_results( $results, [ 'i' .. 'm', 'mh' ], "multi-segment range query" );
+
+# Take a list of args, create a RangeQuery using them, perform a search, and
+# return an array of 'name' values for the sorted results.
+sub test_range_search {
+ my %args = @_;
+ my $string = delete $args{string};
+ my $query = Lucy::Search::RangeQuery->new(%args);
+ if ( defined $string ) {
+ is( $query->to_string, $string );
+ }
+ my $frozen = nfreeze($query);
+ my $thawed = thaw($frozen);
+ ok( $query->equals($thawed), 'equals' );
+
+ my $compiler = $query->make_compiler( searcher => $searcher );
+ $frozen = nfreeze($compiler);
+ $thawed = thaw($frozen);
+ ok( $compiler->equals($thawed), "freeze/thaw compiler" );
+
+ my $hits = $searcher->hits(
+ query => $query,
+ num_wanted => 100,
+ );
+ my @results;
+ while ( my $hit = $hits->next ) {
+ push @results, $hit->{name};
+ }
+
+ return \@results;
+}
+
+sub test_results {
+ my ( $results, $expected, $note ) = @_;
+ @$results = sort @$results;
+ is_deeply( $results, $expected, $note );
+}
diff --git a/perl/t/518-or_scorer.t b/perl/t/518-or_scorer.t
new file mode 100644
index 0000000..f147279
--- /dev/null
+++ b/perl/t/518-or_scorer.t
@@ -0,0 +1,82 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 900;
+use Lucy::Test;
+use LucyX::Search::MockMatcher;
+use Lucy::Test::TestUtils qw( modulo_set doc_ids_from_td_coll );
+
+my $sim = Lucy::Index::Similarity->new;
+
+for my $interval_a ( 1 .. 10 ) {
+ for my $interval_b ( 5 .. 10 ) {
+ check_matcher( $interval_a, $interval_b );
+ for my $interval_c ( 30, 75 ) {
+ check_matcher( $interval_a, $interval_b, $interval_c );
+ check_matcher( $interval_c, $interval_b, $interval_a );
+ }
+ }
+}
+
+sub check_matcher {
+ my @intervals = @_;
+ my @doc_id_arrays = map { modulo_set( $_, 100 ) } @intervals;
+ my $child_matchers
+ = Lucy::Object::VArray->new( capacity => scalar @intervals );
+ for my $doc_id_array (@doc_id_arrays) {
+ my $mock = LucyX::Search::MockMatcher->new(
+ doc_ids => $doc_id_array,
+ scores => [ (1) x scalar @$doc_id_array ],
+ );
+ $child_matchers->push($mock);
+ }
+
+ my $or_scorer = Lucy::Search::ORScorer->new(
+ similarity => $sim,
+ children => $child_matchers,
+ );
+ my $collector
+ = Lucy::Search::Collector::SortCollector->new( wanted => 100 );
+ $or_scorer->collect( collector => $collector );
+ my ( $got_by_score, $got_by_id ) = doc_ids_from_td_coll($collector);
+ my ( $expected_by_count, $expected_by_id )
+ = union_doc_id_sets(@doc_id_arrays);
+ is( scalar @$got_by_id,
+ scalar @$expected_by_id,
+ "total hits: @intervals"
+ );
+ is_deeply( $got_by_id, $expected_by_id, "got all docs: @intervals" );
+ is_deeply( $got_by_score, $expected_by_count,
+ "scores accumulated: @intervals" );
+}
+
+sub union_doc_id_sets {
+ my @arrays = @_;
+ my %scores;
+ for my $array (@arrays) {
+ $scores{$_} += 1 for @$array;
+ }
+ my @by_count_then_id = sort { $scores{$b} <=> $scores{$a} or $a <=> $b }
+ keys %scores;
+ my @by_id = sort { $a <=> $b } keys %scores;
+ return ( \@by_count_then_id, \@by_id );
+}
+
+# Trigger destruction.
+undef $sim;
diff --git a/perl/t/519-req_opt_matcher.t b/perl/t/519-req_opt_matcher.t
new file mode 100644
index 0000000..ca46e48
--- /dev/null
+++ b/perl/t/519-req_opt_matcher.t
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 726;
+use Lucy::Test;
+use LucyX::Search::MockMatcher;
+use Lucy::Test::TestUtils qw( modulo_set doc_ids_from_td_coll );
+
+my $sim = Lucy::Index::Similarity->new;
+
+for my $req_interval ( 1 .. 10, 75 ) {
+ for my $opt_interval ( 1 .. 10, 75 ) {
+ check_matcher( $req_interval, $opt_interval );
+ check_matcher( $opt_interval, $req_interval );
+ }
+}
+
+sub check_matcher {
+ my ( $req_interval, $opt_interval ) = @_;
+ my $req_docs = modulo_set( $req_interval, 100 );
+ my $opt_docs = modulo_set( $opt_interval, 100 );
+ my $req_mock = LucyX::Search::MockMatcher->new(
+ doc_ids => $req_docs,
+ scores => [ (1) x scalar @$req_docs ],
+ );
+ my $opt_mock = LucyX::Search::MockMatcher->new(
+ doc_ids => $opt_docs,
+ scores => [ (1) x scalar @$opt_docs ],
+ );
+ my $req_opt_matcher = Lucy::Search::RequiredOptionalMatcher->new(
+ similarity => $sim,
+ required_matcher => $req_mock,
+ optional_matcher => $opt_mock,
+ );
+ my $collector
+ = Lucy::Search::Collector::SortCollector->new( wanted => 1000 );
+ $req_opt_matcher->collect( collector => $collector );
+ my ( $got_by_score, $got_by_id ) = doc_ids_from_td_coll($collector);
+ my ( $expected_by_count, $expected_by_id )
+ = calc_result_sets( $req_interval, $opt_interval );
+ is( scalar @$got_by_id,
+ scalar @$expected_by_id,
+ "total hits: $req_interval $opt_interval"
+ );
+
+ is_deeply( $got_by_id, $expected_by_id,
+ "got all docs: $req_interval $opt_interval" );
+
+ is_deeply( $got_by_score, $expected_by_count,
+ "scores accumulated: $req_interval $opt_interval" );
+}
+
+sub calc_result_sets {
+ my ( $req_interval, $opt_interval ) = @_;
+
+ my @good;
+ my @better;
+ for my $doc_id ( 1 .. 99 ) {
+ if ( $doc_id % $req_interval == 0 ) {
+ if ( $doc_id % $opt_interval == 0 ) {
+ push @better, $doc_id;
+ }
+ else {
+ push @good, $doc_id;
+ }
+ }
+ }
+ my @by_count_then_id = ( @better, @good );
+ my @by_id = sort { $a <=> $b } @by_count_then_id;
+
+ return ( \@by_count_then_id, \@by_id );
+}
+
+# Trigger destruction.
+undef $sim;
diff --git a/perl/t/520-match_doc.t b/perl/t/520-match_doc.t
new file mode 100644
index 0000000..86d2514
--- /dev/null
+++ b/perl/t/520-match_doc.t
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 7;
+use Storable qw( freeze thaw );
+use Lucy::Test;
+
+my $match_doc = Lucy::Search::MatchDoc->new(
+ doc_id => 31,
+ score => 5.0,
+);
+is( $match_doc->get_doc_id, 31, "get_doc_id" );
+is( $match_doc->get_score, 5.0, "get_score" );
+is( $match_doc->get_values, undef, "get_values" );
+my $match_doc_copy = thaw( freeze($match_doc) );
+is( $match_doc_copy->get_doc_id, $match_doc->get_doc_id,
+ "doc_id survives serialization" );
+is( $match_doc_copy->get_score, $match_doc->get_score,
+ "score survives serialization" );
+is( $match_doc_copy->get_values, $match_doc->get_values,
+ "empty values still empty after serialization" );
+
+my $values = Lucy::Object::VArray->new( capacity => 4 );
+$values->store( 0, Lucy::Object::CharBuf->new("foo") );
+$values->store( 3, Lucy::Object::CharBuf->new("bar") );
+$match_doc = Lucy::Search::MatchDoc->new(
+ doc_id => 120,
+ score => 35,
+ values => $values,
+);
+$match_doc_copy = thaw( freeze($match_doc) );
+is_deeply(
+ $match_doc_copy->get_values,
+ [ 'foo', undef, undef, 'bar' ],
+ "values array survives serialization"
+);
+
diff --git a/perl/t/523-and_query.t b/perl/t/523-and_query.t
new file mode 100644
index 0000000..b6c2dec
--- /dev/null
+++ b/perl/t/523-and_query.t
@@ -0,0 +1,85 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 11;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a', 'b', 'c c c d', 'c d', 'd' .. 'z', );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $reader = $searcher->get_reader->get_seg_readers->[0];
+
+my $a_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'a'
+);
+
+my $b_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'b'
+);
+
+my $and_query = Lucy::Search::ANDQuery->new;
+is( $and_query->to_string, "()", "to_string (empty)" );
+$and_query->add_child($a_query);
+$and_query->add_child($b_query);
+is( $and_query->to_string, "(content:a AND content:b)", "to_string" );
+
+my $frozen = freeze($and_query);
+my $thawed = thaw($frozen);
+ok( $and_query->equals($thawed), "equals" );
+$thawed->set_boost(10);
+ok( !$and_query->equals($thawed), '!equals (boost)' );
+
+my $different_children = Lucy::Search::ANDQuery->new(
+ children => [ $a_query, $a_query ], # a_query added twice
+);
+ok( !$and_query->equals($different_children),
+ '!equals (different children)' );
+
+my $one_child = Lucy::Search::ANDQuery->new( children => [$a_query] );
+ok( !$and_query->equals($one_child), '!equals (too few children)' );
+
+my $and_compiler = $and_query->make_compiler( searcher => $searcher );
+isa_ok( $and_compiler, "Lucy::Search::ANDCompiler", "make_compiler" );
+$frozen = freeze($and_compiler);
+$thawed = thaw($frozen);
+ok( $thawed->equals($and_compiler), "freeze/thaw compiler" );
+
+my $and_matcher = $and_compiler->make_matcher(
+ reader => $reader,
+ need_score => 0,
+);
+isa_ok( $and_matcher, "Lucy::Search::ANDMatcher", "make_matcher" );
+
+my $term_matcher = $one_child->make_compiler( searcher => $searcher )
+ ->make_matcher( reader => $reader, need_score => 0 );
+isa_ok( $term_matcher, "Lucy::Search::TermMatcher",
+ "make_matcher compiles to child's Matcher if there's only one child" );
+
+my $hopeless_query = Lucy::Search::TermQuery->new(
+ field => 'nyet',
+ term => 'nein',
+);
+$and_query->add_child($hopeless_query);
+my $nope = $and_query->make_compiler( searcher => $searcher )
+ ->make_matcher( reader => $reader, need_score => 0 );
+ok( !defined $nope,
+ "If matcher wouldn't return any docs, make_matcher returns undef" );
+
diff --git a/perl/t/524-poly_query.t b/perl/t/524-poly_query.t
new file mode 100644
index 0000000..c49c6df
--- /dev/null
+++ b/perl/t/524-poly_query.t
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 18;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a', 'b', 'c c c d', 'c d', 'd' .. 'z', );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $reader = $searcher->get_reader->get_seg_readers->[0];
+
+my $a_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'a'
+);
+
+my $b_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'b'
+);
+
+for my $conjunction (qw( AND OR )) {
+ my $class = "Lucy::Search::${conjunction}Query";
+ my $polyquery = $class->new( children => [ $a_query, $b_query ] );
+
+ my $frozen = freeze($polyquery);
+ my $thawed = thaw($frozen);
+ ok( $polyquery->equals($thawed), "equals" );
+ $thawed->set_boost(10);
+ ok( !$polyquery->equals($thawed), '!equals (boost)' );
+
+ my $different_kids = $class->new( children => [ $a_query, $a_query ] );
+ ok( !$polyquery->equals($different_kids),
+ '!equals (different children)' );
+
+ my $one_child = $class->new( children => [$a_query] );
+ ok( !$polyquery->equals($one_child), '!equals (too few children)' );
+
+ my $compiler = $polyquery->make_compiler( searcher => $searcher );
+ isa_ok( $compiler, "Lucy::Search::${conjunction}Compiler",
+ "make_compiler" );
+ $frozen = freeze($compiler);
+ $thawed = thaw($frozen);
+ ok( $thawed->equals($compiler), "freeze/thaw compiler" );
+
+ my $matcher
+ = $compiler->make_matcher( reader => $reader, need_score => 1 );
+ my $wanted_class
+ = $conjunction eq 'AND'
+ ? 'Lucy::Search::ANDMatcher'
+ : 'Lucy::Search::ORScorer';
+ isa_ok( $matcher, $wanted_class, "make_matcher with need_score" );
+
+ my $term_matcher = $one_child->make_compiler( searcher => $searcher )
+ ->make_matcher( reader => $reader, need_score => 0 );
+ isa_ok( $term_matcher, "Lucy::Search::TermMatcher",
+ "make_matcher compiles to child's Matcher if there's only one child"
+ );
+
+ my $hopeless_query = Lucy::Search::TermQuery->new(
+ field => 'nyet',
+ term => 'nein',
+ );
+ my $doomed_query = Lucy::Search::TermQuery->new(
+ field => 'luckless',
+ term => 'zero',
+ );
+ $polyquery
+ = $class->new( children => [ $hopeless_query, $doomed_query ] );
+ my $nope = $polyquery->make_compiler( searcher => $searcher )
+ ->make_matcher( reader => $reader, need_score => 0 );
+ ok( !defined $nope,
+ "If Matcher wouldn't return any docs, make_matcher returns undef" );
+}
+
diff --git a/perl/t/525-match_all_query.t b/perl/t/525-match_all_query.t
new file mode 100644
index 0000000..c9b8d22
--- /dev/null
+++ b/perl/t/525-match_all_query.t
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 7;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a' .. 'z' );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $match_all_query = Lucy::Search::MatchAllQuery->new;
+is( $match_all_query->to_string, "[MATCHALL]", "to_string" );
+
+my $hits = $searcher->hits( query => $match_all_query );
+is( $hits->total_hits, 26, "match all" );
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => Lucy::Test::TestSchema->new,
+);
+$indexer->delete_by_term( field => 'content', term => 'b' );
+$indexer->commit;
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => $match_all_query, num_wanted => 100 );
+is( $hits->total_hits, 25, "match all minus a deletion" );
+my @got;
+while ( my $hit = $hits->next ) {
+ push @got, $hit->{content};
+}
+is_deeply( \@got, [ 'a', 'c' .. 'z' ], "correct hits" );
+
+my $frozen = freeze($match_all_query);
+my $thawed = thaw($frozen);
+ok( $match_all_query->equals($thawed), "equals" );
+$thawed->set_boost(10);
+ok( !$match_all_query->equals($thawed), '!equals (boost)' );
+
+my $compiler = $match_all_query->make_compiler( searcher => $searcher );
+$frozen = freeze($compiler);
+$thawed = thaw($frozen);
+ok( $thawed->equals($compiler), "freeze/thaw compiler" );
+
diff --git a/perl/t/526-not_query.t b/perl/t/526-not_query.t
new file mode 100644
index 0000000..2c3c246
--- /dev/null
+++ b/perl/t/526-not_query.t
@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 61;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+use LucyX::Search::MockMatcher;
+
+my @got;
+
+my $folder = create_index( 'a' .. 'z' );
+
+my $b_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'b'
+);
+my $c_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'c'
+);
+my $not_b_query = Lucy::Search::NOTQuery->new( negated_query => $b_query );
+my $not_c_query = Lucy::Search::NOTQuery->new( negated_query => $c_query );
+
+is( $not_b_query->to_string, "-content:b", "to_string" );
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $reader = $searcher->get_reader;
+my $hits = $searcher->hits(
+ query => $not_b_query,
+ num_wanted => 100
+);
+is( $hits->total_hits, 25, "not b" );
+@got = ();
+while ( my $hit = $hits->next ) {
+ push @got, $hit->{content};
+}
+is_deeply( \@got, [ 'a', 'c' .. 'z' ], "correct hits" );
+
+my $frozen = freeze($not_b_query);
+my $thawed = thaw($frozen);
+ok( $not_b_query->equals($thawed), "equals" );
+$thawed->set_boost(10);
+ok( !$not_b_query->equals($thawed), '!equals (boost)' );
+ok( !$not_b_query->equals($not_c_query),
+ "!equals (different negated query)" );
+
+my $compiler = $not_b_query->make_compiler( searcher => $searcher );
+$frozen = freeze($compiler);
+$thawed = thaw($frozen);
+ok( $thawed->equals($compiler), 'freeze/thaw compiler' );
+
+# Air out NOTMatcher with random patterns.
+for my $num_negated ( 1 .. 26 ) {
+ my @source_ids = ( 1 .. 26 );
+ my @mock_ids;
+ for ( 1 .. $num_negated ) {
+ my $tick = int( rand @source_ids );
+ push @mock_ids, splice( @source_ids, $tick, 1 );
+ }
+ @mock_ids = sort { $a <=> $b } @mock_ids;
+ my $mock_matcher = LucyX::Search::MockMatcher->new(
+ doc_ids => \@mock_ids,
+ scores => [ (1) x scalar @mock_ids ],
+ );
+ my $not_matcher = Lucy::Search::NOTMatcher->new(
+ doc_max => $reader->doc_max,
+ negated_matcher => $mock_matcher,
+ );
+ my $bit_vec = Lucy::Object::BitVector->new( capacity => 30 );
+ my $collector
+ = Lucy::Search::Collector::BitCollector->new( bit_vector => $bit_vec,
+ );
+ $not_matcher->collect( collector => $collector );
+ my $got = $bit_vec->to_arrayref;
+ is( scalar @$got, scalar @source_ids, "got all docs ($num_negated)" );
+ is_deeply( $got, \@source_ids, "correct retrieval ($num_negated)" );
+}
+
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => Lucy::Test::TestSchema->new,
+);
+$indexer->delete_by_term( field => 'content', term => 'b' );
+$indexer->commit;
+
+@got = ();
+$searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+$hits = $searcher->hits( query => $not_b_query, num_wanted => 100 );
+is( $hits->total_hits, 25, "still correct after deletion" );
+while ( my $hit = $hits->next ) {
+ push @got, $hit->{content};
+}
+is_deeply( \@got, [ 'a', 'c' .. 'z' ], "correct hits after deletion" );
diff --git a/perl/t/527-req_opt_query.t b/perl/t/527-req_opt_query.t
new file mode 100644
index 0000000..2ccf318
--- /dev/null
+++ b/perl/t/527-req_opt_query.t
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 7;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a', 'b', 'b c', 'c', 'c d', 'd', 'e' );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $reader = $searcher->get_reader->get_seg_readers->[0];
+
+my $b_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'b'
+);
+my $c_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'c'
+);
+my $x_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'x'
+);
+
+my $req_opt_query = Lucy::Search::RequiredOptionalQuery->new(
+ required_query => $b_query,
+ optional_query => $c_query,
+);
+is( $req_opt_query->to_string, "(+content:b content:c)", "to_string" );
+
+my $compiler = $req_opt_query->make_compiler( searcher => $searcher );
+my $frozen = freeze($compiler);
+my $thawed = thaw($frozen);
+ok( $thawed->equals($compiler), "freeze/thaw compiler" );
+my $matcher = $compiler->make_matcher( reader => $reader, need_score => 1 );
+isa_ok( $matcher, 'Lucy::Search::RequiredOptionalMatcher' );
+
+$req_opt_query = Lucy::Search::RequiredOptionalQuery->new(
+ required_query => $b_query,
+ optional_query => $x_query,
+);
+$matcher = $req_opt_query->make_compiler( searcher => $searcher )
+ ->make_matcher( reader => $reader, need_score => 0 );
+isa_ok( $matcher, 'Lucy::Search::TermMatcher',
+ "return required matcher only when opt matcher doesn't match" );
+
+$req_opt_query = Lucy::Search::RequiredOptionalQuery->new(
+ required_query => $x_query,
+ optional_query => $b_query,
+);
+$matcher = $req_opt_query->make_compiler( searcher => $searcher )
+ ->make_matcher( reader => $reader, need_score => 0 );
+ok( !defined($matcher), "if required matcher has no match, return undef" );
+
+$frozen = freeze($req_opt_query);
+$thawed = thaw($frozen);
+ok( $req_opt_query->equals($thawed), "equals" );
+$thawed->set_boost(10);
+ok( !$req_opt_query->equals($thawed), '!equals (boost)' );
+
diff --git a/perl/t/528-leaf_query.t b/perl/t/528-leaf_query.t
new file mode 100644
index 0000000..2f83dec
--- /dev/null
+++ b/perl/t/528-leaf_query.t
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 12;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a', 'b', 'b c', 'c', 'c d', 'd', 'e' );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $reader = $searcher->get_reader;
+
+my $leaf_query = Lucy::Search::LeafQuery->new(
+ field => 'content',
+ text => 'b'
+);
+my $no_field_leaf_query = Lucy::Search::LeafQuery->new( text => 'b' );
+is( $leaf_query->to_string, "content:b", "to_string" );
+is( $no_field_leaf_query->to_string, "b", "no field to_string" );
+
+is( $leaf_query->get_field, "content", "get field" );
+ok( !defined $no_field_leaf_query->get_field, "get null field" );
+is( $leaf_query->get_text, 'b', 'get text' );
+
+ok( !$leaf_query->equals($no_field_leaf_query), "!equals (field/nofield)" );
+ok( !$no_field_leaf_query->equals($leaf_query), "!equals (nofield/field)" );
+
+my $diff_field = Lucy::Search::LeafQuery->new( field => 'oink', text => 'b' );
+ok( !$diff_field->equals($leaf_query), "!equals (different field)" );
+
+my $diff_text
+ = Lucy::Search::LeafQuery->new( field => 'content', text => 'c' );
+ok( !$diff_text->equals($leaf_query), "!equals (different text)" );
+
+eval { $leaf_query->make_compiler( searcher => $searcher ); };
+like( $@, qr/Make_Compiler/, "Make_Compiler throws error" );
+
+my $frozen = freeze($leaf_query);
+my $thawed = thaw($frozen);
+ok( $leaf_query->equals($thawed), "freeze/thaw and equals" );
+$leaf_query->set_boost(2);
+ok( !$leaf_query->equals($thawed), "!equals (boost)" );
+
diff --git a/perl/t/529-no_match_query.t b/perl/t/529-no_match_query.t
new file mode 100644
index 0000000..d78f1a8
--- /dev/null
+++ b/perl/t/529-no_match_query.t
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 5;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = create_index( 'a' .. 'z' );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $no_match_query = Lucy::Search::NoMatchQuery->new;
+is( $no_match_query->to_string, "[NOMATCH]", "to_string" );
+
+my $hits = $searcher->hits( query => $no_match_query );
+is( $hits->total_hits, 0, "no matches" );
+
+my $frozen = freeze($no_match_query);
+my $thawed = thaw($frozen);
+ok( $no_match_query->equals($thawed), "equals" );
+$thawed->set_boost(10);
+ok( !$no_match_query->equals($thawed), '!equals (boost)' );
+
+my $compiler = $no_match_query->make_compiler( searcher => $searcher );
+$frozen = freeze($compiler);
+$thawed = thaw($frozen);
+ok( $compiler->equals($thawed), "freeze/thaw compiler" );
diff --git a/perl/t/532-sort_collector.t b/perl/t/532-sort_collector.t
new file mode 100644
index 0000000..6d0ecda
--- /dev/null
+++ b/perl/t/532-sort_collector.t
@@ -0,0 +1,111 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 32;
+use Lucy::Test;
+use List::Util qw( shuffle );
+use LucyX::Search::MockMatcher;
+
+my $schema = Lucy::Plan::Schema->new;
+my $type = Lucy::Plan::StringType->new( sortable => 1 );
+$schema->spec_field( name => 'letter', type => $type );
+$schema->spec_field( name => 'number', type => $type );
+$schema->spec_field( name => 'id', type => $type );
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+
+my @letters = 'a' .. 'z';
+my @numbers = 1 .. 5;
+my @docs = (
+ { letter => 'c', number => '4', id => 1, },
+ { letter => 'b', number => '2', id => 2, },
+ { letter => 'a', number => '5', id => 3, },
+);
+for my $id ( 4 .. 100 ) {
+ my $doc = {
+ letter => $letters[ rand @letters ],
+ number => $numbers[ rand @numbers ],
+ id => $id,
+ };
+ push @docs, $doc;
+}
+$indexer->add_doc($_) for @docs;
+$indexer->commit;
+
+my $polyreader = Lucy::Index::IndexReader->open( index => $folder );
+my $seg_reader = $polyreader->get_seg_readers->[0];
+
+my $by_letter = Lucy::Search::SortSpec->new(
+ rules => [
+ Lucy::Search::SortRule->new( field => 'letter' ),
+ Lucy::Search::SortRule->new( type => 'doc_id' ),
+ ]
+);
+
+my $collector = Lucy::Search::Collector::SortCollector->new(
+ sort_spec => $by_letter,
+ schema => $schema,
+ wanted => 1,
+);
+
+$collector->set_reader($seg_reader);
+$collector->collect($_) for 1 .. 100;
+my $match_docs = $collector->pop_match_docs;
+is( $match_docs->[0]->get_doc_id,
+ 3, "Early doc numbers preferred by collector" );
+
+my @docs_and_scores;
+my %uniq_doc_ids;
+for ( 1 .. 30 ) {
+ my $doc_num = int( rand(10000) ) + 1;
+ while ( $uniq_doc_ids{$doc_num} ) {
+ $doc_num = int( rand(10000) ) + 1;
+ }
+ $uniq_doc_ids{$doc_num} = 1;
+ push @docs_and_scores, [ $doc_num, rand(10) ];
+}
+@docs_and_scores = sort { $a->[0] <=> $b->[0] } @docs_and_scores;
+my @ranked
+ = sort { $b->[1] <=> $a->[1] || $a->[1] <=> $b->[1] } @docs_and_scores;
+my @doc_ids = map { $_->[0] } @docs_and_scores;
+my @scores = map { $_->[1] } @docs_and_scores;
+
+for my $size ( 0 .. @doc_ids ) {
+ my $matcher = LucyX::Search::MockMatcher->new(
+ doc_ids => \@doc_ids,
+ scores => \@scores,
+ );
+ my $collector
+ = Lucy::Search::Collector::SortCollector->new( wanted => $size, );
+ $collector->set_matcher($matcher);
+ $matcher->collect( collector => $collector );
+
+ my @wanted;
+ if ($size) {
+ @wanted = map { $_->[0] } @ranked[ 0 .. $size - 1 ];
+ }
+ else {
+ @wanted = ();
+ }
+ my @got = map { $_->get_doc_id } @{ $collector->pop_match_docs };
+ is_deeply( \@got, \@wanted, "random docs and scores, wanted = $size" );
+}
diff --git a/perl/t/601-queryparser.t b/perl/t/601-queryparser.t
new file mode 100644
index 0000000..6e29bcd
--- /dev/null
+++ b/perl/t/601-queryparser.t
@@ -0,0 +1,239 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+package PlainSchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new( pattern => '\S+' );
+ my $type = Lucy::Plan::FullTextType->new( analyzer => $tokenizer, );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package StopSchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $whitespace_tokenizer
+ = Lucy::Analysis::RegexTokenizer->new( token_re => qr/\S+/ );
+ my $stopfilter
+ = Lucy::Analysis::SnowballStopFilter->new( stoplist => { x => 1 } );
+ my $polyanalyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $whitespace_tokenizer, $stopfilter, ], );
+ my $type = Lucy::Plan::FullTextType->new( analyzer => $polyanalyzer, );
+ $self->spec_field( name => 'content', type => $type );
+ return $self;
+}
+
+package MyTermQuery;
+use base qw( Lucy::Search::TermQuery );
+
+package MyPhraseQuery;
+use base qw( Lucy::Search::PhraseQuery );
+
+package MyANDQuery;
+use base qw( Lucy::Search::ANDQuery );
+
+package MyORQuery;
+use base qw( Lucy::Search::ORQuery );
+
+package MyNOTQuery;
+use base qw( Lucy::Search::NOTQuery );
+
+package MyReqOptQuery;
+use base qw( Lucy::Search::RequiredOptionalQuery );
+
+package MyQueryParser;
+use base qw( Lucy::Search::QueryParser );
+
+sub make_term_query { shift; MyTermQuery->new(@_) }
+sub make_phrase_query { shift; MyPhraseQuery->new(@_) }
+sub make_and_query { shift; MyANDQuery->new( children => shift ) }
+sub make_or_query { shift; MyORQuery->new( children => shift ) }
+sub make_not_query { shift; MyNOTQuery->new( negated_query => shift ) }
+sub make_req_opt_query { shift; MyReqOptQuery->new(@_) }
+
+package main;
+use Test::More tests => 224;
+use Lucy::Util::StringHelper qw( utf8_flag_on utf8ify );
+use Lucy::Test::TestUtils qw( create_index );
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $stop_folder = Lucy::Store::RAMFolder->new;
+my $plain_schema = PlainSchema->new;
+my $stop_schema = StopSchema->new;
+
+my @docs = ( 'x', 'y', 'z', 'x a', 'x a b', 'x a b c', 'x foo a b c d', );
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $plain_schema,
+);
+my $stop_indexer = Lucy::Index::Indexer->new(
+ index => $stop_folder,
+ schema => $stop_schema,
+);
+
+for (@docs) {
+ $indexer->add_doc( { content => $_ } );
+ $stop_indexer->add_doc( { content => $_ } );
+}
+$indexer->commit;
+$stop_indexer->commit;
+
+my $OR_parser = Lucy::Search::QueryParser->new( schema => $plain_schema, );
+my $AND_parser = Lucy::Search::QueryParser->new(
+ schema => $plain_schema,
+ default_boolop => 'AND',
+);
+$OR_parser->set_heed_colons(1);
+$AND_parser->set_heed_colons(1);
+
+my $OR_stop_parser
+ = Lucy::Search::QueryParser->new( schema => $stop_schema, );
+my $AND_stop_parser = Lucy::Search::QueryParser->new(
+ schema => $stop_schema,
+ default_boolop => 'AND',
+);
+$OR_stop_parser->set_heed_colons(1);
+$AND_stop_parser->set_heed_colons(1);
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+my $stop_searcher = Lucy::Search::IndexSearcher->new( index => $stop_folder );
+
+my @logical_tests = (
+
+ 'b' => [ 3, 3, 3, 3, ],
+ '(a)' => [ 4, 4, 4, 4, ],
+ '"a"' => [ 4, 4, 4, 4, ],
+ '"(a)"' => [ 0, 0, 0, 0, ],
+ '("a")' => [ 4, 4, 4, 4, ],
+
+ 'a b' => [ 4, 3, 4, 3, ],
+ 'a (b)' => [ 4, 3, 4, 3, ],
+ 'a "b"' => [ 4, 3, 4, 3, ],
+ 'a ("b")' => [ 4, 3, 4, 3, ],
+ 'a "(b)"' => [ 4, 0, 4, 0, ],
+
+ '(a b)' => [ 4, 3, 4, 3, ],
+ '"a b"' => [ 3, 3, 3, 3, ],
+ '("a b")' => [ 3, 3, 3, 3, ],
+ '"(a b)"' => [ 0, 0, 0, 0, ],
+
+ 'a b c' => [ 4, 2, 4, 2, ],
+ 'a (b c)' => [ 4, 2, 4, 2, ],
+ 'a "b c"' => [ 4, 2, 4, 2, ],
+ 'a ("b c")' => [ 4, 2, 4, 2, ],
+ 'a "(b c)"' => [ 4, 0, 4, 0, ],
+ '"a b c"' => [ 2, 2, 2, 2, ],
+
+ '-x' => [ 0, 0, 0, 0, ],
+ 'x -c' => [ 3, 3, 0, 0, ],
+ 'x "-c"' => [ 5, 0, 0, 0, ],
+ 'x +c' => [ 2, 2, 2, 2, ],
+ 'x "+c"' => [ 5, 0, 0, 0, ],
+
+ '+x +c' => [ 2, 2, 2, 2, ],
+ '+x -c' => [ 3, 3, 0, 0, ],
+ '-x +c' => [ 0, 0, 2, 2, ],
+ '-x -c' => [ 0, 0, 0, 0, ],
+
+ 'x y' => [ 6, 0, 1, 1, ],
+ 'x a d' => [ 5, 1, 4, 1, ],
+ 'x "a d"' => [ 5, 0, 0, 0, ],
+ '"x a"' => [ 3, 3, 4, 4, ],
+
+ 'x AND y' => [ 0, 0, 1, 1, ],
+ 'x OR y' => [ 6, 6, 1, 1, ],
+ 'x AND NOT y' => [ 5, 5, 0, 0, ],
+
+ 'x (b OR c)' => [ 5, 3, 3, 3, ],
+ 'x AND (b OR c)' => [ 3, 3, 3, 3, ],
+ 'x OR (b OR c)' => [ 5, 5, 3, 3, ],
+ 'x (y OR c)' => [ 6, 2, 3, 3, ],
+ 'x AND (y OR c)' => [ 2, 2, 3, 3, ],
+
+ 'a AND NOT (b OR "c d")' => [ 1, 1, 1, 1, ],
+ 'a AND NOT "a b"' => [ 1, 1, 1, 1, ],
+ 'a AND NOT ("a b" OR "c d")' => [ 1, 1, 1, 1, ],
+
+ '+"b c" -d' => [ 1, 1, 1, 1, ],
+ '"a b" +d' => [ 1, 1, 1, 1, ],
+
+ 'x AND NOT (b OR (c AND d))' => [ 2, 2, 0, 0, ],
+
+ '-(+notthere)' => [ 0, 0, 0, 0 ],
+
+ 'content:b' => [ 3, 3, 3, 3, ],
+ 'bogusfield:a' => [ 0, 0, 0, 0, ],
+ 'bogusfield:a content:b' => [ 3, 0, 3, 0, ],
+
+ 'content:b content:c' => [ 3, 2, 3, 2 ],
+ 'content:(b c)' => [ 3, 2, 3, 2 ],
+ 'bogusfield:(b c)' => [ 0, 0, 0, 0 ],
+
+);
+
+my $i = 0;
+while ( $i < @logical_tests ) {
+ my $qstring = $logical_tests[$i];
+ $i++;
+
+ my $query = $OR_parser->parse($qstring);
+ my $hits = $searcher->hits( query => $query );
+ is( $hits->total_hits, $logical_tests[$i][0], "OR: $qstring" );
+
+ $query = $AND_parser->parse($qstring);
+ $hits = $searcher->hits( query => $query );
+ is( $hits->total_hits, $logical_tests[$i][1], "AND: $qstring" );
+
+ $query = $OR_stop_parser->parse($qstring);
+ $hits = $stop_searcher->hits( query => $query );
+ is( $hits->total_hits, $logical_tests[$i][2], "stoplist-OR: $qstring" );
+
+ $query = $AND_stop_parser->parse($qstring);
+ $hits = $stop_searcher->hits( query => $query );
+ is( $hits->total_hits, $logical_tests[$i][3],
+ "stoplist-AND: $qstring" );
+
+ $i++;
+}
+
+my $motorhead = "Mot\xF6rhead";
+utf8ify($motorhead);
+my $unicode_folder = create_index($motorhead);
+$searcher = Lucy::Search::IndexSearcher->new( index => $unicode_folder );
+
+my $hits = $searcher->hits( query => 'Mot' );
+is( $hits->total_hits, 0, "Pre-test - indexing worked properly" );
+$hits = $searcher->hits( query => $motorhead );
+is( $hits->total_hits, 1, "QueryParser parses UTF-8 strings correctly" );
+
+my $custom_parser = MyQueryParser->new( schema => PlainSchema->new );
+isa_ok( $custom_parser->parse('foo'), 'MyTermQuery' );
+isa_ok( $custom_parser->parse('"foo bar"'), 'MyPhraseQuery' );
+isa_ok( $custom_parser->parse('foo AND bar'), 'MyANDQuery' );
+isa_ok( $custom_parser->parse('foo OR bar'), 'MyORQuery' );
+isa_ok( $custom_parser->tree('NOT foo'), 'MyNOTQuery' );
+isa_ok( $custom_parser->parse('+foo bar'), 'MyReqOptQuery' );
diff --git a/perl/t/602-boosts.t b/perl/t/602-boosts.t
new file mode 100644
index 0000000..e59de86
--- /dev/null
+++ b/perl/t/602-boosts.t
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+package ControlSchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new );
+ $self->spec_field( name => 'content', type => $type );
+ $self->spec_field( name => 'category', type => $type );
+ return $self;
+}
+
+package BoostedFieldSchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+ my $plain_type = Lucy::Plan::FullTextType->new( analyzer => $tokenizer );
+ my $boosted_type = Lucy::Plan::FullTextType->new(
+ analyzer => $tokenizer,
+ boost => 100,
+ );
+ $self->spec_field( name => 'content', type => $plain_type );
+ $self->spec_field( name => 'category', type => $boosted_type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 3;
+
+my $control_folder = Lucy::Store::RAMFolder->new;
+my $boosted_doc_folder = Lucy::Store::RAMFolder->new;
+my $boosted_field_folder = Lucy::Store::RAMFolder->new;
+my $control_indexer = Lucy::Index::Indexer->new(
+ schema => ControlSchema->new,
+ index => $control_folder,
+);
+my $boosted_field_indexer = Lucy::Index::Indexer->new(
+ schema => BoostedFieldSchema->new,
+ index => $boosted_field_folder,
+);
+my $boosted_doc_indexer = Lucy::Index::Indexer->new(
+ schema => ControlSchema->new,
+ index => $boosted_doc_folder,
+);
+
+my %source_docs = (
+ 'x' => '',
+ 'x a a a a' => 'x a',
+ 'a b' => 'x a a',
+);
+
+while ( my ( $content, $cat ) = each %source_docs ) {
+ my %fields = (
+ content => $content,
+ category => $cat,
+ );
+ $control_indexer->add_doc( \%fields );
+ $boosted_field_indexer->add_doc( \%fields );
+
+ my $boost = $content =~ /b/ ? 2 : 1;
+ $boosted_doc_indexer->add_doc( doc => \%fields, boost => $boost );
+}
+
+$control_indexer->commit;
+$boosted_field_indexer->commit;
+$boosted_doc_indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $control_folder, );
+my $hits = $searcher->hits( query => 'a' );
+my $hit = $hits->next;
+is( $hit->{content}, "x a a a a", "best doc ranks highest with no boosting" );
+
+$searcher
+ = Lucy::Search::IndexSearcher->new( index => $boosted_field_folder, );
+$hits = $searcher->hits( query => 'a' );
+$hit = $hits->next;
+is( $hit->{content}, 'a b', "boost in FieldType works" );
+
+$searcher = Lucy::Search::IndexSearcher->new( index => $boosted_doc_folder, );
+$hits = $searcher->hits( query => 'a' );
+$hit = $hits->next;
+is( $hit->{content}, 'a b', "boost from \$doc->set_boost works" );
diff --git a/perl/t/603-query_boosts.t b/perl/t/603-query_boosts.t
new file mode 100644
index 0000000..6018661
--- /dev/null
+++ b/perl/t/603-query_boosts.t
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 2;
+use Lucy::Test::TestUtils qw( create_index );
+
+my $doc_1
+ = 'a a a a a a a a a a a a a a a a a a a b c d x y ' . ( 'z ' x 100 );
+my $doc_2 = 'a b c d x y x y ' . ( 'z ' x 100 );
+
+my $folder = create_index( $doc_1, $doc_2 );
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $a_query = Lucy::Search::TermQuery->new(
+ field => 'content',
+ term => 'a',
+);
+my $x_y_query = Lucy::Search::PhraseQuery->new(
+ field => 'content',
+ terms => [qw( x y )],
+);
+
+my $combined_query
+ = Lucy::Search::ORQuery->new( children => [ $a_query, $x_y_query ], );
+my $hits = $searcher->hits( query => $combined_query );
+my $hit = $hits->next;
+is( $hit->{content}, $doc_1, "best doc ranks highest with no boosting" );
+
+$x_y_query->set_boost(2);
+$hits = $searcher->hits( query => $combined_query );
+$hit = $hits->next;
+is( $hit->{content}, $doc_2, "boosting a sub query succeeds" );
diff --git a/perl/t/604-simple_search.t b/perl/t/604-simple_search.t
new file mode 100644
index 0000000..cbf032f
--- /dev/null
+++ b/perl/t/604-simple_search.t
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new, );
+ $self->spec_field( name => 'title', type => $type );
+ $self->spec_field( name => 'body', type => $type );
+ return $self;
+}
+
+package main;
+
+use Test::More tests => 12;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+my %docs = (
+ 'a' => 'foo',
+ 'b' => 'bar',
+);
+
+while ( my ( $title, $body ) = each %docs ) {
+ $indexer->add_doc(
+ { title => $title,
+ body => $body,
+ }
+ );
+}
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $tokenizer = Lucy::Analysis::RegexTokenizer->new;
+my $or_parser = Lucy::Search::QueryParser->new(
+ schema => $schema,
+ analyzer => $tokenizer,
+ fields => [ 'title', 'body', ],
+);
+my $and_parser = Lucy::Search::QueryParser->new(
+ schema => $schema,
+ analyzer => $tokenizer,
+ fields => [ 'title', 'body', ],
+ default_boolop => 'AND',
+);
+
+sub test_qstring {
+ my ( $qstring, $expected, $message ) = @_;
+
+ my $hits = $searcher->hits( query => $qstring );
+ is( $hits->total_hits, $expected, $message );
+
+ my $query = $or_parser->parse($qstring);
+ $hits = $searcher->hits( query => $query );
+ is( $hits->total_hits, $expected, "OR: $message" );
+
+ $query = $and_parser->parse($qstring);
+ $hits = $searcher->hits( query => $query );
+ is( $hits->total_hits, $expected, "AND: $message" );
+}
+
+test_qstring( 'a foo', 1, "simple match across multiple fields" );
+test_qstring( 'a -foo', 0,
+ "match of negated term on any field should exclude document" );
+test_qstring(
+ 'a +foo',
+ 1,
+ "failure to match of required term on a field "
+ . "should not exclude doc if another field matches."
+);
+test_qstring( '+a +foo', 1,
+ "required terms spread across disparate fields should match" );
diff --git a/perl/t/605-store_pos_boost.t b/perl/t/605-store_pos_boost.t
new file mode 100644
index 0000000..4f07d0a
--- /dev/null
+++ b/perl/t/605-store_pos_boost.t
@@ -0,0 +1,122 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MyRegexTokenizer;
+use base qw( Lucy::Analysis::Analyzer );
+use Lucy::Analysis::Inversion;
+
+sub transform {
+ my ( $self, $inversion ) = @_;
+ my $new_inversion = Lucy::Analysis::Inversion->new;
+
+ while ( my $token = $inversion->next ) {
+ for ( $token->get_text ) {
+ my $this_time = /z/ ? 1 : 0;
+ # Accumulate token start_offsets and end_offsets.
+ while (/(\w)/g) {
+ # Special boost just for one doc.
+ my $boost = ( $1 eq 'a' and $this_time ) ? 100 : 1;
+ $new_inversion->append(
+ Lucy::Analysis::Token->new(
+ text => "$1",
+ start_offset => $-[0],
+ end_offset => $+[0],
+ boost => $boost,
+ ),
+ );
+ }
+ }
+ }
+
+ return $new_inversion;
+}
+
+sub equals {
+ my ( $self, $other ) = @_;
+ return 0 unless ref($self) eq ref($other);
+ return 1;
+}
+
+package RichSim;
+use base qw( Lucy::Index::Similarity );
+use Lucy::Index::Posting::RichPosting;
+
+sub make_posting {
+ Lucy::Index::Posting::RichPosting->new( similarity => shift );
+}
+
+package MySchema::boosted;
+use base qw( Lucy::Plan::FullTextType );
+
+sub make_similarity { RichSim->new }
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $plain_type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new );
+ my $boosted_type
+ = MySchema::boosted->new( analyzer => MyRegexTokenizer->new, );
+ $self->spec_field( name => 'plain', type => $plain_type );
+ $self->spec_field( name => 'boosted', type => $boosted_type );
+ return $self;
+}
+
+package main;
+
+use Test::More tests => 2;
+
+my $good = "x x x a a x x x x x x x x";
+my $better = "x x x a a a x x x x x x x";
+my $best = "x x x a a a a a a a a a a";
+my $boosted = "z x x a x x x x x x x x x";
+
+my $schema = MySchema->new;
+my $folder = Lucy::Store::RAMFolder->new;
+my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+
+for ( $good, $better, $best, $boosted ) {
+ $indexer->add_doc( { plain => $_, boosted => $_ } );
+}
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $q_for_plain = Lucy::Search::TermQuery->new(
+ field => 'plain',
+ term => 'a',
+);
+my $hits = $searcher->hits( query => $q_for_plain );
+is( $hits->next->{plain},
+ $best, "verify that search on unboosted field returns best match" );
+
+my $q_for_boosted = Lucy::Search::TermQuery->new(
+ field => 'boosted',
+ term => 'a',
+);
+$hits = $searcher->hits( query => $q_for_boosted );
+is( $hits->next->{boosted},
+ $boosted, "artificially boosted token overrides better match" );
+
diff --git a/perl/t/607-queryparser_multi_field.t b/perl/t/607-queryparser_multi_field.t
new file mode 100644
index 0000000..064593d
--- /dev/null
+++ b/perl/t/607-queryparser_multi_field.t
@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+package MultiFieldSchema;
+use base qw( Lucy::Plan::Schema );
+use Lucy::Analysis::RegexTokenizer;
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $plain_type = Lucy::Plan::FullTextType->new(
+ analyzer => Lucy::Analysis::RegexTokenizer->new );
+ my $not_analyzed_type = Lucy::Plan::StringType->new;
+ $self->spec_field( name => 'a', type => $plain_type );
+ $self->spec_field( name => 'b', type => $plain_type );
+ $self->spec_field( name => 'c', type => $not_analyzed_type );
+ return $self;
+}
+
+package main;
+use Test::More tests => 13;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MultiFieldSchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ index => $folder,
+ schema => $schema,
+);
+$indexer->add_doc( { a => 'foo' } );
+$indexer->add_doc( { b => 'foo' } );
+$indexer->add_doc( { a => 'United States unit state' } );
+$indexer->add_doc( { a => 'unit state' } );
+$indexer->add_doc( { c => 'unit' } );
+$indexer->commit;
+
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $hits = $searcher->hits( query => 'foo' );
+is( $hits->total_hits, 2, "Searcher's default is to find all fields" );
+
+my $qparser = Lucy::Search::QueryParser->new( schema => $schema );
+
+my $foo_leaf = Lucy::Search::LeafQuery->new( text => 'foo' );
+my $multi_field_foo = Lucy::Search::ORQuery->new;
+$multi_field_foo->add_child(
+ Lucy::Search::TermQuery->new(
+ field => $_,
+ term => 'foo'
+ )
+) for qw( a b c );
+my $expanded = $qparser->expand($foo_leaf);
+ok( $multi_field_foo->equals($expanded), "Expand LeafQuery" );
+
+my $multi_field_bar = Lucy::Search::ORQuery->new;
+$multi_field_bar->add_child(
+ Lucy::Search::TermQuery->new(
+ field => $_,
+ term => 'bar'
+ )
+) for qw( a b c );
+my $not_multi_field_bar
+ = Lucy::Search::NOTQuery->new( negated_query => $multi_field_bar );
+my $bar_leaf = Lucy::Search::LeafQuery->new( text => 'bar' );
+my $not_bar_leaf = Lucy::Search::NOTQuery->new( negated_query => $bar_leaf );
+$expanded = $qparser->expand($not_bar_leaf);
+ok( $not_multi_field_bar->equals($expanded), "Expand NOTQuery" );
+
+my $query = $qparser->parse('foo');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 2, "QueryParser's default is to find all fields" );
+
+$query = $qparser->parse('b:foo');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 0, "no set_heed_colons" );
+
+$qparser->set_heed_colons(1);
+$query = $qparser->parse('b:foo');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 1, "set_heed_colons" );
+
+$query = $qparser->parse('a:boffo.moffo');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 0,
+ "no crash for non-existent phrases under heed_colons" );
+
+$query = $qparser->parse('a:x.nope');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 0,
+ "no crash for non-existent terms under heed_colons" );
+
+$query = $qparser->parse('nyet:x.x');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 0,
+ "no crash for non-existent fields under heed_colons" );
+
+$qparser = Lucy::Search::QueryParser->new(
+ schema => $schema,
+ fields => ['a'],
+);
+$query = $qparser->parse('foo');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 1, "QueryParser fields param works" );
+
+my $analyzer_parser = Lucy::Search::QueryParser->new(
+ schema => $schema,
+ analyzer => Lucy::Analysis::PolyAnalyzer->new( language => 'en' ),
+);
+
+$hits = $searcher->hits( query => 'United States' );
+is( $hits->total_hits, 1, "search finds 1 doc (prep for next text)" );
+
+$query = $analyzer_parser->parse('unit');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 3, "QueryParser uses supplied Analyzer" );
+
+$query = $analyzer_parser->parse('United States');
+$hits = $searcher->hits( query => $query );
+is( $hits->total_hits, 3,
+ "QueryParser uses supplied analyzer even for non-analyzed fields" );
+
diff --git a/perl/t/610-queryparser_logic.t b/perl/t/610-queryparser_logic.t
new file mode 100644
index 0000000..b2a4744
--- /dev/null
+++ b/perl/t/610-queryparser_logic.t
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use Lucy::Test;
+Lucy::Test::run_tests("TestQueryParserLogic");
+
diff --git a/perl/t/611-queryparser_syntax.t b/perl/t/611-queryparser_syntax.t
new file mode 100644
index 0000000..2d428b3
--- /dev/null
+++ b/perl/t/611-queryparser_syntax.t
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use lib 'buildlib';
+use Lucy::Test;
+
+package MySchema;
+use base qw( Lucy::Plan::Schema );
+
+sub new {
+ my $self = shift->SUPER::new(@_);
+ my $tokenizer = Lucy::Analysis::RegexTokenizer->new( pattern => '\S+' );
+ my $wordchar_tokenizer
+ = Lucy::Analysis::RegexTokenizer->new( pattern => '\w+', );
+ my $stopfilter
+ = Lucy::Analysis::SnowballStopFilter->new( stoplist => { x => 1 } );
+ my $fancy_analyzer = Lucy::Analysis::PolyAnalyzer->new(
+ analyzers => [ $wordchar_tokenizer, $stopfilter, ], );
+
+ my $plain = Lucy::Plan::FullTextType->new( analyzer => $tokenizer );
+ my $fancy = Lucy::Plan::FullTextType->new( analyzer => $fancy_analyzer );
+ $self->spec_field( name => 'plain', type => $plain );
+ $self->spec_field( name => 'fancy', type => $fancy );
+ return $self;
+}
+
+package main;
+
+# Build index.
+my $doc_set = Lucy::Test::TestUtils::doc_set()->to_perl;
+my $folder = Lucy::Store::RAMFolder->new;
+my $schema = MySchema->new;
+my $indexer = Lucy::Index::Indexer->new(
+ schema => $schema,
+ index => $folder,
+);
+for my $content_string (@$doc_set) {
+ $indexer->add_doc(
+ { plain => $content_string,
+ fancy => $content_string,
+ }
+ );
+}
+$indexer->commit;
+
+Lucy::Test::Search::TestQueryParserSyntax::run_tests($folder);
+
diff --git a/perl/t/613-proximityquery.t b/perl/t/613-proximityquery.t
new file mode 100644
index 0000000..ad9fba2
--- /dev/null
+++ b/perl/t/613-proximityquery.t
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 11;
+use Storable qw( freeze thaw );
+use Lucy::Test;
+use Lucy::Test::TestUtils qw( create_index );
+use LucyX::Search::ProximityQuery;
+
+# this is better than 'x a b c d a b c d' because its
+# posting weight is higher, presumably because
+# it is a shorter doc (higher density?)
+my $best_match = 'a b c d x x a';
+
+my @docs = (
+ 1 .. 20,
+ 'a b c a b c a b c d',
+ 'x a b c d a b c d',
+ 'a c b d', 'a x x x b x x x c x x x x x x d x',
+ $best_match, 'a' .. 'z',
+);
+
+my $folder = create_index(@docs);
+my $searcher = Lucy::Search::IndexSearcher->new( index => $folder );
+
+my $proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [],
+ within => 10,
+);
+is( $proximity_query->to_string, 'content:""~10',
+ "empty ProximityQuery to_string" );
+$proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [qw( d a )],
+ within => 10,
+);
+is( $proximity_query->to_string, 'content:"d a"~10', "to_string" );
+
+my $hits = $searcher->hits( query => $proximity_query );
+is( $hits->total_hits, 2, "correct number of hits" );
+my $first_hit = $hits->next;
+is( $first_hit->{content}, $best_match, 'best match appears first' );
+
+my $second_hit = $hits->next;
+ok( $first_hit->get_score > $second_hit->get_score,
+ "best match scores higher: "
+ . $first_hit->get_score . " > "
+ . $second_hit->get_score
+);
+
+$proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [qw( c a )],
+ within => 10,
+);
+$hits = $searcher->hits( query => $proximity_query );
+is( $hits->total_hits, 3, 'avoid underflow when subtracting offset' );
+
+$proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [qw( b d )],
+ within => 10,
+);
+$hits = $searcher->hits( query => $proximity_query );
+is( $hits->total_hits, 4, 'offset starting from zero' );
+
+my $frozen = freeze($proximity_query);
+my $thawed = thaw($frozen);
+$hits = $searcher->hits( query => $thawed );
+is( $hits->total_hits, 4, 'freeze/thaw' );
+
+my $proximity_compiler
+ = $proximity_query->make_compiler( searcher => $searcher, );
+$frozen = freeze($proximity_compiler);
+$thawed = thaw($frozen);
+ok( $proximity_compiler->equals($thawed), "freeze/thaw compiler" );
+
+$proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [qw( x d )],
+ within => 4,
+);
+$hits = $searcher->hits( query => $proximity_query );
+is( $hits->total_hits, 2, 'within range is exclusive' );
+$proximity_query = LucyX::Search::ProximityQuery->new(
+ field => 'content',
+ terms => [qw( x d )],
+ within => 3,
+);
+$hits = $searcher->hits( query => $proximity_query );
+is( $hits->total_hits, 1, 'within range is exclusive' );
diff --git a/perl/t/701-uscon.t b/perl/t/701-uscon.t
new file mode 100644
index 0000000..22682fe
--- /dev/null
+++ b/perl/t/701-uscon.t
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 9;
+use Lucy::Test::TestUtils qw( persistent_test_index_loc );
+use Lucy::Test::USConSchema;
+
+my $searcher = Lucy::Search::IndexSearcher->new(
+ index => persistent_test_index_loc() );
+isa_ok( $searcher, 'Lucy::Search::IndexSearcher' );
+
+my %searches = (
+ 'United' => 34,
+ 'shall' => 50,
+ 'not' => 27,
+ '"shall not"' => 21,
+ 'shall not' => 51,
+ 'Congress' => 31,
+ 'Congress AND United' => 22,
+ '(Congress AND United) OR ((Vice AND President) OR "free exercise")' =>
+ 28,
+);
+
+while ( my ( $qstring, $num_expected ) = each %searches ) {
+ my $hits = $searcher->hits( query => $qstring );
+ is( $hits->total_hits, $num_expected, $qstring );
+}
diff --git a/perl/t/999-remove_indexes.t b/perl/t/999-remove_indexes.t
new file mode 100644
index 0000000..880ae41
--- /dev/null
+++ b/perl/t/999-remove_indexes.t
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 1;
+use Lucy::Test::TestUtils qw( remove_working_dir working_dir );
+
+remove_working_dir();
+ok( !-e working_dir(), "working_dir is no more" );
+
diff --git a/perl/t/binding/016-varray.t b/perl/t/binding/016-varray.t
new file mode 100644
index 0000000..ef3fa94
--- /dev/null
+++ b/perl/t/binding/016-varray.t
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 3;
+use Storable qw( nfreeze thaw );
+use Lucy::Test;
+
+my ( $varray, $twin );
+
+$varray = Lucy::Object::VArray->new( capacity => 5 );
+$varray->push( Lucy::Object::CharBuf->new($_) ) for 1 .. 5;
+$varray->delete(3);
+my $frozen = nfreeze($varray);
+my $thawed = thaw($frozen);
+is_deeply( $thawed->to_perl, $varray->to_perl, "freeze/thaw" );
+
+my $ram_file = Lucy::Store::RAMFile->new;
+my $outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or die Lucy->error;
+$varray->serialize($outstream);
+$outstream->close;
+my $instream = Lucy::Store::InStream->open( file => $ram_file )
+ or die Lucy->error;
+my $deserialized = $varray->deserialize($instream);
+is_deeply( $varray->to_perl, $deserialized->to_perl,
+ "serialize/deserialize" );
+
+$twin = $varray->_clone;
+is_deeply( $twin->to_perl, $varray->to_perl, "clone" );
+
diff --git a/perl/t/binding/017-hash.t b/perl/t/binding/017-hash.t
new file mode 100644
index 0000000..bf9655c
--- /dev/null
+++ b/perl/t/binding/017-hash.t
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 4;
+use Storable qw( nfreeze thaw );
+use Lucy::Test;
+use Lucy qw( to_perl to_clownfish );
+
+my $hash = Lucy::Object::Hash->new( capacity => 10 );
+$hash->store( "foo", Lucy::Object::CharBuf->new("bar") );
+$hash->store( "baz", Lucy::Object::CharBuf->new("banana") );
+
+ok( !defined( $hash->fetch("blah") ),
+ "fetch for a non-existent key returns undef" );
+
+my $frozen = nfreeze($hash);
+my $thawed = thaw($frozen);
+is_deeply( $thawed->to_perl, $hash->to_perl, "freeze/thaw" );
+
+my $ram_file = Lucy::Store::RAMFile->new;
+my $outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or die Lucy->error;
+$hash->serialize($outstream);
+$outstream->close;
+my $instream = Lucy::Store::InStream->open( file => $ram_file )
+ or die Lucy->error;
+my $deserialized = $hash->deserialize($instream);
+is_deeply( $hash->to_perl, $deserialized->to_perl, "serialize/deserialize" );
+
+my %hash_with_utf8_keys = ( "\x{263a}" => "foo" );
+my $round_tripped = to_perl( to_clownfish( \%hash_with_utf8_keys ) );
+is_deeply( $round_tripped, \%hash_with_utf8_keys,
+ "Round trip conversion of hash with UTF-8 keys" );
diff --git a/perl/t/binding/019-obj.t b/perl/t/binding/019-obj.t
new file mode 100644
index 0000000..bc11470
--- /dev/null
+++ b/perl/t/binding/019-obj.t
@@ -0,0 +1,130 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 20;
+
+package TestObj;
+use base qw( Lucy::Object::Obj );
+
+our $version = $Lucy::VERSION;
+
+package SonOfTestObj;
+use base qw( TestObj );
+{
+ sub to_string {
+ my $self = shift;
+ return "STRING: " . $self->SUPER::to_string;
+ }
+
+ sub serialize {
+ my ( $self, $outstream ) = @_;
+ $self->SUPER::serialize($outstream);
+ $outstream->write_string("zowie");
+ }
+
+ sub deserialize {
+ my ( $self, $instream ) = @_;
+ $self = $self->SUPER::deserialize($instream);
+ $instream->read_string;
+ return $self;
+ }
+}
+
+package BadSerialize;
+use base qw( Lucy::Object::Obj );
+{
+ sub serialize { }
+}
+
+package BadDump;
+use base qw( Lucy::Object::Obj );
+{
+ sub dump { }
+}
+
+package main;
+use Storable qw( freeze thaw );
+
+ok( defined $TestObj::version,
+ "Using base class should grant access to "
+ . "package globals in the Lucy:: namespace"
+);
+
+# TODO: Port this test to C.
+eval { my $foo = Lucy::Object::Obj->new };
+like( $@, qr/abstract/i, "Obj is an abstract class" );
+
+my $object = TestObj->new;
+isa_ok( $object, "Lucy::Object::Obj",
+ "Clownfish objects can be subclassed outside the Lucy hierarchy" );
+
+# TODO: Port this test to C.
+eval { my $twin = $object->clone };
+like( $@, qr/abstract/i, "clone throws an abstract method exception" );
+
+ok( $object->is_a("Lucy::Object::Obj"), "custom is_a correct" );
+ok( !$object->is_a("Lucy::Object"), "custom is_a too long" );
+ok( !$object->is_a("Lucy"), "custom is_a substring" );
+ok( !$object->is_a(""), "custom is_a blank" );
+ok( !$object->is_a("thing"), "custom is_a wrong" );
+
+eval { my $another_obj = TestObj->new( kill_me_now => 1 ) };
+like( $@, qr/kill_me_now/, "reject bad param" );
+
+my $stringified_perl_obj = "$object";
+require Lucy::Object::Hash;
+my $hash = Lucy::Object::Hash->new;
+$hash->store( foo => $object );
+is( $object->get_refcount, 2, "refcount increased via C code" );
+is( $object->get_refcount, 2, "refcount increased via C code" );
+undef $object;
+$object = $hash->fetch("foo");
+is( "$object", $stringified_perl_obj, "same perl object as before" );
+
+is( $object->get_refcount, 2, "correct refcount after retrieval" );
+undef $hash;
+is( $object->get_refcount, 1, "correct refcount after destruction of ref" );
+
+my $copy = thaw( freeze($object) );
+is( ref($copy), ref($object), "freeze/thaw" );
+
+$object = SonOfTestObj->new;
+like( $object->to_string, qr/STRING:.*?SonOfTestObj/,
+ "overridden XS bindings can be called via SUPER" );
+
+my $frozen = freeze($object);
+my $dupe = thaw($frozen);
+is( ref($dupe), ref($object), "override serialize/deserialize" );
+
+SKIP: {
+ skip( "Invalid serialization causes leaks", 1 ) if $ENV{LUCY_VALGRIND};
+ my $bad = BadSerialize->new;
+ eval { my $froze = freeze($bad); };
+ like( $@, qr/empty/i,
+ "Don't allow subclasses to perform invalid serialization" );
+}
+
+SKIP: {
+ skip( "Exception thrown within callback leaks", 1 )
+ if $ENV{LUCY_VALGRIND};
+ $hash = Lucy::Object::Hash->new;
+ $hash->store( foo => BadDump->new );
+ eval { $hash->dump };
+ like( $@, qr/NULL/,
+ "Don't allow methods without nullable return values to return NULL" );
+}
diff --git a/perl/t/binding/022-bytebuf.t b/perl/t/binding/022-bytebuf.t
new file mode 100644
index 0000000..38182c9
--- /dev/null
+++ b/perl/t/binding/022-bytebuf.t
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 1;
+use Storable qw( freeze thaw );
+use Lucy::Test;
+
+my $orig = Lucy::Object::ByteBuf->new("foo");
+my $frozen = freeze($orig);
+my $thawed = thaw($frozen);
+is( $thawed->to_perl, $orig->to_perl, "freeze/thaw" );
+
diff --git a/perl/t/binding/029-charbuf.t b/perl/t/binding/029-charbuf.t
new file mode 100644
index 0000000..fbb6ae2
--- /dev/null
+++ b/perl/t/binding/029-charbuf.t
@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 6;
+use Storable qw( freeze thaw );
+use Lucy::Test::TestUtils qw( utf8_test_strings );
+
+my ( $smiley, $not_a_smiley, $frowny ) = utf8_test_strings();
+
+my $charbuf = Lucy::Object::CharBuf->new($smiley);
+isa_ok( $charbuf, "Lucy::Object::CharBuf" );
+is( $charbuf->to_perl, $smiley, "round trip UTF-8" );
+
+$charbuf = Lucy::Object::CharBuf->new($smiley);
+my $dupe = thaw( freeze($charbuf) );
+isa_ok( $dupe, "Lucy::Object::CharBuf",
+ "thaw/freeze produces correct object" );
+is( $dupe->to_perl, $charbuf->to_perl, "freeze/thaw" );
+
+my $clone = $charbuf->clone;
+is( $clone->to_perl, Lucy::Object::CharBuf->new($smiley)->to_perl, "clone" );
+
+my $ram_file = Lucy::Store::RAMFile->new;
+my $outstream = Lucy::Store::OutStream->open( file => $ram_file )
+ or die Lucy->error;
+$charbuf->serialize($outstream);
+$outstream->close;
+my $instream = Lucy::Store::InStream->open( file => $ram_file )
+ or die Lucy->error;
+my $deserialized = Lucy::Object::CharBuf->deserialize($instream);
+is_deeply( $charbuf->to_perl, $deserialized->to_perl,
+ "serialize/deserialize" );
+
diff --git a/perl/t/binding/034-err.t b/perl/t/binding/034-err.t
new file mode 100644
index 0000000..fa0fd90
--- /dev/null
+++ b/perl/t/binding/034-err.t
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More tests => 1;
+use Lucy::Test;
+
+my $err = Lucy::Object::Err->new("Bad stuff happened");
+
+isa_ok( $err, 'Lucy::Object::Err', "new" );
+
diff --git a/perl/t/binding/038-lock_free_registry.t b/perl/t/binding/038-lock_free_registry.t
new file mode 100644
index 0000000..e864fe0
--- /dev/null
+++ b/perl/t/binding/038-lock_free_registry.t
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Config;
+use Test::More;
+BEGIN {
+ if ( $ENV{LUCY_VALGRIND} ) {
+ plan( skip_all => 'Known leaks' );
+ }
+ elsif ( !defined( $ENV{LUCY_DEBUG} ) ) {
+ plan( skip_all => 'Debug-only test' );
+ }
+ elsif ( $Config{usethreads} and $^O !~ /mswin/i ) {
+ plan( tests => 1 );
+ }
+ else {
+ plan( skip_all => 'No thread support' );
+ }
+}
+use threads;
+use threads::shared;
+use Time::HiRes qw( time usleep );
+use List::Util qw( shuffle );
+use Lucy::Test;
+
+my $registry = Lucy::Object::LockFreeRegistry->new( capacity => 32 );
+
+sub register_many {
+ my ( $nums, $delay ) = @_;
+
+ # Encourage contention, so that all threads try to register at the same
+ # time.
+ sleep $delay;
+ threads->yield();
+
+ my $succeeded = 0;
+ for my $number (@$nums) {
+ my $obj = Lucy::Object::CharBuf->new($number);
+ $succeeded += $registry->register( key => $obj, value => $obj );
+ }
+
+ return $succeeded;
+}
+
+my @threads;
+
+my $target_time = time() + .5;
+my @num_sets = map { [ shuffle( 1 .. 10000 ) ] } 1 .. 5;
+for my $num ( 1 .. 5 ) {
+ my $delay = $target_time - time();
+ my $thread = threads->create( \®ister_many, pop @num_sets, $delay );
+ push @threads, $thread;
+}
+
+my $total_succeeded = 0;
+$total_succeeded += $_->join for @threads;
+
+is( $total_succeeded, 10000,
+ "registered exactly the right number of entries across all threads" );
+
diff --git a/perl/t/binding/101-simple_io.t b/perl/t/binding/101-simple_io.t
new file mode 100644
index 0000000..3602cee
--- /dev/null
+++ b/perl/t/binding/101-simple_io.t
@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+
+use Test::More tests => 28;
+use Lucy::Test::TestUtils qw( utf8_test_strings );
+use Lucy::Util::StringHelper qw( utf8ify utf8_flag_off );
+use bytes;
+no bytes;
+
+my ( @nums, $packed, $template, $ram_file );
+
+sub check_round_trip {
+ my ( $type, $expected ) = @_;
+ my $write_method = "write_$type";
+ my $read_method = "read_$type";
+ my $file = Lucy::Store::RAMFile->new;
+ my $outstream = Lucy::Store::OutStream->open( file => $file )
+ or die Lucy->error;
+ $outstream->$write_method($_) for @$expected;
+ $outstream->close;
+ my $instream = Lucy::Store::InStream->open( file => $file )
+ or die Lucy->error;
+ my @got;
+ push @got, $instream->$read_method for @$expected;
+ is_deeply( \@got, $expected, $type );
+ return $file;
+}
+
+sub check_round_trip_bytes {
+ my ( $message, $expected ) = @_;
+ my $file = Lucy::Store::RAMFile->new;
+ my $outstream = Lucy::Store::OutStream->open( file => $file );
+ for (@$expected) {
+ $outstream->write_c32( bytes::length($_) );
+ $outstream->print($_);
+ }
+ $outstream->close;
+ my $instream = Lucy::Store::InStream->open( file => $file )
+ or die Lucy->error;
+ my @got;
+ for (@$expected) {
+ my $buf;
+ my $len = $instream->read_c32;
+ $instream->read( $buf, $len );
+ push @got, $buf;
+ }
+ is_deeply( \@got, $expected, $message );
+ return $file;
+}
+
+@nums = ( -128 .. 127 );
+$packed = pack( 'c256', @nums );
+$ram_file = check_round_trip( 'i8', \@nums );
+is( $ram_file->get_contents, $packed,
+ "pack and write_i8 handle signed bytes identically" );
+
+@nums = ( 0 .. 255 );
+$packed = pack( 'C*', @nums );
+$ram_file = check_round_trip( 'u8', \@nums );
+is( $ram_file->get_contents, $packed,
+ "pack and write_u8 handle unsigned bytes identically" );
+
+@nums = map { $_ * 1_000_000 + int( rand() * 1_000_000 ) } -1000 .. 1000;
+push @nums, ( -1 * ( 2**31 ), 2**31 - 1 );
+check_round_trip( 'i32', \@nums );
+
+@nums = map { $_ * 1_000_000 + int( rand() * 1_000_000 ) } 1000 .. 3000;
+push @nums, ( 0, 1, 2**32 - 1 );
+$packed = pack( 'N*', @nums );
+$ram_file = check_round_trip( 'u32', \@nums );
+is( $ram_file->get_contents, $packed,
+ "pack and write_u32 handle unsigned int32s identically" );
+
+@nums = map { $_ * 2 } 0 .. 5;
+check_round_trip( 'u64', \@nums );
+
+@nums = map { $_ * 2**31 } 0 .. 2000;
+$_ += int( rand( 2**16 ) ) for @nums;
+check_round_trip( 'u64', \@nums );
+
+@nums = ( 0 .. 127 );
+check_round_trip( 'c32', \@nums );
+
+@nums = ( 128 .. 500 );
+$packed = pack( 'w*', @nums );
+$ram_file = check_round_trip( 'c32', \@nums );
+is( $ram_file->get_contents, $packed, "C32 is equivalent to Perl's pack w" );
+
+@nums = ( 0 .. 127 );
+check_round_trip( 'c64', \@nums );
+
+@nums = ( 128 .. 500 );
+$packed = pack( 'w*', @nums );
+$ram_file = check_round_trip( 'c64', \@nums );
+is( $ram_file->get_contents, $packed, "C64 is equivalent to Perl's pack w" );
+
+@nums = map { $_ * 2**31 } 0 .. 2000;
+$_ += int( rand( 2**16 ) ) for @nums;
+check_round_trip( 'c64', \@nums );
+
+# rand (always?) has 64-bit precision, but we need 32-bit - so truncate via
+# pack/unpack.
+@nums = map {rand} 0 .. 100;
+$packed = pack( 'f*', @nums );
+@nums = unpack( 'f*', $packed );
+check_round_trip( 'f32', \@nums );
+
+@nums = map {rand} 0 .. 100;
+check_round_trip( 'f64', \@nums );
+
+my @items;
+for ( 0, 22, 300 ) {
+ @items = ( 'a' x $_ );
+ check_round_trip_bytes( "buf of length $_", \@items );
+ check_round_trip( 'string', \@items );
+}
+
+{
+ my @stuff = ( qw( a b c d 1 ), "\n", "\0", " ", " ", "\xf0\x9d\x84\x9e" );
+ my @items = ();
+ for ( 1 .. 50 ) {
+ my $string_len = int( rand() * 5 );
+ my $str = '';
+ $str .= $stuff[ rand @stuff ] for 1 .. $string_len;
+ push @items, $str;
+ }
+ check_round_trip_bytes( "50 binary bufs", \@items );
+}
+
+my ( $smiley, $not_a_smiley, $frowny ) = utf8_test_strings();
+check_round_trip( "string", [ $smiley, $frowny ] );
+
+my $latin = "ma\x{f1}ana";
+$ram_file = check_round_trip( "string", [$latin] );
+my $unibytes = $latin;
+utf8ify($unibytes);
+utf8_flag_off($unibytes);
+my $slurped = $ram_file->get_contents;
+substr( $slurped, 0, 1, "" ); # ditch c32 at head of string;
+is( $slurped, $unibytes, "write_string upgrades to utf8" );
+
diff --git a/perl/t/binding/206-snapshot.t b/perl/t/binding/206-snapshot.t
new file mode 100644
index 0000000..5003ae9
--- /dev/null
+++ b/perl/t/binding/206-snapshot.t
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use Test::More tests => 4;
+use Lucy::Test;
+
+my $folder = Lucy::Store::RAMFolder->new;
+my $snapshot = Lucy::Index::Snapshot->new;
+$snapshot->add_entry("foo");
+$snapshot->add_entry("bar");
+ok( $snapshot->delete_entry("bar"), "delete_entry" );
+is_deeply( $snapshot->list, ['foo'], "add_entry, list" );
+$snapshot->write_file( folder => $folder );
+is( $snapshot->read_file( folder => $folder ),
+ $snapshot, "write_file, read_file" );
+$snapshot->set_path("snapfile");
+is( $snapshot->get_path, "snapfile", "set_path, get_path" );
+
diff --git a/perl/t/binding/506-collector.t b/perl/t/binding/506-collector.t
new file mode 100644
index 0000000..3989605
--- /dev/null
+++ b/perl/t/binding/506-collector.t
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package EvensOnlyCollector;
+use base qw( Lucy::Search::Collector );
+
+our %doc_ids;
+
+sub new {
+ my $self = shift->SUPER::new;
+ $doc_ids{$$self} = [];
+ return $self;
+}
+
+sub collect {
+ my ( $self, $doc_id ) = @_;
+ if ( $doc_id % 2 == 0 ) {
+ push @{ $doc_ids{$$self} }, $doc_id;
+ }
+}
+
+sub get_doc_ids { $doc_ids{ ${ +shift } } }
+
+sub DESTROY {
+ my $self = shift;
+ delete $doc_ids{$$self};
+ $self->SUPER::DESTROY;
+}
+
+package main;
+
+use Test::More tests => 1;
+use Lucy::Test;
+use LucyX::Search::MockMatcher;
+
+my $collector = EvensOnlyCollector->new;
+my $matcher
+ = LucyX::Search::MockMatcher->new( doc_ids => [ 1, 5, 10, 1000 ], );
+$collector->set_matcher($matcher);
+while ( my $doc_id = $matcher->next ) {
+ $collector->collect($doc_id);
+}
+is_deeply(
+ $collector->get_doc_ids,
+ [ 10, 1000 ],
+ "Collector can be subclassed"
+);
+
diff --git a/perl/t/binding/702-sample.t b/perl/t/binding/702-sample.t
new file mode 100644
index 0000000..77b4ba6
--- /dev/null
+++ b/perl/t/binding/702-sample.t
@@ -0,0 +1,59 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+use lib 'buildlib';
+use Test::More tests => 1;
+use File::Spec::Functions qw( catfile catdir );
+use File::Path qw( rmtree );
+
+my $search_cgi_orig_path = catfile(qw( sample search.cgi ));
+my $indexer_pl_orig_path = catfile(qw( sample indexer.pl ));
+
+# Ensure that all @INC dirs make it into the scripts. We can't use PERL5LIB
+# because search.cgi runs with taint mode and environment vars are tainted.
+my $blib_arch = catdir(qw( blib arch ));
+my $blib_lib = catdir(qw( blib lib ));
+my @inc_dirs = map {"use lib '$_';"} ( $blib_arch, $blib_lib, @INC );
+my $use_dirs = join( "\n", @inc_dirs );
+
+for my $filename (qw( search.cgi indexer.pl )) {
+ my $orig_path = catfile( 'sample', $filename );
+ open( my $fh, '<', $orig_path ) or die "Can't open $orig_path: $!";
+ my $content = do { local $/; <$fh> };
+ close $fh or die "Close failed: $!";
+ $content =~ s/(path_to_index\s+=\s+).*?;/$1'_sample_index';/
+ or die "no match";
+ my $uscon_source = catdir(qw( sample us_constitution ));
+ $content =~ s/(uscon_source\s+=\s+).*?;/$1'$uscon_source';/;
+ $content =~ s/^use/$use_dirs;\nuse/m;
+ open( $fh, '>', "_$filename" ) or die $!;
+ print $fh $content;
+ close $fh or die "Close failed: $!";
+}
+
+`$^X _indexer.pl 2>&1`; # Run indexer. Discard output.
+my $html = `$^X -T _search.cgi q=congress`;
+$html =~ s#</?strong>##g; # Delete all strong tags.
+$html =~ s/\s+/ /g; # Collapse all whitespace.
+ok( $html =~ /Results 1-10 of 31/, "indexing and search succeeded" );
+
+END {
+ unlink('_indexer.pl');
+ unlink('_search.cgi');
+ rmtree("_sample_index");
+}
+
diff --git a/perl/t/binding/800-stack.t b/perl/t/binding/800-stack.t
new file mode 100644
index 0000000..3900537
--- /dev/null
+++ b/perl/t/binding/800-stack.t
@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+package MockSearcher;
+use base qw( Lucy::Search::Searcher );
+
+package MyQuery;
+use base qw( Lucy::Search::Query );
+
+sub make_compiler {
+ my $self = shift;
+ return MyCompiler->new( @_, parent => $self );
+}
+
+package MyCompiler;
+use base qw( Lucy::Search::Compiler );
+
+sub apply_norm_factor {
+ my ( $self, $factor ) = @_;
+ $self->SUPER::apply_norm_factor($factor);
+}
+
+package main;
+use Test::More tests => 1;
+
+my $q = Lucy::Search::ORQuery->new;
+for ( 1 .. 50 ) {
+ my @kids = ( $q, ( MyQuery->new ) x 10 );
+ $q = Lucy::Search::ORQuery->new( children => \@kids );
+}
+my $searcher = MockSearcher->new( schema => Lucy::Plan::Schema->new );
+my $compiler = $q->make_compiler( searcher => $searcher );
+
+pass("Made it through deep recursion with multiple stack reallocations");
+
diff --git a/perl/t/binding/801-pod_checker.t b/perl/t/binding/801-pod_checker.t
new file mode 100644
index 0000000..2769e77
--- /dev/null
+++ b/perl/t/binding/801-pod_checker.t
@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Test::More;
+use Pod::Checker;
+BEGIN {
+ if ( $] < 5.010 ) {
+ plan( 'skip_all', "Old Pod::Checker is buggy" );
+ }
+ else {
+ plan('no_plan');
+ }
+}
+
+use File::Find qw( find );
+
+my @filepaths;
+find(
+ { no_chdir => 1,
+ wanted => sub {
+ return unless $File::Find::name =~ /\.(pm|pod)$/;
+ push @filepaths, $File::Find::name;
+ }
+ },
+ 'lib'
+);
+
+for my $path (@filepaths) {
+ my $pod_ok = podchecker( $path, undef, -warnings => 0 );
+ if ( $pod_ok == -1 ) {
+ # No POD.
+ }
+ elsif ( $pod_ok == 0 ) {
+ pass("POD ok for '$path'");
+ }
+ else {
+ fail("Bad POD for '$path'");
+ }
+}
diff --git a/perl/t/charmonizer/001-integers.t b/perl/t/charmonizer/001-integers.t
new file mode 100644
index 0000000..d11bed1
--- /dev/null
+++ b/perl/t/charmonizer/001-integers.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("integers");
+
diff --git a/perl/t/charmonizer/002-func_macro.t b/perl/t/charmonizer/002-func_macro.t
new file mode 100644
index 0000000..e3a3781
--- /dev/null
+++ b/perl/t/charmonizer/002-func_macro.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("func_macro");
+
diff --git a/perl/t/charmonizer/003-headers.t b/perl/t/charmonizer/003-headers.t
new file mode 100644
index 0000000..9f5574a
--- /dev/null
+++ b/perl/t/charmonizer/003-headers.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("headers");
+
diff --git a/perl/t/charmonizer/004-large_files.t b/perl/t/charmonizer/004-large_files.t
new file mode 100644
index 0000000..c4c7307
--- /dev/null
+++ b/perl/t/charmonizer/004-large_files.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("large_files");
+
diff --git a/perl/t/charmonizer/005-unused_vars.t b/perl/t/charmonizer/005-unused_vars.t
new file mode 100644
index 0000000..7cc136d
--- /dev/null
+++ b/perl/t/charmonizer/005-unused_vars.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("unused_vars");
+
diff --git a/perl/t/charmonizer/006-variadic_macros.t b/perl/t/charmonizer/006-variadic_macros.t
new file mode 100644
index 0000000..844dea0
--- /dev/null
+++ b/perl/t/charmonizer/006-variadic_macros.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("variadic_macros");
+
diff --git a/perl/t/charmonizer/007-dirmanip.t b/perl/t/charmonizer/007-dirmanip.t
new file mode 100644
index 0000000..bf5d74c
--- /dev/null
+++ b/perl/t/charmonizer/007-dirmanip.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::TestCharmonizer::run_tests("dirmanip");
+
diff --git a/perl/t/core/012-priority_queue.t b/perl/t/core/012-priority_queue.t
new file mode 100644
index 0000000..ed63c97
--- /dev/null
+++ b/perl/t/core/012-priority_queue.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestPriorityQueue");
+
diff --git a/perl/t/core/013-bit_vector.t b/perl/t/core/013-bit_vector.t
new file mode 100644
index 0000000..5cb9a49
--- /dev/null
+++ b/perl/t/core/013-bit_vector.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestBitVector");
+
diff --git a/perl/t/core/016-varray.t b/perl/t/core/016-varray.t
new file mode 100644
index 0000000..21bb8f8
--- /dev/null
+++ b/perl/t/core/016-varray.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestVArray");
+
diff --git a/perl/t/core/017-hash.t b/perl/t/core/017-hash.t
new file mode 100644
index 0000000..00b04a1
--- /dev/null
+++ b/perl/t/core/017-hash.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestHash");
+
diff --git a/perl/t/core/019-obj.t b/perl/t/core/019-obj.t
new file mode 100644
index 0000000..9e2b15e
--- /dev/null
+++ b/perl/t/core/019-obj.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestObj");
+
diff --git a/perl/t/core/022-bytebuf.t b/perl/t/core/022-bytebuf.t
new file mode 100644
index 0000000..74a0576
--- /dev/null
+++ b/perl/t/core/022-bytebuf.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestByteBuf");
+
diff --git a/perl/t/core/024-memory_pool.t b/perl/t/core/024-memory_pool.t
new file mode 100644
index 0000000..3f72a01
--- /dev/null
+++ b/perl/t/core/024-memory_pool.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestMemoryPool");
+
diff --git a/perl/t/core/029-charbuf.t b/perl/t/core/029-charbuf.t
new file mode 100644
index 0000000..fb508da
--- /dev/null
+++ b/perl/t/core/029-charbuf.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestCharBuf");
+
diff --git a/perl/t/core/030-number_utils.t b/perl/t/core/030-number_utils.t
new file mode 100644
index 0000000..259129c
--- /dev/null
+++ b/perl/t/core/030-number_utils.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestNumberUtils");
+
diff --git a/perl/t/core/031-num.t b/perl/t/core/031-num.t
new file mode 100644
index 0000000..b49a17c
--- /dev/null
+++ b/perl/t/core/031-num.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestNum");
+
diff --git a/perl/t/core/032-string_helper.t b/perl/t/core/032-string_helper.t
new file mode 100644
index 0000000..b340ded
--- /dev/null
+++ b/perl/t/core/032-string_helper.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestStringHelper");
+
diff --git a/perl/t/core/033-index_file_names.t b/perl/t/core/033-index_file_names.t
new file mode 100644
index 0000000..cabd45e
--- /dev/null
+++ b/perl/t/core/033-index_file_names.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestIndexFileNames");
+
diff --git a/perl/t/core/035-json.t b/perl/t/core/035-json.t
new file mode 100644
index 0000000..39078dc
--- /dev/null
+++ b/perl/t/core/035-json.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestJson");
+
diff --git a/perl/t/core/036-i32_array.t b/perl/t/core/036-i32_array.t
new file mode 100644
index 0000000..8863574
--- /dev/null
+++ b/perl/t/core/036-i32_array.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestI32Array");
+
diff --git a/perl/t/core/037-atomic.t b/perl/t/core/037-atomic.t
new file mode 100644
index 0000000..7418fa7
--- /dev/null
+++ b/perl/t/core/037-atomic.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestAtomic");
+
diff --git a/perl/t/core/038-lock_free_registry.t b/perl/t/core/038-lock_free_registry.t
new file mode 100644
index 0000000..4126fe2
--- /dev/null
+++ b/perl/t/core/038-lock_free_registry.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestLockFreeRegistry");
+
diff --git a/perl/t/core/039-memory.t b/perl/t/core/039-memory.t
new file mode 100644
index 0000000..0f4ce95
--- /dev/null
+++ b/perl/t/core/039-memory.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestMemory");
+
diff --git a/perl/t/core/050-ram_file_handle.t b/perl/t/core/050-ram_file_handle.t
new file mode 100644
index 0000000..1f08ced
--- /dev/null
+++ b/perl/t/core/050-ram_file_handle.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestRAMFileHandle");
+
diff --git a/perl/t/core/051-fs_file_handle.t b/perl/t/core/051-fs_file_handle.t
new file mode 100644
index 0000000..0f4a31b
--- /dev/null
+++ b/perl/t/core/051-fs_file_handle.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFSFileHandle");
+
diff --git a/perl/t/core/052-instream.t b/perl/t/core/052-instream.t
new file mode 100644
index 0000000..7417aae
--- /dev/null
+++ b/perl/t/core/052-instream.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestInStream");
+
diff --git a/perl/t/core/053-file_handle.t b/perl/t/core/053-file_handle.t
new file mode 100644
index 0000000..bc7bb6f
--- /dev/null
+++ b/perl/t/core/053-file_handle.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFileHandle");
+
diff --git a/perl/t/core/054-io_primitives.t b/perl/t/core/054-io_primitives.t
new file mode 100644
index 0000000..4ab011d
--- /dev/null
+++ b/perl/t/core/054-io_primitives.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestIOPrimitives");
+
diff --git a/perl/t/core/055-io_chunks.t b/perl/t/core/055-io_chunks.t
new file mode 100644
index 0000000..f042703
--- /dev/null
+++ b/perl/t/core/055-io_chunks.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestIOChunks");
+
diff --git a/perl/t/core/061-ram_dir_handle.t b/perl/t/core/061-ram_dir_handle.t
new file mode 100644
index 0000000..5d38599
--- /dev/null
+++ b/perl/t/core/061-ram_dir_handle.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestRAMDirHandle");
+
diff --git a/perl/t/core/062-fs_dir_handle.t b/perl/t/core/062-fs_dir_handle.t
new file mode 100644
index 0000000..b1c0f1f
--- /dev/null
+++ b/perl/t/core/062-fs_dir_handle.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFSDirHandle");
+
diff --git a/perl/t/core/103-fs_folder.t b/perl/t/core/103-fs_folder.t
new file mode 100644
index 0000000..b628762
--- /dev/null
+++ b/perl/t/core/103-fs_folder.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFSFolder");
+
diff --git a/perl/t/core/104-ram_folder.t b/perl/t/core/104-ram_folder.t
new file mode 100644
index 0000000..c7007ea
--- /dev/null
+++ b/perl/t/core/104-ram_folder.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestRAMFolder");
+
diff --git a/perl/t/core/105-folder.t b/perl/t/core/105-folder.t
new file mode 100644
index 0000000..96523eb
--- /dev/null
+++ b/perl/t/core/105-folder.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFolder");
+
diff --git a/perl/t/core/111-index_manager.t b/perl/t/core/111-index_manager.t
new file mode 100644
index 0000000..ba31713
--- /dev/null
+++ b/perl/t/core/111-index_manager.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestIndexManager");
+
diff --git a/perl/t/core/112-cf_writer.t b/perl/t/core/112-cf_writer.t
new file mode 100644
index 0000000..aeaeab9
--- /dev/null
+++ b/perl/t/core/112-cf_writer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestCompoundFileWriter");
+
diff --git a/perl/t/core/113-cf_reader.t b/perl/t/core/113-cf_reader.t
new file mode 100644
index 0000000..1dc7a4a
--- /dev/null
+++ b/perl/t/core/113-cf_reader.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestCompoundFileReader");
+
diff --git a/perl/t/core/150-analyzer.t b/perl/t/core/150-analyzer.t
new file mode 100644
index 0000000..c845881
--- /dev/null
+++ b/perl/t/core/150-analyzer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestAnalyzer");
+
diff --git a/perl/t/core/150-polyanalyzer.t b/perl/t/core/150-polyanalyzer.t
new file mode 100644
index 0000000..c4cd02a
--- /dev/null
+++ b/perl/t/core/150-polyanalyzer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestPolyAnalyzer");
+
diff --git a/perl/t/core/153-case_folder.t b/perl/t/core/153-case_folder.t
new file mode 100644
index 0000000..b56433b
--- /dev/null
+++ b/perl/t/core/153-case_folder.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestCaseFolder");
+
diff --git a/perl/t/core/154-regex_tokenizer.t b/perl/t/core/154-regex_tokenizer.t
new file mode 100644
index 0000000..3a315b9
--- /dev/null
+++ b/perl/t/core/154-regex_tokenizer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestRegexTokenizer");
+
diff --git a/perl/t/core/155-snowball_stop_filter.t b/perl/t/core/155-snowball_stop_filter.t
new file mode 100644
index 0000000..84c0b22
--- /dev/null
+++ b/perl/t/core/155-snowball_stop_filter.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSnowballStopFilter");
+
diff --git a/perl/t/core/156-snowball_stemmer.t b/perl/t/core/156-snowball_stemmer.t
new file mode 100644
index 0000000..e0d4f46
--- /dev/null
+++ b/perl/t/core/156-snowball_stemmer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSnowStemmer");
+
diff --git a/perl/t/core/206-snapshot.t b/perl/t/core/206-snapshot.t
new file mode 100644
index 0000000..a6ad117
--- /dev/null
+++ b/perl/t/core/206-snapshot.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSnapshot");
+
diff --git a/perl/t/core/216-schema.t b/perl/t/core/216-schema.t
new file mode 100644
index 0000000..2bbbe40
--- /dev/null
+++ b/perl/t/core/216-schema.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSchema");
+
diff --git a/perl/t/core/220-doc_writer.t b/perl/t/core/220-doc_writer.t
new file mode 100644
index 0000000..c35b84e
--- /dev/null
+++ b/perl/t/core/220-doc_writer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestDocWriter");
+
diff --git a/perl/t/core/221-highlight_writer.t b/perl/t/core/221-highlight_writer.t
new file mode 100644
index 0000000..c17f68b
--- /dev/null
+++ b/perl/t/core/221-highlight_writer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestHighlightWriter");
+
diff --git a/perl/t/core/222-posting_list_writer.t b/perl/t/core/222-posting_list_writer.t
new file mode 100644
index 0000000..e2e9680
--- /dev/null
+++ b/perl/t/core/222-posting_list_writer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestPostingListWriter");
+
diff --git a/perl/t/core/223-seg_writer.t b/perl/t/core/223-seg_writer.t
new file mode 100644
index 0000000..479d718
--- /dev/null
+++ b/perl/t/core/223-seg_writer.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSegWriter");
+
diff --git a/perl/t/core/225-polyreader.t b/perl/t/core/225-polyreader.t
new file mode 100644
index 0000000..e72b24c
--- /dev/null
+++ b/perl/t/core/225-polyreader.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestPolyReader");
+
diff --git a/perl/t/core/230-full_text_type.t b/perl/t/core/230-full_text_type.t
new file mode 100644
index 0000000..02cb10b
--- /dev/null
+++ b/perl/t/core/230-full_text_type.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFullTextType");
+
diff --git a/perl/t/core/231-blob_type.t b/perl/t/core/231-blob_type.t
new file mode 100644
index 0000000..9002a61
--- /dev/null
+++ b/perl/t/core/231-blob_type.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestBlobType");
+
diff --git a/perl/t/core/232-numeric_type.t b/perl/t/core/232-numeric_type.t
new file mode 100644
index 0000000..ee18817
--- /dev/null
+++ b/perl/t/core/232-numeric_type.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestNumericType");
+
diff --git a/perl/t/core/234-field_type.t b/perl/t/core/234-field_type.t
new file mode 100644
index 0000000..36f9f65
--- /dev/null
+++ b/perl/t/core/234-field_type.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestFieldType");
+
diff --git a/perl/t/core/301-segment.t b/perl/t/core/301-segment.t
new file mode 100644
index 0000000..b863179
--- /dev/null
+++ b/perl/t/core/301-segment.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSegment");
+
diff --git a/perl/t/core/501-termquery.t b/perl/t/core/501-termquery.t
new file mode 100644
index 0000000..517fe15
--- /dev/null
+++ b/perl/t/core/501-termquery.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestTermQuery");
+
diff --git a/perl/t/core/502-phrasequery.t b/perl/t/core/502-phrasequery.t
new file mode 100644
index 0000000..2c73c91
--- /dev/null
+++ b/perl/t/core/502-phrasequery.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestPhraseQuery");
+
diff --git a/perl/t/core/515-range_query.t b/perl/t/core/515-range_query.t
new file mode 100644
index 0000000..97196a1
--- /dev/null
+++ b/perl/t/core/515-range_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestRangeQuery");
+
diff --git a/perl/t/core/523-and_query.t b/perl/t/core/523-and_query.t
new file mode 100644
index 0000000..85b90b9
--- /dev/null
+++ b/perl/t/core/523-and_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestANDQuery");
+
diff --git a/perl/t/core/525-match_all_query.t b/perl/t/core/525-match_all_query.t
new file mode 100644
index 0000000..fb7b599
--- /dev/null
+++ b/perl/t/core/525-match_all_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestMatchAllQuery");
+
diff --git a/perl/t/core/526-not_query.t b/perl/t/core/526-not_query.t
new file mode 100644
index 0000000..f570e71
--- /dev/null
+++ b/perl/t/core/526-not_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestNOTQuery");
+
diff --git a/perl/t/core/527-req_opt_query.t b/perl/t/core/527-req_opt_query.t
new file mode 100644
index 0000000..5434671
--- /dev/null
+++ b/perl/t/core/527-req_opt_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestReqOptQuery");
+
diff --git a/perl/t/core/528-leaf_query.t b/perl/t/core/528-leaf_query.t
new file mode 100644
index 0000000..65ac539
--- /dev/null
+++ b/perl/t/core/528-leaf_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestLeafQuery");
+
diff --git a/perl/t/core/529-no_match_query.t b/perl/t/core/529-no_match_query.t
new file mode 100644
index 0000000..653dc2e
--- /dev/null
+++ b/perl/t/core/529-no_match_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestNoMatchQuery");
+
diff --git a/perl/t/core/530-series_matcher.t b/perl/t/core/530-series_matcher.t
new file mode 100644
index 0000000..4ed89b5
--- /dev/null
+++ b/perl/t/core/530-series_matcher.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestSeriesMatcher");
+
diff --git a/perl/t/core/531-or_query.t b/perl/t/core/531-or_query.t
new file mode 100644
index 0000000..404e614
--- /dev/null
+++ b/perl/t/core/531-or_query.t
@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestORQuery");
+
diff --git a/perl/xs/Lucy/Analysis/CaseFolder.c b/perl/xs/Lucy/Analysis/CaseFolder.c
new file mode 100644
index 0000000..a5f566b
--- /dev/null
+++ b/perl/xs/Lucy/Analysis/CaseFolder.c
@@ -0,0 +1,99 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_CASEFOLDER
+#define C_LUCY_BYTEBUF
+#define C_LUCY_TOKEN
+#include "XSBind.h"
+
+#include "Lucy/Analysis/CaseFolder.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Object/ByteBuf.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/StringHelper.h"
+
+static size_t
+S_lc_to_work_buf(lucy_CaseFolder *self, uint8_t *source, size_t len,
+ uint8_t **buf, uint8_t **limit) {
+ lucy_ByteBuf *const work_buf = self->work_buf;
+ uint8_t *dest = *buf;
+ uint8_t *dest_start = dest;
+ uint8_t *const end = source + len;
+ uint8_t utf8_buf[7];
+
+ while (source < end) {
+ STRLEN buf_utf8_len;
+ (void)to_utf8_lower(source, utf8_buf, &buf_utf8_len);
+
+ // Grow if necessary.
+ if (((STRLEN)(*limit - dest)) < buf_utf8_len) {
+ size_t bytes_so_far = dest - dest_start;
+ size_t amount = bytes_so_far + (end - source) + 10;
+ Lucy_BB_Set_Size(work_buf, bytes_so_far);
+ *buf = (uint8_t*)Lucy_BB_Grow(work_buf, amount);
+ dest_start = *buf;
+ dest = dest_start + bytes_so_far;
+ *limit = dest_start + work_buf->cap;
+ }
+ memcpy(dest, utf8_buf, buf_utf8_len);
+
+ source += lucy_StrHelp_UTF8_COUNT[*source];
+ dest += buf_utf8_len;
+ }
+
+ {
+ size_t size = dest - dest_start;
+ Lucy_BB_Set_Size(work_buf, size);
+ return size;
+ }
+}
+
+lucy_Inversion*
+lucy_CaseFolder_transform(lucy_CaseFolder *self, lucy_Inversion *inversion) {
+ lucy_Token *token;
+ uint8_t *buf = (uint8_t*)Lucy_BB_Get_Buf(self->work_buf);
+ uint8_t *limit = buf + Lucy_BB_Get_Capacity(self->work_buf);
+ while (NULL != (token = Lucy_Inversion_Next(inversion))) {
+ size_t size = S_lc_to_work_buf(self, (uint8_t*)token->text,
+ token->len, &buf, &limit);
+ if (size > token->len) {
+ LUCY_FREEMEM(token->text);
+ token->text = (char*)LUCY_MALLOCATE(size + 1);
+ }
+ memcpy(token->text, buf, size);
+ token->text[size] = '\0';
+ token->len = size;
+ }
+ Lucy_Inversion_Reset(inversion);
+ return (lucy_Inversion*)LUCY_INCREF(inversion);
+}
+
+lucy_Inversion*
+lucy_CaseFolder_transform_text(lucy_CaseFolder *self, lucy_CharBuf *text) {
+ lucy_Inversion *retval;
+ lucy_Token *token;
+ uint8_t *buf = (uint8_t*)Lucy_BB_Get_Buf(self->work_buf);
+ uint8_t *limit = buf + Lucy_BB_Get_Capacity(self->work_buf);
+ size_t size = S_lc_to_work_buf(self, Lucy_CB_Get_Ptr8(text),
+ Lucy_CB_Get_Size(text), &buf, &limit);
+ token = lucy_Token_new((char*)buf, size, 0, size, 1.0f, 1);
+ retval = lucy_Inversion_new(token);
+ LUCY_DECREF(token);
+ return retval;
+}
+
+
diff --git a/perl/xs/Lucy/Analysis/RegexTokenizer.c b/perl/xs/Lucy/Analysis/RegexTokenizer.c
new file mode 100644
index 0000000..72b8d3d
--- /dev/null
+++ b/perl/xs/Lucy/Analysis/RegexTokenizer.c
@@ -0,0 +1,180 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_REGEXTOKENIZER
+#define C_LUCY_TOKEN
+#include "XSBind.h"
+
+#include "Lucy/Analysis/RegexTokenizer.h"
+#include "Lucy/Analysis/Token.h"
+#include "Lucy/Analysis/Inversion.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Util/Memory.h"
+#include "Lucy/Util/StringHelper.h"
+
+static void
+S_set_token_re_but_not_pattern(lucy_RegexTokenizer *self, void *token_re);
+
+static void
+S_set_pattern_from_token_re(lucy_RegexTokenizer *self, void *token_re);
+
+lucy_RegexTokenizer*
+lucy_RegexTokenizer_init(lucy_RegexTokenizer *self,
+ const lucy_CharBuf *pattern) {
+ SV *token_re_sv;
+
+ lucy_Analyzer_init((lucy_Analyzer*)self);
+ #define DEFAULT_PATTERN "\\w+(?:['\\x{2019}]\\w+)*"
+ if (pattern) {
+ if (Lucy_CB_Find_Str(pattern, "\\p", 2) != -1
+ || Lucy_CB_Find_Str(pattern, "\\P", 2) != -1
+ ) {
+ LUCY_DECREF(self);
+ THROW(LUCY_ERR, "\\p and \\P constructs forbidden");
+ }
+ self->pattern = Lucy_CB_Clone(pattern);
+ }
+ else {
+ self->pattern = lucy_CB_new_from_trusted_utf8(
+ DEFAULT_PATTERN, sizeof(DEFAULT_PATTERN) - 1);
+ }
+
+ // Acquire a compiled regex engine for matching one token.
+ token_re_sv = (SV*)lucy_Host_callback_host(
+ LUCY_REGEXTOKENIZER, "compile_token_re", 1,
+ CFISH_ARG_STR("pattern", self->pattern));
+ S_set_token_re_but_not_pattern(self, SvRV(token_re_sv));
+ SvREFCNT_dec(token_re_sv);
+
+ return self;
+}
+
+static void
+S_set_token_re_but_not_pattern(lucy_RegexTokenizer *self, void *token_re) {
+#if (PERL_VERSION > 10)
+ REGEXP *rx = SvRX((SV*)token_re);
+#else
+ MAGIC *magic = NULL;
+ if (SvMAGICAL((SV*)token_re)) {
+ magic = mg_find((SV*)token_re, PERL_MAGIC_qr);
+ }
+ if (!magic) {
+ THROW(LUCY_ERR, "token_re is not a qr// entity");
+ }
+ REGEXP *rx = (REGEXP*)magic->mg_obj;
+#endif
+ if (rx == NULL) {
+ THROW(LUCY_ERR, "Failed to extract REGEXP from token_re '%s'",
+ SvPV_nolen((SV*)token_re));
+ }
+ if (self->token_re) { ReREFCNT_dec(((REGEXP*)self->token_re)); }
+ self->token_re = rx;
+ (void)ReREFCNT_inc(((REGEXP*)self->token_re));
+}
+
+static void
+S_set_pattern_from_token_re(lucy_RegexTokenizer *self, void *token_re) {
+ SV *rv = newRV((SV*)token_re);
+ STRLEN len = 0;
+ char *ptr = SvPVutf8((SV*)rv, len);
+ Lucy_CB_Mimic_Str(self->pattern, ptr, len);
+ SvREFCNT_dec(rv);
+}
+
+void
+lucy_RegexTokenizer_set_token_re(lucy_RegexTokenizer *self, void *token_re) {
+ S_set_token_re_but_not_pattern(self, token_re);
+ // Set pattern as a side effect.
+ S_set_pattern_from_token_re(self, token_re);
+}
+
+void
+lucy_RegexTokenizer_destroy(lucy_RegexTokenizer *self) {
+ LUCY_DECREF(self->pattern);
+ ReREFCNT_dec(((REGEXP*)self->token_re));
+ LUCY_SUPER_DESTROY(self, LUCY_REGEXTOKENIZER);
+}
+
+void
+lucy_RegexTokenizer_tokenize_str(lucy_RegexTokenizer *self,
+ const char *string, size_t string_len,
+ lucy_Inversion *inversion) {
+ uint32_t num_code_points = 0;
+ SV *wrapper = sv_newmortal();
+#if (PERL_VERSION > 10)
+ REGEXP *rx = (REGEXP*)self->token_re;
+ regexp *rx_struct = (regexp*)SvANY(rx);
+#else
+ REGEXP *rx = (REGEXP*)self->token_re;
+ regexp *rx_struct = rx;
+#endif
+ char *string_beg = (char*)string;
+ char *string_end = string_beg + string_len;
+ char *string_arg = string_beg;
+
+
+ // Fake up an SV wrapper to feed to the regex engine.
+ sv_upgrade(wrapper, SVt_PV);
+ SvREADONLY_on(wrapper);
+ SvLEN(wrapper) = 0;
+ SvUTF8_on(wrapper);
+
+ // Wrap the string in an SV to please the regex engine.
+ SvPVX(wrapper) = string_beg;
+ SvCUR_set(wrapper, string_len);
+ SvPOK_on(wrapper);
+
+ while (pregexec(rx, string_arg, string_end, string_arg, 1, wrapper, 1)) {
+#if ((PERL_VERSION >= 10) || (PERL_VERSION == 9 && PERL_SUBVERSION >= 5))
+ char *const start_ptr = string_arg + rx_struct->offs[0].start;
+ char *const end_ptr = string_arg + rx_struct->offs[0].end;
+#else
+ char *const start_ptr = string_arg + rx_struct->startp[0];
+ char *const end_ptr = string_arg + rx_struct->endp[0];
+#endif
+ uint32_t start, end;
+
+ // Get start and end offsets in Unicode code points.
+ for (; string_arg < start_ptr; num_code_points++) {
+ string_arg += lucy_StrHelp_UTF8_COUNT[(uint8_t)(*string_arg)];
+ if (string_arg > string_end) {
+ THROW(LUCY_ERR, "scanned past end of '%s'", string_beg);
+ }
+ }
+ start = num_code_points;
+ for (; string_arg < end_ptr; num_code_points++) {
+ string_arg += lucy_StrHelp_UTF8_COUNT[(uint8_t)(*string_arg)];
+ if (string_arg > string_end) {
+ THROW(LUCY_ERR, "scanned past end of '%s'", string_beg);
+ }
+ }
+ end = num_code_points;
+
+ // Add a token to the new inversion.
+ Lucy_Inversion_Append(inversion,
+ lucy_Token_new(
+ start_ptr,
+ (end_ptr - start_ptr),
+ start,
+ end,
+ 1.0f, // boost always 1 for now
+ 1 // position increment
+ )
+ );
+ }
+}
+
+
diff --git a/perl/xs/Lucy/Document/Doc.c b/perl/xs/Lucy/Document/Doc.c
new file mode 100644
index 0000000..d3a0765
--- /dev/null
+++ b/perl/xs/Lucy/Document/Doc.c
@@ -0,0 +1,191 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DOC
+#include "XSBind.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Util/Memory.h"
+
+lucy_Doc*
+lucy_Doc_init(lucy_Doc *self, void *fields, int32_t doc_id) {
+ // Assign.
+ if (fields) {
+ if (SvTYPE((SV*)fields) != SVt_PVHV) { THROW(LUCY_ERR, "Not a hash"); }
+ self->fields = SvREFCNT_inc((SV*)fields);
+ }
+ else {
+ self->fields = newHV();
+ }
+ self->doc_id = doc_id;
+
+ return self;
+}
+
+void
+lucy_Doc_set_fields(lucy_Doc *self, void *fields) {
+ if (self->fields) { SvREFCNT_dec((SV*)self->fields); }
+ self->fields = SvREFCNT_inc((SV*)fields);
+}
+
+uint32_t
+lucy_Doc_get_size(lucy_Doc *self) {
+ return self->fields ? HvKEYS((HV*)self->fields) : 0;
+}
+
+void
+lucy_Doc_store(lucy_Doc *self, const lucy_CharBuf *field, lucy_Obj *value) {
+ char *key = (char*)Lucy_CB_Get_Ptr8(field);
+ size_t key_size = Lucy_CB_Get_Size(field);
+ SV *key_sv = newSVpvn(key, key_size);
+ SV *val_sv = value == NULL
+ ? newSV(0)
+ : Lucy_Obj_Is_A(value, LUCY_CHARBUF)
+ ? XSBind_cb_to_sv((lucy_CharBuf*)value)
+ : (SV*)Lucy_Obj_To_Host(value);
+ SvUTF8_on(key_sv);
+ (void)hv_store_ent((HV*)self->fields, key_sv, val_sv, 0);
+ // TODO: make this a thread-local instead of creating it every time?
+ SvREFCNT_dec(key_sv);
+}
+
+void
+lucy_Doc_serialize(lucy_Doc *self, lucy_OutStream *outstream) {
+ Lucy_OutStream_Write_C32(outstream, self->doc_id);
+ lucy_Host_callback(self, "serialize_fields", 1,
+ CFISH_ARG_OBJ("outstream", outstream));
+}
+
+lucy_Doc*
+lucy_Doc_deserialize(lucy_Doc *self, lucy_InStream *instream) {
+ int32_t doc_id = (int32_t)Lucy_InStream_Read_C32(instream);
+
+ self = self ? self : (lucy_Doc*)Lucy_VTable_Make_Obj(LUCY_DOC);
+ lucy_Doc_init(self, NULL, doc_id);
+ lucy_Host_callback(self, "deserialize_fields", 1,
+ CFISH_ARG_OBJ("instream", instream));
+
+ return self;
+}
+
+lucy_Obj*
+lucy_Doc_extract(lucy_Doc *self, lucy_CharBuf *field,
+ lucy_ViewCharBuf *target) {
+ lucy_Obj *retval = NULL;
+ SV **sv_ptr = hv_fetch((HV*)self->fields, (char*)Lucy_CB_Get_Ptr8(field),
+ Lucy_CB_Get_Size(field), 0);
+
+ if (sv_ptr && XSBind_sv_defined(*sv_ptr)) {
+ SV *const sv = *sv_ptr;
+ if (sv_isobject(sv) && sv_derived_from(sv, "Lucy::Object::Obj")) {
+ IV tmp = SvIV(SvRV(sv));
+ retval = INT2PTR(lucy_Obj*, tmp);
+ }
+ else {
+ STRLEN size;
+ char *ptr = SvPVutf8(sv, size);
+ Lucy_ViewCB_Assign_Str(target, ptr, size);
+ retval = (lucy_Obj*)target;
+ }
+ }
+
+ return retval;
+}
+
+void*
+lucy_Doc_to_host(lucy_Doc *self) {
+ lucy_Doc_to_host_t super_to_host
+ = (lucy_Doc_to_host_t)LUCY_SUPER_METHOD(LUCY_DOC, Doc, To_Host);
+ SV *perl_obj = (SV*)super_to_host(self);
+ XSBind_enable_overload(perl_obj);
+ return perl_obj;
+}
+
+lucy_Hash*
+lucy_Doc_dump(lucy_Doc *self) {
+ lucy_Hash *dump = lucy_Hash_new(0);
+ Lucy_Hash_Store_Str(dump, "_class", 6,
+ (lucy_Obj*)Lucy_CB_Clone(Lucy_Doc_Get_Class_Name(self)));
+ Lucy_Hash_Store_Str(dump, "doc_id", 7,
+ (lucy_Obj*)lucy_CB_newf("%i32", self->doc_id));
+ Lucy_Hash_Store_Str(dump, "fields", 6,
+ XSBind_perl_to_cfish((SV*)self->fields));
+ return dump;
+}
+
+lucy_Doc*
+lucy_Doc_load(lucy_Doc *self, lucy_Obj *dump) {
+ lucy_Hash *source = (lucy_Hash*)CFISH_CERTIFY(dump, LUCY_HASH);
+ lucy_CharBuf *class_name = (lucy_CharBuf*)CFISH_CERTIFY(
+ Lucy_Hash_Fetch_Str(source, "_class", 6),
+ LUCY_CHARBUF);
+ lucy_VTable *vtable = lucy_VTable_singleton(class_name, NULL);
+ lucy_Doc *loaded = (lucy_Doc*)Lucy_VTable_Make_Obj(vtable);
+ lucy_Obj *doc_id = CFISH_CERTIFY(
+ Lucy_Hash_Fetch_Str(source, "doc_id", 7),
+ LUCY_OBJ);
+ lucy_Hash *fields = (lucy_Hash*)CFISH_CERTIFY(
+ Lucy_Hash_Fetch_Str(source, "fields", 6),
+ LUCY_HASH);
+ SV *fields_sv = XSBind_cfish_to_perl((lucy_Obj*)fields);
+ CHY_UNUSED_VAR(self);
+
+ loaded->doc_id = (int32_t)Lucy_Obj_To_I64(doc_id);
+ loaded->fields = SvREFCNT_inc(SvRV(fields_sv));
+ SvREFCNT_dec(fields_sv);
+
+ return loaded;
+}
+
+chy_bool_t
+lucy_Doc_equals(lucy_Doc *self, lucy_Obj *other) {
+ lucy_Doc *twin = (lucy_Doc*)other;
+ HV *my_fields;
+ HV *other_fields;
+ I32 num_fields;
+
+ if (twin == self) { return true; }
+ if (!Lucy_Obj_Is_A(other, LUCY_DOC)) { return false; }
+ if (!self->doc_id == twin->doc_id) { return false; }
+ if (!!self->fields ^ !!twin->fields) { return false; }
+
+ // Verify fields. Don't allow any deep data structures.
+ my_fields = (HV*)self->fields;
+ other_fields = (HV*)twin->fields;
+ if (HvKEYS(my_fields) != HvKEYS(other_fields)) { return false; }
+ num_fields = hv_iterinit(my_fields);
+ while (num_fields--) {
+ HE *my_entry = hv_iternext(my_fields);
+ SV *my_val_sv = HeVAL(my_entry);
+ STRLEN key_len = HeKLEN(my_entry);
+ char *key = HeKEY(my_entry);
+ SV **const other_val = hv_fetch(other_fields, key, key_len, 0);
+ if (!other_val) { return false; }
+ if (!sv_eq(my_val_sv, *other_val)) { return false; }
+ }
+
+ return true;
+}
+
+void
+lucy_Doc_destroy(lucy_Doc *self) {
+ if (self->fields) { SvREFCNT_dec((SV*)self->fields); }
+ LUCY_SUPER_DESTROY(self, LUCY_DOC);
+}
+
+
diff --git a/perl/xs/Lucy/Index/DocReader.c b/perl/xs/Lucy/Index/DocReader.c
new file mode 100644
index 0000000..f59e401
--- /dev/null
+++ b/perl/xs/Lucy/Index/DocReader.c
@@ -0,0 +1,127 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_DOCREADER
+#define C_LUCY_DEFAULTDOCREADER
+#define C_LUCY_ZOMBIECHARBUF
+#include "XSBind.h"
+
+#include "Lucy/Index/DocReader.h"
+#include "Lucy/Document/HitDoc.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/BlobType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Plan/TextType.h"
+#include "Lucy/Plan/NumericType.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Store/InStream.h"
+
+lucy_HitDoc*
+lucy_DefDocReader_fetch_doc(lucy_DefaultDocReader *self, int32_t doc_id) {
+ lucy_Schema *const schema = self->schema;
+ lucy_InStream *const dat_in = self->dat_in;
+ lucy_InStream *const ix_in = self->ix_in;
+ HV *fields = newHV();
+ int64_t start;
+ uint32_t num_fields;
+ SV *field_name_sv = newSV(1);
+
+ // Get data file pointer from index, read number of fields.
+ Lucy_InStream_Seek(ix_in, (int64_t)doc_id * 8);
+ start = Lucy_InStream_Read_U64(ix_in);
+ Lucy_InStream_Seek(dat_in, start);
+ num_fields = Lucy_InStream_Read_C32(dat_in);
+
+ // Decode stored data and build up the doc field by field.
+ while (num_fields--) {
+ STRLEN field_name_len;
+ char *field_name_ptr;
+ SV *value_sv;
+ lucy_FieldType *type;
+
+ // Read field name.
+ field_name_len = Lucy_InStream_Read_C32(dat_in);
+ field_name_ptr = SvGROW(field_name_sv, field_name_len + 1);
+ Lucy_InStream_Read_Bytes(dat_in, field_name_ptr, field_name_len);
+ SvPOK_on(field_name_sv);
+ SvCUR_set(field_name_sv, field_name_len);
+ SvUTF8_on(field_name_sv);
+ *SvEND(field_name_sv) = '\0';
+
+ // Find the Field's FieldType.
+ lucy_ZombieCharBuf *field_name_zcb
+ = CFISH_ZCB_WRAP_STR(field_name_ptr, field_name_len);
+ Lucy_ZCB_Assign_Str(field_name_zcb, field_name_ptr, field_name_len);
+ type = Lucy_Schema_Fetch_Type(schema, (lucy_CharBuf*)field_name_zcb);
+
+ // Read the field value.
+ switch (Lucy_FType_Primitive_ID(type) & lucy_FType_PRIMITIVE_ID_MASK) {
+ case lucy_FType_TEXT: {
+ STRLEN value_len = Lucy_InStream_Read_C32(dat_in);
+ value_sv = newSV((value_len ? value_len : 1));
+ Lucy_InStream_Read_Bytes(dat_in, SvPVX(value_sv), value_len);
+ SvCUR_set(value_sv, value_len);
+ *SvEND(value_sv) = '\0';
+ SvPOK_on(value_sv);
+ SvUTF8_on(value_sv);
+ break;
+ }
+ case lucy_FType_BLOB: {
+ STRLEN value_len = Lucy_InStream_Read_C32(dat_in);
+ value_sv = newSV((value_len ? value_len : 1));
+ Lucy_InStream_Read_Bytes(dat_in, SvPVX(value_sv), value_len);
+ SvCUR_set(value_sv, value_len);
+ *SvEND(value_sv) = '\0';
+ SvPOK_on(value_sv);
+ break;
+ }
+ case lucy_FType_FLOAT32:
+ value_sv = newSVnv(Lucy_InStream_Read_F32(dat_in));
+ break;
+ case lucy_FType_FLOAT64:
+ value_sv = newSVnv(Lucy_InStream_Read_F64(dat_in));
+ break;
+ case lucy_FType_INT32:
+ value_sv = newSViv((int32_t)Lucy_InStream_Read_C32(dat_in));
+ break;
+ case lucy_FType_INT64:
+ if (sizeof(IV) == 8) {
+ int64_t val = (int64_t)Lucy_InStream_Read_C64(dat_in);
+ value_sv = newSViv((IV)val);
+ }
+ else { // (lossy)
+ int64_t val = (int64_t)Lucy_InStream_Read_C64(dat_in);
+ value_sv = newSVnv((double)val);
+ }
+ break;
+ default:
+ value_sv = NULL;
+ CFISH_THROW(LUCY_ERR, "Unrecognized type: %o", type);
+ }
+
+ // Store the value.
+ (void)hv_store_ent(fields, field_name_sv, value_sv, 0);
+ }
+ SvREFCNT_dec(field_name_sv);
+
+ {
+ lucy_HitDoc *retval = lucy_HitDoc_new(fields, doc_id, 0.0);
+ SvREFCNT_dec((SV*)fields);
+ return retval;
+ }
+}
+
+
diff --git a/perl/xs/Lucy/Index/Inverter.c b/perl/xs/Lucy/Index/Inverter.c
new file mode 100644
index 0000000..1f532c5
--- /dev/null
+++ b/perl/xs/Lucy/Index/Inverter.c
@@ -0,0 +1,147 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_INVERTER
+#define C_LUCY_ZOMBIECHARBUF
+#define C_LUCY_INVERTERENTRY
+#include "XSBind.h"
+#include "Lucy/Index/Inverter.h"
+#include "Lucy/Document/Doc.h"
+#include "Lucy/Index/Segment.h"
+#include "Lucy/Object/ByteBuf.h"
+#include "Lucy/Plan/FieldType.h"
+#include "Lucy/Plan/BlobType.h"
+#include "Lucy/Plan/NumericType.h"
+#include "Lucy/Plan/Schema.h"
+#include "Lucy/Plan/TextType.h"
+#include "Lucy/Util/StringHelper.h"
+
+static lucy_InverterEntry*
+S_fetch_entry(lucy_Inverter *self, HE *hash_entry) {
+ lucy_Schema *const schema = self->schema;
+ char *key;
+ STRLEN key_len;
+ STRLEN he_key_len = HeKLEN(hash_entry);
+
+ // Force field name to UTF-8 if necessary.
+ if (he_key_len == (STRLEN)HEf_SVKEY) {
+ SV *key_sv = HeKEY_sv(hash_entry);
+ key = SvPVutf8(key_sv, key_len);
+ }
+ else {
+ key = HeKEY(hash_entry);
+ key_len = he_key_len;
+ if (!lucy_StrHelp_utf8_valid(key, key_len)) {
+ SV *key_sv = HeSVKEY_force(hash_entry);
+ key = SvPVutf8(key_sv, key_len);
+ }
+ }
+
+ lucy_ZombieCharBuf *field = CFISH_ZCB_WRAP_STR(key, key_len);
+ int32_t field_num
+ = Lucy_Seg_Field_Num(self->segment, (lucy_CharBuf*)field);
+ if (!field_num) {
+ // This field seems not to be in the segment yet. Try to find it in
+ // the Schema.
+ if (Lucy_Schema_Fetch_Type(schema, (lucy_CharBuf*)field)) {
+ // The field is in the Schema. Get a field num from the Segment.
+ field_num = Lucy_Seg_Add_Field(self->segment,
+ (lucy_CharBuf*)field);
+ }
+ else {
+ // We've truly failed to find the field. The user must
+ // not have spec'd it.
+ THROW(LUCY_ERR, "Unknown field name: '%s'", key);
+ }
+ }
+
+ {
+ lucy_InverterEntry *entry
+ = (lucy_InverterEntry*)Lucy_VA_Fetch(self->entry_pool, field_num);
+ if (!entry) {
+ entry = lucy_InvEntry_new(schema, (lucy_CharBuf*)field,
+ field_num);
+ Lucy_VA_Store(self->entry_pool, field_num, (lucy_Obj*)entry);
+ }
+ return entry;
+ }
+}
+
+void
+lucy_Inverter_invert_doc(lucy_Inverter *self, lucy_Doc *doc) {
+ HV *const fields = (HV*)Lucy_Doc_Get_Fields(doc);
+ I32 num_keys = hv_iterinit(fields);
+
+ // Prepare for the new doc.
+ Lucy_Inverter_Set_Doc(self, doc);
+
+ // Extract and invert the doc's fields.
+ while (num_keys--) {
+ HE *hash_entry = hv_iternext(fields);
+ lucy_InverterEntry *inv_entry = S_fetch_entry(self, hash_entry);
+ SV *value_sv = HeVAL(hash_entry);
+ lucy_FieldType *type = inv_entry->type;
+
+ // Get the field value, forcing text fields to UTF-8.
+ switch (Lucy_FType_Primitive_ID(type) & lucy_FType_PRIMITIVE_ID_MASK) {
+ case lucy_FType_TEXT: {
+ STRLEN val_len;
+ char *val_ptr = SvPVutf8(value_sv, val_len);
+ lucy_ViewCharBuf *value
+ = (lucy_ViewCharBuf*)inv_entry->value;
+ Lucy_ViewCB_Assign_Str(value, val_ptr, val_len);
+ break;
+ }
+ case lucy_FType_BLOB: {
+ STRLEN val_len;
+ char *val_ptr = SvPV(value_sv, val_len);
+ lucy_ViewByteBuf *value
+ = (lucy_ViewByteBuf*)inv_entry->value;
+ Lucy_ViewBB_Assign_Bytes(value, val_ptr, val_len);
+ break;
+ }
+ case lucy_FType_INT32: {
+ lucy_Integer32* value = (lucy_Integer32*)inv_entry->value;
+ Lucy_Int32_Set_Value(value, SvIV(value_sv));
+ break;
+ }
+ case lucy_FType_INT64: {
+ lucy_Integer64* value = (lucy_Integer64*)inv_entry->value;
+ int64_t val = sizeof(IV) == 8
+ ? SvIV(value_sv)
+ : (int64_t)SvNV(value_sv); // lossy
+ Lucy_Int64_Set_Value(value, val);
+ break;
+ }
+ case lucy_FType_FLOAT32: {
+ lucy_Float32* value = (lucy_Float32*)inv_entry->value;
+ Lucy_Float32_Set_Value(value, (float)SvNV(value_sv));
+ break;
+ }
+ case lucy_FType_FLOAT64: {
+ lucy_Float64* value = (lucy_Float64*)inv_entry->value;
+ Lucy_Float64_Set_Value(value, SvNV(value_sv));
+ break;
+ }
+ default:
+ THROW(LUCY_ERR, "Unrecognized type: %o", type);
+ }
+
+ Lucy_Inverter_Add_Field(self, inv_entry);
+ }
+}
+
+
diff --git a/perl/xs/Lucy/Index/PolyReader.c b/perl/xs/Lucy/Index/PolyReader.c
new file mode 100644
index 0000000..14d6f87
--- /dev/null
+++ b/perl/xs/Lucy/Index/PolyReader.c
@@ -0,0 +1,39 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/PolyReader.h"
+#include "Lucy/Index/Snapshot.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Store/Folder.h"
+
+Obj*
+PolyReader_try_open_segreaders(PolyReader *self, VArray *segments) {
+ return Host_callback_obj(self, "try_open_segreaders", 1,
+ ARG_OBJ("segments", segments));
+}
+
+CharBuf*
+PolyReader_try_read_snapshot(Snapshot *snapshot, Folder *folder,
+ const CharBuf *path) {
+ return (CharBuf*)Host_callback_obj(POLYREADER, "try_read_snapshot", 3,
+ ARG_OBJ("snapshot", snapshot),
+ ARG_OBJ("folder", folder),
+ ARG_STR("path", path));
+}
+
+
diff --git a/perl/xs/Lucy/Index/SegReader.c b/perl/xs/Lucy/Index/SegReader.c
new file mode 100644
index 0000000..f0823df
--- /dev/null
+++ b/perl/xs/Lucy/Index/SegReader.c
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Index/SegReader.h"
+#include "Lucy/Object/Host.h"
+
+CharBuf*
+SegReader_try_init_components(SegReader *self) {
+ return (CharBuf*)Host_callback_obj(self, "try_init_components", 0);
+}
+
+
diff --git a/perl/xs/Lucy/Object/Err.c b/perl/xs/Lucy/Object/Err.c
new file mode 100644
index 0000000..cb7cb0c
--- /dev/null
+++ b/perl/xs/Lucy/Object/Err.c
@@ -0,0 +1,77 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "XSBind.h"
+#include "Lucy/Object/Host.h"
+
+lucy_Err*
+lucy_Err_get_error() {
+ lucy_Err *error
+ = (lucy_Err*)lucy_Host_callback_obj(LUCY_ERR, "get_error", 0);
+ LUCY_DECREF(error); // Cancel out incref from callback.
+ return error;
+}
+
+void
+lucy_Err_set_error(lucy_Err *error) {
+ lucy_Host_callback(LUCY_ERR, "set_error", 1,
+ CFISH_ARG_OBJ("error", error));
+ LUCY_DECREF(error);
+}
+
+void
+lucy_Err_do_throw(lucy_Err *err) {
+ dSP;
+ SV *error_sv = (SV*)Lucy_Err_To_Host(err);
+ LUCY_DECREF(err);
+ ENTER;
+ SAVETMPS;
+ PUSHMARK(SP);
+ XPUSHs(sv_2mortal(error_sv));
+ PUTBACK;
+ call_pv("Lucy::Object::Err::do_throw", G_DISCARD);
+ FREETMPS;
+ LEAVE;
+}
+
+void*
+lucy_Err_to_host(lucy_Err *self) {
+ lucy_Err_to_host_t super_to_host
+ = (lucy_Err_to_host_t)LUCY_SUPER_METHOD(LUCY_ERR, Err, To_Host);
+ SV *perl_obj = (SV*)super_to_host(self);
+ XSBind_enable_overload(perl_obj);
+ return perl_obj;
+}
+
+void
+lucy_Err_throw_mess(lucy_VTable *vtable, lucy_CharBuf *message) {
+ lucy_Err_make_t make = (lucy_Err_make_t)LUCY_METHOD(
+ CFISH_CERTIFY(vtable, LUCY_VTABLE), Err, Make);
+ lucy_Err *err = (lucy_Err*)CFISH_CERTIFY(make(NULL), LUCY_ERR);
+ Lucy_Err_Cat_Mess(err, message);
+ LUCY_DECREF(message);
+ lucy_Err_do_throw(err);
+}
+
+void
+lucy_Err_warn_mess(lucy_CharBuf *message) {
+ SV *error_sv = XSBind_cb_to_sv(message);
+ LUCY_DECREF(message);
+ warn("%s", SvPV_nolen(error_sv));
+ SvREFCNT_dec(error_sv);
+}
+
+
diff --git a/perl/xs/Lucy/Object/Host.c b/perl/xs/Lucy/Object/Host.c
new file mode 100644
index 0000000..0503697
--- /dev/null
+++ b/perl/xs/Lucy/Object/Host.c
@@ -0,0 +1,249 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "XSBind.h"
+
+#include "Lucy/Object/VTable.h"
+
+#include "Lucy/Object/Obj.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Util/Memory.h"
+
+static SV*
+S_do_callback_sv(void *vobj, char *method, uint32_t num_args, va_list args);
+
+// Convert all arguments to Perl and place them on the Perl stack.
+static CHY_INLINE void
+SI_push_args(void *vobj, va_list args, uint32_t num_args) {
+ lucy_Obj *obj = (lucy_Obj*)vobj;
+ SV *invoker;
+ uint32_t i;
+ dSP;
+
+ uint32_t stack_slots_needed = num_args < 2
+ ? num_args + 1
+ : (num_args * 2) + 1;
+ EXTEND(SP, stack_slots_needed);
+
+ if (Lucy_Obj_Is_A(obj, LUCY_VTABLE)) {
+ lucy_VTable *vtable = (lucy_VTable*)obj;
+ // TODO: Creating a new class name SV every time is wasteful.
+ invoker = XSBind_cb_to_sv(Lucy_VTable_Get_Name(vtable));
+ }
+ else {
+ invoker = (SV*)Lucy_Obj_To_Host(obj);
+ }
+
+ ENTER;
+ SAVETMPS;
+ PUSHMARK(SP);
+ PUSHs(sv_2mortal(invoker));
+
+ for (i = 0; i < num_args; i++) {
+ uint32_t arg_type = va_arg(args, uint32_t);
+ char *label = va_arg(args, char*);
+ if (num_args > 1) {
+ PUSHs(sv_2mortal(newSVpvn(label, strlen(label))));
+ }
+ switch (arg_type & CFISH_HOST_ARGTYPE_MASK) {
+ case CFISH_HOST_ARGTYPE_I32: {
+ int32_t value = va_arg(args, int32_t);
+ PUSHs(sv_2mortal(newSViv(value)));
+ }
+ break;
+ case CFISH_HOST_ARGTYPE_I64: {
+ int64_t value = va_arg(args, int64_t);
+ if (sizeof(IV) == 8) {
+ PUSHs(sv_2mortal(newSViv((IV)value)));
+ }
+ else {
+ // lossy
+ PUSHs(sv_2mortal(newSVnv((double)value)));
+ }
+ }
+ break;
+ case CFISH_HOST_ARGTYPE_F32:
+ case CFISH_HOST_ARGTYPE_F64: {
+ // Floats are promoted to doubles by variadic calling.
+ double value = va_arg(args, double);
+ PUSHs(sv_2mortal(newSVnv(value)));
+ }
+ break;
+ case CFISH_HOST_ARGTYPE_STR: {
+ lucy_CharBuf *string = va_arg(args, lucy_CharBuf*);
+ PUSHs(sv_2mortal(XSBind_cb_to_sv(string)));
+ }
+ break;
+ case CFISH_HOST_ARGTYPE_OBJ: {
+ lucy_Obj* anObj = va_arg(args, lucy_Obj*);
+ SV *arg_sv = anObj == NULL
+ ? newSV(0)
+ : XSBind_cfish_to_perl(anObj);
+ PUSHs(sv_2mortal(arg_sv));
+ }
+ break;
+ default:
+ CFISH_THROW(LUCY_ERR, "Unrecognized arg type: %u32",
+ arg_type);
+ }
+ }
+
+ PUTBACK;
+}
+
+void
+lucy_Host_callback(void *vobj, char *method, uint32_t num_args, ...) {
+ va_list args;
+
+ va_start(args, num_args);
+ SI_push_args(vobj, args, num_args);
+ va_end(args);
+
+ {
+ int count = call_method(method, G_VOID | G_DISCARD);
+ if (count != 0) {
+ CFISH_THROW(LUCY_ERR, "callback '%s' returned too many values: %i32",
+ method, (int32_t)count);
+ }
+ FREETMPS;
+ LEAVE;
+ }
+}
+
+int64_t
+lucy_Host_callback_i64(void *vobj, char *method, uint32_t num_args, ...) {
+ va_list args;
+ SV *return_sv;
+ int64_t retval;
+
+ va_start(args, num_args);
+ return_sv = S_do_callback_sv(vobj, method, num_args, args);
+ va_end(args);
+ if (sizeof(IV) == 8) {
+ retval = (int64_t)SvIV(return_sv);
+ }
+ else {
+ if (SvIOK(return_sv)) {
+ // It's already no more than 32 bits, so don't convert.
+ retval = SvIV(return_sv);
+ }
+ else {
+ // Maybe lossy.
+ double temp = SvNV(return_sv);
+ retval = (int64_t)temp;
+ }
+ }
+
+ FREETMPS;
+ LEAVE;
+
+ return retval;
+}
+
+double
+lucy_Host_callback_f64(void *vobj, char *method, uint32_t num_args, ...) {
+ va_list args;
+ SV *return_sv;
+ double retval;
+
+ va_start(args, num_args);
+ return_sv = S_do_callback_sv(vobj, method, num_args, args);
+ va_end(args);
+ retval = SvNV(return_sv);
+
+ FREETMPS;
+ LEAVE;
+
+ return retval;
+}
+
+lucy_Obj*
+lucy_Host_callback_obj(void *vobj, char *method, uint32_t num_args, ...) {
+ va_list args;
+ SV *temp_retval;
+ lucy_Obj *retval = NULL;
+
+ va_start(args, num_args);
+ temp_retval = S_do_callback_sv(vobj, method, num_args, args);
+ va_end(args);
+
+ retval = XSBind_perl_to_cfish(temp_retval);
+
+ FREETMPS;
+ LEAVE;
+
+ return retval;
+}
+
+lucy_CharBuf*
+lucy_Host_callback_str(void *vobj, char *method, uint32_t num_args, ...) {
+ va_list args;
+ SV *temp_retval;
+ lucy_CharBuf *retval = NULL;
+
+ va_start(args, num_args);
+ temp_retval = S_do_callback_sv(vobj, method, num_args, args);
+ va_end(args);
+
+ // Make a stringified copy.
+ if (temp_retval && XSBind_sv_defined(temp_retval)) {
+ STRLEN len;
+ char *ptr = SvPVutf8(temp_retval, len);
+ retval = lucy_CB_new_from_trusted_utf8(ptr, len);
+ }
+
+ FREETMPS;
+ LEAVE;
+
+ return retval;
+}
+
+void*
+lucy_Host_callback_host(void *vobj, char *method, uint32_t num_args, ...) {
+ va_list args;
+ SV *retval;
+
+ va_start(args, num_args);
+ retval = S_do_callback_sv(vobj, method, num_args, args);
+ va_end(args);
+ SvREFCNT_inc(retval);
+
+ FREETMPS;
+ LEAVE;
+
+ return retval;
+}
+
+static SV*
+S_do_callback_sv(void *vobj, char *method, uint32_t num_args, va_list args) {
+ SV *return_val;
+ SI_push_args(vobj, args, num_args);
+ {
+ int num_returned = call_method(method, G_SCALAR);
+ dSP;
+ if (num_returned != 1) {
+ CFISH_THROW(LUCY_ERR, "Bad number of return vals from %s: %i32",
+ method, (int32_t)num_returned);
+ }
+ return_val = POPs;
+ PUTBACK;
+ }
+ return return_val;
+}
+
+
diff --git a/perl/xs/Lucy/Object/LockFreeRegistry.c b/perl/xs/Lucy/Object/LockFreeRegistry.c
new file mode 100644
index 0000000..68bc113
--- /dev/null
+++ b/perl/xs/Lucy/Object/LockFreeRegistry.c
@@ -0,0 +1,36 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_OBJ
+#define C_LUCY_LOCKFREEREGISTRY
+#include "XSBind.h"
+
+#include "Lucy/Object/LockFreeRegistry.h"
+#include "Lucy/Object/Host.h"
+
+void*
+lucy_LFReg_to_host(lucy_LockFreeRegistry *self) {
+ chy_bool_t first_time = self->ref.count < 4 ? true : false;
+ lucy_LFReg_to_host_t to_host = (lucy_LFReg_to_host_t)LUCY_SUPER_METHOD(
+ LUCY_LOCKFREEREGISTRY, LFReg, To_Host);
+ SV *host_obj = (SV*)to_host(self);
+ if (first_time) {
+ SvSHARE((SV*)self->ref.host_obj);
+ }
+ return host_obj;
+}
+
+
diff --git a/perl/xs/Lucy/Object/Obj.c b/perl/xs/Lucy/Object/Obj.c
new file mode 100644
index 0000000..bb5e92b
--- /dev/null
+++ b/perl/xs/Lucy/Object/Obj.c
@@ -0,0 +1,113 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_OBJ
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+#include "ppport.h"
+
+#include "Lucy/Object/Obj.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Util/Memory.h"
+
+static void
+S_lazy_init_host_obj(lucy_Obj *self) {
+ SV *inner_obj = newSV(0);
+ SvOBJECT_on(inner_obj);
+ PL_sv_objcount++;
+ SvUPGRADE(inner_obj, SVt_PVMG);
+ sv_setiv(inner_obj, PTR2IV(self));
+
+ // Connect class association.
+ lucy_CharBuf *class_name = Lucy_VTable_Get_Name(self->vtable);
+ HV *stash = gv_stashpvn((char*)Lucy_CB_Get_Ptr8(class_name),
+ Lucy_CB_Get_Size(class_name), TRUE);
+ SvSTASH_set(inner_obj, (HV*)SvREFCNT_inc(stash));
+
+ /* Up till now we've been keeping track of the refcount in
+ * self->ref.count. We're replacing ref.count with ref.host_obj, which
+ * will assume responsibility for maintaining the refcount. ref.host_obj
+ * starts off with a refcount of 1, so we need to transfer any refcounts
+ * in excess of that. */
+ size_t old_refcount = self->ref.count;
+ self->ref.host_obj = inner_obj;
+ while (old_refcount > 1) {
+ SvREFCNT_inc_simple_void_NN(inner_obj);
+ old_refcount--;
+ }
+}
+
+uint32_t
+lucy_Obj_get_refcount(lucy_Obj *self) {
+ return self->ref.count < 4
+ ? self->ref.count
+ : SvREFCNT((SV*)self->ref.host_obj);
+}
+
+lucy_Obj*
+lucy_Obj_inc_refcount(lucy_Obj *self) {
+ switch (self->ref.count) {
+ case 0:
+ CFISH_THROW(LUCY_ERR, "Illegal refcount of 0");
+ break; // useless
+ case 1:
+ case 2:
+ self->ref.count++;
+ break;
+ case 3:
+ S_lazy_init_host_obj(self);
+ // fall through
+ default:
+ SvREFCNT_inc_simple_void_NN((SV*)self->ref.host_obj);
+ }
+ return self;
+}
+
+uint32_t
+lucy_Obj_dec_refcount(lucy_Obj *self) {
+ uint32_t modified_refcount = I32_MAX;
+ switch (self->ref.count) {
+ case 0:
+ CFISH_THROW(LUCY_ERR, "Illegal refcount of 0");
+ break; // useless
+ case 1:
+ modified_refcount = 0;
+ Lucy_Obj_Destroy(self);
+ break;
+ case 2:
+ case 3:
+ modified_refcount = --self->ref.count;
+ break;
+ default:
+ modified_refcount = SvREFCNT((SV*)self->ref.host_obj) - 1;
+ // If the SV's refcount falls to 0, DESTROY will be invoked from
+ // Perl-space.
+ SvREFCNT_dec((SV*)self->ref.host_obj);
+ }
+ return modified_refcount;
+}
+
+void*
+lucy_Obj_to_host(lucy_Obj *self) {
+ if (self->ref.count < 4) { S_lazy_init_host_obj(self); }
+ return newRV_inc((SV*)self->ref.host_obj);
+}
+
+
diff --git a/perl/xs/Lucy/Object/VTable.c b/perl/xs/Lucy/Object/VTable.c
new file mode 100644
index 0000000..a347709
--- /dev/null
+++ b/perl/xs/Lucy/Object/VTable.c
@@ -0,0 +1,70 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_OBJ
+#define C_LUCY_VTABLE
+#include "XSBind.h"
+
+#include "Lucy/Object/VTable.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Util/Memory.h"
+
+lucy_Obj*
+lucy_VTable_foster_obj(lucy_VTable *self, void *host_obj) {
+ lucy_Obj *obj
+ = (lucy_Obj*)lucy_Memory_wrapped_calloc(self->obj_alloc_size, 1);
+ SV *inner_obj = SvRV((SV*)host_obj);
+ obj->vtable = self;
+ sv_setiv(inner_obj, PTR2IV(obj));
+ obj->ref.host_obj = inner_obj;
+ return obj;
+}
+
+void
+lucy_VTable_register_with_host(lucy_VTable *singleton, lucy_VTable *parent) {
+ // Register class with host.
+ lucy_Host_callback(LUCY_VTABLE, "_register", 2,
+ CFISH_ARG_OBJ("singleton", singleton),
+ CFISH_ARG_OBJ("parent", parent));
+}
+
+lucy_VArray*
+lucy_VTable_novel_host_methods(const lucy_CharBuf *class_name) {
+ return (lucy_VArray*)lucy_Host_callback_obj(
+ LUCY_VTABLE,
+ "novel_host_methods", 1,
+ CFISH_ARG_STR("class_name", class_name));
+}
+
+lucy_CharBuf*
+lucy_VTable_find_parent_class(const lucy_CharBuf *class_name) {
+ return lucy_Host_callback_str(LUCY_VTABLE, "find_parent_class", 1,
+ CFISH_ARG_STR("class_name", class_name));
+}
+
+void*
+lucy_VTable_to_host(lucy_VTable *self) {
+ chy_bool_t first_time = self->ref.count < 4 ? true : false;
+ lucy_VTable_to_host_t to_host = (lucy_VTable_to_host_t)LUCY_SUPER_METHOD(
+ LUCY_VTABLE, VTable, To_Host);
+ SV *host_obj = (SV*)to_host(self);
+ if (first_time) {
+ SvSHARE((SV*)self->ref.host_obj);
+ }
+ return host_obj;
+}
+
+
diff --git a/perl/xs/Lucy/Store/FSFolder.c b/perl/xs/Lucy/Store/FSFolder.c
new file mode 100644
index 0000000..ac3eb94
--- /dev/null
+++ b/perl/xs/Lucy/Store/FSFolder.c
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Store/FSFolder.h"
+
+CharBuf*
+FSFolder_absolutify(const CharBuf *path) {
+
+ return Host_callback_str(FSFOLDER, "absolutify", 1,
+ ARG_STR("path", path));
+}
+
diff --git a/perl/xs/Lucy/Util/Json.c b/perl/xs/Lucy/Util/Json.c
new file mode 100644
index 0000000..c58eac6
--- /dev/null
+++ b/perl/xs/Lucy/Util/Json.c
@@ -0,0 +1,59 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Util/Json.h"
+#include "Lucy/Object/Host.h"
+#include "Lucy/Store/Folder.h"
+
+bool_t
+Json_spew_json(Obj *dump, Folder *folder, const CharBuf *path) {
+ bool_t result = (bool_t)Host_callback_i64(JSON, "spew_json", 3,
+ ARG_OBJ("dump", dump),
+ ARG_OBJ("folder", folder),
+ ARG_STR("path", path));
+ if (!result) { ERR_ADD_FRAME(Err_get_error()); }
+ return result;
+}
+
+Obj*
+Json_slurp_json(Folder *folder, const CharBuf *path) {
+ Obj *dump = Host_callback_obj(JSON, "slurp_json", 2,
+ ARG_OBJ("folder", folder),
+ ARG_STR("path", path));
+ if (!dump) { ERR_ADD_FRAME(Err_get_error()); }
+ return dump;
+}
+
+CharBuf*
+Json_to_json(Obj *dump) {
+ return Host_callback_str(JSON, "to_json", 1,
+ ARG_OBJ("dump", dump));
+}
+
+Obj*
+Json_from_json(CharBuf *json) {
+ return Host_callback_obj(JSON, "from_json", 1,
+ ARG_STR("json", json));
+}
+
+void
+Json_set_tolerant(bool_t tolerant) {
+ Host_callback(JSON, "set_tolerant", 1,
+ ARG_I32("tolerant", tolerant));
+}
+
diff --git a/perl/xs/Lucy/Util/StringHelper.c b/perl/xs/Lucy/Util/StringHelper.c
new file mode 100644
index 0000000..6eeb1c8
--- /dev/null
+++ b/perl/xs/Lucy/Util/StringHelper.c
@@ -0,0 +1,27 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "XSBind.h"
+#include "Lucy/Util/StringHelper.h"
+
+// TODO: replace with code from ICU in common/ucnv_u8.c.
+chy_bool_t
+lucy_StrHelp_utf8_valid(const char *ptr, size_t size) {
+ const U8 *uptr = (const U8*)ptr;
+ return size == 0 ? true : !!is_utf8_string(uptr, size);
+}
+
+
diff --git a/perl/xs/XSBind.c b/perl/xs/XSBind.c
new file mode 100644
index 0000000..c331e11
--- /dev/null
+++ b/perl/xs/XSBind.c
@@ -0,0 +1,553 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define C_LUCY_OBJ
+#define NEED_newRV_noinc
+#include "XSBind.h"
+#include "Lucy/Util/StringHelper.h"
+
+// Convert a Perl hash into a Clownfish Hash. Caller takes responsibility for
+// a refcount.
+static cfish_Hash*
+S_perl_hash_to_cfish_hash(HV *phash);
+
+// Convert a Perl array into a Clownfish VArray. Caller takes responsibility
+// for a refcount.
+static cfish_VArray*
+S_perl_array_to_cfish_array(AV *parray);
+
+// Convert a VArray to a Perl array. Caller takes responsibility for a
+// refcount.
+static SV*
+S_cfish_array_to_perl_array(cfish_VArray *varray);
+
+// Convert a Hash to a Perl hash. Caller takes responsibility for a refcount.
+static SV*
+S_cfish_hash_to_perl_hash(cfish_Hash *hash);
+
+cfish_Obj*
+XSBind_new_blank_obj(SV *either_sv) {
+ cfish_VTable *vtable;
+
+ // Get a VTable.
+ if (sv_isobject(either_sv)
+ && sv_derived_from(either_sv, "Lucy::Object::Obj")
+ ) {
+ // Use the supplied object's VTable.
+ IV iv_ptr = SvIV(SvRV(either_sv));
+ cfish_Obj *self = INT2PTR(cfish_Obj*, iv_ptr);
+ vtable = self->vtable;
+ }
+ else {
+ // Use the supplied class name string to find a VTable.
+ STRLEN len;
+ char *ptr = SvPVutf8(either_sv, len);
+ cfish_ZombieCharBuf *klass = CFISH_ZCB_WRAP_STR(ptr, len);
+ vtable = cfish_VTable_singleton((cfish_CharBuf*)klass, NULL);
+ }
+
+ // Use the VTable to allocate a new blank object of the right size.
+ return Cfish_VTable_Make_Obj(vtable);
+}
+
+cfish_Obj*
+XSBind_sv_to_cfish_obj(SV *sv, cfish_VTable *vtable, void *allocation) {
+ cfish_Obj *retval = XSBind_maybe_sv_to_cfish_obj(sv, vtable, allocation);
+ if (!retval) {
+ THROW(CFISH_ERR, "Not a %o", Cfish_VTable_Get_Name(vtable));
+ }
+ return retval;
+}
+
+cfish_Obj*
+XSBind_maybe_sv_to_cfish_obj(SV *sv, cfish_VTable *vtable, void *allocation) {
+ cfish_Obj *retval = NULL;
+ if (XSBind_sv_defined(sv)) {
+ if (sv_isobject(sv)
+ && sv_derived_from(sv, (char*)Cfish_CB_Get_Ptr8(Cfish_VTable_Get_Name(vtable)))
+ ) {
+ // Unwrap a real Clownfish object.
+ IV tmp = SvIV(SvRV(sv));
+ retval = INT2PTR(cfish_Obj*, tmp);
+ }
+ else if (allocation &&
+ (vtable == CFISH_ZOMBIECHARBUF
+ || vtable == CFISH_VIEWCHARBUF
+ || vtable == CFISH_CHARBUF
+ || vtable == CFISH_OBJ)
+ ) {
+ // Wrap the string from an ordinary Perl scalar inside a
+ // ZombieCharBuf.
+ STRLEN size;
+ char *ptr = SvPVutf8(sv, size);
+ retval = (cfish_Obj*)cfish_ZCB_wrap_str(allocation, ptr, size);
+ }
+ else if (SvROK(sv)) {
+ // Attempt to convert Perl hashes and arrays into their Clownfish
+ // analogues.
+ SV *inner = SvRV(sv);
+ if (SvTYPE(inner) == SVt_PVAV && vtable == CFISH_VARRAY) {
+ retval = (cfish_Obj*)S_perl_array_to_cfish_array((AV*)inner);
+ }
+ else if (SvTYPE(inner) == SVt_PVHV && vtable == CFISH_HASH) {
+ retval = (cfish_Obj*)S_perl_hash_to_cfish_hash((HV*)inner);
+ }
+
+ if (retval) {
+ // Mortalize the converted object -- which is somewhat
+ // dangerous, but is the only way to avoid requiring that the
+ // caller take responsibility for a refcount.
+ SV *mortal = (SV*)Cfish_Obj_To_Host(retval);
+ LUCY_DECREF(retval);
+ sv_2mortal(mortal);
+ }
+ }
+ }
+
+ return retval;
+}
+
+SV*
+XSBind_cfish_to_perl(cfish_Obj *obj) {
+ if (obj == NULL) {
+ return newSV(0);
+ }
+ else if (Cfish_Obj_Is_A(obj, CFISH_CHARBUF)) {
+ return XSBind_cb_to_sv((cfish_CharBuf*)obj);
+ }
+ else if (Cfish_Obj_Is_A(obj, CFISH_BYTEBUF)) {
+ return XSBind_bb_to_sv((cfish_ByteBuf*)obj);
+ }
+ else if (Cfish_Obj_Is_A(obj, CFISH_VARRAY)) {
+ return S_cfish_array_to_perl_array((cfish_VArray*)obj);
+ }
+ else if (Cfish_Obj_Is_A(obj, CFISH_HASH)) {
+ return S_cfish_hash_to_perl_hash((cfish_Hash*)obj);
+ }
+ else if (Cfish_Obj_Is_A(obj, CFISH_FLOATNUM)) {
+ return newSVnv(Cfish_Obj_To_F64(obj));
+ }
+ else if (sizeof(IV) == 8 && Cfish_Obj_Is_A(obj, CFISH_INTNUM)) {
+ int64_t num = Cfish_Obj_To_I64(obj);
+ return newSViv((IV)num);
+ }
+ else if (sizeof(IV) == 4 && Cfish_Obj_Is_A(obj, CFISH_INTEGER32)) {
+ int32_t num = (int32_t)Cfish_Obj_To_I64(obj);
+ return newSViv((IV)num);
+ }
+ else if (sizeof(IV) == 4 && Cfish_Obj_Is_A(obj, CFISH_INTEGER64)) {
+ int64_t num = Cfish_Obj_To_I64(obj);
+ return newSVnv((double)num); // lossy
+ }
+ else {
+ return (SV*)Cfish_Obj_To_Host(obj);
+ }
+}
+
+cfish_Obj*
+XSBind_perl_to_cfish(SV *sv) {
+ cfish_Obj *retval = NULL;
+
+ if (XSBind_sv_defined(sv)) {
+ if (SvROK(sv)) {
+ // Deep conversion of references.
+ SV *inner = SvRV(sv);
+ if (SvTYPE(inner) == SVt_PVAV) {
+ retval = (cfish_Obj*)S_perl_array_to_cfish_array((AV*)inner);
+ }
+ else if (SvTYPE(inner) == SVt_PVHV) {
+ retval = (cfish_Obj*)S_perl_hash_to_cfish_hash((HV*)inner);
+ }
+ else if (sv_isobject(sv)
+ && sv_derived_from(sv, "Lucy::Object::Obj")
+ ) {
+ IV tmp = SvIV(inner);
+ retval = INT2PTR(cfish_Obj*, tmp);
+ (void)LUCY_INCREF(retval);
+ }
+ }
+
+ // It's either a plain scalar or a non-Clownfish Perl object, so
+ // stringify.
+ if (!retval) {
+ STRLEN len;
+ char *ptr = SvPVutf8(sv, len);
+ retval = (cfish_Obj*)cfish_CB_new_from_trusted_utf8(ptr, len);
+ }
+ }
+ else if (sv) {
+ // Deep conversion of raw AVs and HVs.
+ if (SvTYPE(sv) == SVt_PVAV) {
+ retval = (cfish_Obj*)S_perl_array_to_cfish_array((AV*)sv);
+ }
+ else if (SvTYPE(sv) == SVt_PVHV) {
+ retval = (cfish_Obj*)S_perl_hash_to_cfish_hash((HV*)sv);
+ }
+ }
+
+ return retval;
+}
+
+SV*
+XSBind_bb_to_sv(const cfish_ByteBuf *bb) {
+ return bb
+ ? newSVpvn(Cfish_BB_Get_Buf(bb), Cfish_BB_Get_Size(bb))
+ : newSV(0);
+}
+
+SV*
+XSBind_cb_to_sv(const cfish_CharBuf *cb) {
+ if (!cb) {
+ return newSV(0);
+ }
+ else {
+ SV *sv = newSVpvn((char*)Cfish_CB_Get_Ptr8(cb), Cfish_CB_Get_Size(cb));
+ SvUTF8_on(sv);
+ return sv;
+ }
+}
+
+static cfish_Hash*
+S_perl_hash_to_cfish_hash(HV *phash) {
+ uint32_t num_keys = hv_iterinit(phash);
+ cfish_Hash *retval = cfish_Hash_new(num_keys);
+ cfish_ZombieCharBuf *key = CFISH_ZCB_WRAP_STR("", 0);
+
+ while (num_keys--) {
+ HE *entry = hv_iternext(phash);
+ STRLEN key_len = HeKLEN(entry);
+ SV *value_sv = HeVAL(entry);
+ cfish_Obj *value = XSBind_perl_to_cfish(value_sv); // Recurse.
+
+ // Force key to UTF-8 if necessary.
+ if (key_len == (STRLEN)HEf_SVKEY) {
+ // Key is stored as an SV. Use its UTF-8 flag? Not sure about
+ // this.
+ SV *key_sv = HeKEY_sv(entry);
+ char *key_str = SvPVutf8(key_sv, key_len);
+ Cfish_ZCB_Assign_Trusted_Str(key, key_str, key_len);
+ Cfish_Hash_Store(retval, (cfish_Obj*)key, value);
+ }
+ else if (HeKUTF8(entry)) {
+ Cfish_ZCB_Assign_Trusted_Str(key, HeKEY(entry), key_len);
+ Cfish_Hash_Store(retval, (cfish_Obj*)key, value);
+ }
+ else {
+ char *key_str = HeKEY(entry);
+ chy_bool_t pure_ascii = true;
+ for (STRLEN i = 0; i < key_len; i++) {
+ if ((key_str[i] & 0x80) == 0x80) { pure_ascii = false; }
+ }
+ if (pure_ascii) {
+ Cfish_ZCB_Assign_Trusted_Str(key, key_str, key_len);
+ Cfish_Hash_Store(retval, (cfish_Obj*)key, value);
+ }
+ else {
+ SV *key_sv = HeSVKEY_force(entry);
+ key_str = SvPVutf8(key_sv, key_len);
+ Cfish_ZCB_Assign_Trusted_Str(key, key_str, key_len);
+ Cfish_Hash_Store(retval, (cfish_Obj*)key, value);
+ }
+ }
+ }
+
+ return retval;
+}
+
+static cfish_VArray*
+S_perl_array_to_cfish_array(AV *parray) {
+ const uint32_t size = av_len(parray) + 1;
+ cfish_VArray *retval = cfish_VA_new(size);
+ uint32_t i;
+
+ // Iterate over array elems.
+ for (i = 0; i < size; i++) {
+ SV **elem_sv = av_fetch(parray, i, false);
+ if (elem_sv) {
+ cfish_Obj *elem = XSBind_perl_to_cfish(*elem_sv);
+ if (elem) { Cfish_VA_Store(retval, i, elem); }
+ }
+ }
+ Cfish_VA_Resize(retval, size); // needed if last elem is NULL
+
+ return retval;
+}
+
+static SV*
+S_cfish_array_to_perl_array(cfish_VArray *varray) {
+ AV *perl_array = newAV();
+ uint32_t num_elems = Cfish_VA_Get_Size(varray);
+
+ // Iterate over array elems.
+ if (num_elems) {
+ uint32_t i;
+ av_fill(perl_array, num_elems - 1);
+ for (i = 0; i < num_elems; i++) {
+ cfish_Obj *val = Cfish_VA_Fetch(varray, i);
+ if (val == NULL) {
+ continue;
+ }
+ else {
+ // Recurse for each value.
+ SV *const val_sv = XSBind_cfish_to_perl(val);
+ av_store(perl_array, i, val_sv);
+ }
+ }
+ }
+
+ return newRV_noinc((SV*)perl_array);
+}
+
+static SV*
+S_cfish_hash_to_perl_hash(cfish_Hash *hash) {
+ HV *perl_hash = newHV();
+ SV *key_sv = newSV(1);
+ cfish_CharBuf *key;
+ cfish_Obj *val;
+
+ // Prepare the SV key.
+ SvPOK_on(key_sv);
+ SvUTF8_on(key_sv);
+
+ // Iterate over key-value pairs.
+ Cfish_Hash_Iterate(hash);
+ while (Cfish_Hash_Next(hash, (cfish_Obj**)&key, &val)) {
+ // Recurse for each value.
+ SV *val_sv = XSBind_cfish_to_perl(val);
+ if (!Cfish_Obj_Is_A((cfish_Obj*)key, CFISH_CHARBUF)) {
+ CFISH_THROW(CFISH_ERR,
+ "Can't convert a key of class %o to a Perl hash key",
+ Cfish_Obj_Get_Class_Name((cfish_Obj*)key));
+ }
+ else {
+ STRLEN key_size = Cfish_CB_Get_Size(key);
+ char *key_sv_ptr = SvGROW(key_sv, key_size + 1);
+ memcpy(key_sv_ptr, Cfish_CB_Get_Ptr8(key), key_size);
+ SvCUR_set(key_sv, key_size);
+ *SvEND(key_sv) = '\0';
+ (void)hv_store_ent(perl_hash, key_sv, val_sv, 0);
+ }
+ }
+ SvREFCNT_dec(key_sv);
+
+ return newRV_noinc((SV*)perl_hash);
+}
+
+void
+XSBind_enable_overload(void *pobj) {
+ SV *perl_obj = (SV*)pobj;
+ HV *stash = SvSTASH(SvRV(perl_obj));
+#if (PERL_VERSION > 10)
+ Gv_AMupdate(stash, false);
+#else
+ Gv_AMupdate(stash);
+#endif
+ SvAMAGIC_on(perl_obj);
+}
+
+static chy_bool_t
+S_extract_from_sv(SV *value, void *target, const char *label,
+ chy_bool_t required, int type, cfish_VTable *vtable,
+ void *allocation) {
+ chy_bool_t valid_assignment = false;
+
+ if (XSBind_sv_defined(value)) {
+ switch (type) {
+ case XSBIND_WANT_I8:
+ *((int8_t*)target) = (int8_t)SvIV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_I16:
+ *((int16_t*)target) = (int16_t)SvIV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_I32:
+ *((int32_t*)target) = (int32_t)SvIV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_I64:
+ if (sizeof(IV) == 8) {
+ *((int64_t*)target) = (int64_t)SvIV(value);
+ }
+ else { // sizeof(IV) == 4
+ // lossy.
+ *((int64_t*)target) = (int64_t)SvNV(value);
+ }
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_U8:
+ *((uint8_t*)target) = (uint8_t)SvUV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_U16:
+ *((uint16_t*)target) = (uint16_t)SvUV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_U32:
+ *((uint32_t*)target) = (uint32_t)SvUV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_U64:
+ if (sizeof(UV) == 8) {
+ *((uint64_t*)target) = (uint64_t)SvUV(value);
+ }
+ else { // sizeof(UV) == 4
+ // lossy.
+ *((uint64_t*)target) = (uint64_t)SvNV(value);
+ }
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_BOOL:
+ *((chy_bool_t*)target) = !!SvTRUE(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_F32:
+ *((float*)target) = (float)SvNV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_F64:
+ *((double*)target) = SvNV(value);
+ valid_assignment = true;
+ break;
+ case XSBIND_WANT_OBJ: {
+ cfish_Obj *object
+ = XSBind_maybe_sv_to_cfish_obj(value, vtable,
+ allocation);
+ if (object) {
+ *((cfish_Obj**)target) = object;
+ valid_assignment = true;
+ }
+ else {
+ cfish_CharBuf *mess
+ = CFISH_MAKE_MESS(
+ "Invalid value for '%s' - not a %o",
+ label, Cfish_VTable_Get_Name(vtable));
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+ }
+ break;
+ case XSBIND_WANT_SV:
+ *((SV**)target) = value;
+ valid_assignment = true;
+ break;
+ default: {
+ cfish_CharBuf *mess
+ = CFISH_MAKE_MESS("Unrecognized type: %i32 for param '%s'",
+ (int32_t)type, label);
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+ }
+ }
+
+ // Enforce that required params cannot be undef and must present valid
+ // values.
+ if (required && !valid_assignment) {
+ cfish_CharBuf *mess = CFISH_MAKE_MESS("Missing required param %s",
+ label);
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+
+ return true;
+}
+
+chy_bool_t
+XSBind_allot_params(SV** stack, int32_t start, int32_t num_stack_elems,
+ char* params_hash_name, ...) {
+ va_list args;
+ HV *params_hash = get_hv(params_hash_name, 0);
+ int32_t args_left = (num_stack_elems - start) / 2;
+
+ // Retrieve the params hash, which must be a package global.
+ if (params_hash == NULL) {
+ cfish_CharBuf *mess = CFISH_MAKE_MESS("Can't find hash named %s",
+ params_hash_name);
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+
+ // Verify that our args come in pairs. Return success if there are no
+ // args.
+ if (num_stack_elems == start) { return true; }
+ if ((num_stack_elems - start) % 2 != 0) {
+ cfish_CharBuf *mess
+ = CFISH_MAKE_MESS(
+ "Expecting hash-style params, got odd number of args");
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+
+ // Validate param names.
+ for (int32_t i = start; i < num_stack_elems; i += 2) {
+ SV *const key_sv = stack[i];
+ STRLEN key_len;
+ const char *key = SvPV(key_sv, key_len); // assume ASCII labels
+ if (!hv_exists(params_hash, key, key_len)) {
+ cfish_CharBuf *mess
+ = CFISH_MAKE_MESS("Invalid parameter: '%s'", key);
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+ }
+
+ void *target;
+ va_start(args, params_hash_name);
+ while (args_left && NULL != (target = va_arg(args, void*))) {
+ char *label = va_arg(args, char*);
+ int label_len = va_arg(args, int);
+ int required = va_arg(args, int);
+ int type = va_arg(args, int);
+ cfish_VTable *vtable = va_arg(args, cfish_VTable*);
+ void *allocation = va_arg(args, void*);
+
+ // Iterate through stack looking for a label match. Work backwards so
+ // that if the label is doubled up we get the last one.
+ chy_bool_t got_arg = false;
+ for (int32_t i = num_stack_elems; i >= start + 2; i -= 2) {
+ int32_t tick = i - 2;
+ SV *const key_sv = stack[tick];
+ if (SvCUR(key_sv) == (STRLEN)label_len) {
+ if (memcmp(SvPVX(key_sv), label, label_len) == 0) {
+ SV *value = stack[tick + 1];
+ got_arg = S_extract_from_sv(value, target, label,
+ required, type, vtable,
+ allocation);
+ if (!got_arg) {
+ CFISH_ERR_ADD_FRAME(cfish_Err_get_error());
+ return false;
+ }
+ args_left--;
+ break;
+ }
+ }
+ }
+
+ // Enforce required params.
+ if (required && !got_arg) {
+ cfish_CharBuf *mess
+ = CFISH_MAKE_MESS("Missing required parameter: '%s'", label);
+ cfish_Err_set_error(cfish_Err_new(mess));
+ return false;
+ }
+ }
+ va_end(args);
+
+ return true;
+}
+
+
diff --git a/perl/xs/XSBind.h b/perl/xs/XSBind.h
new file mode 100644
index 0000000..4d4194c
--- /dev/null
+++ b/perl/xs/XSBind.h
@@ -0,0 +1,340 @@
+/* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* XSBind.h -- Functions to help bind Clownfish to Perl XS api.
+ */
+
+#ifndef H_CFISH_XSBIND
+#define H_CFISH_XSBIND 1
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "charmony.h"
+#include "Lucy/Object/Obj.h"
+#include "Lucy/Object/ByteBuf.h"
+#include "Lucy/Object/CharBuf.h"
+#include "Lucy/Object/Err.h"
+#include "Lucy/Object/Hash.h"
+#include "Lucy/Object/Num.h"
+#include "Lucy/Object/VArray.h"
+#include "Lucy/Object/VTable.h"
+
+#include "EXTERN.h"
+#include "perl.h"
+#include "XSUB.h"
+
+#define NEED_newRV_noinc_GLOBAL
+#include "ppport.h"
+
+/** Given either a class name or a perl object, manufacture a new Clownfish
+ * object suitable for supplying to a cfish_Foo_init() function.
+ */
+cfish_Obj*
+cfish_XSBind_new_blank_obj(SV *either_sv);
+
+/** Test whether an SV is defined. Handles "get" magic, unlike SvOK on its
+ * own.
+ */
+static CHY_INLINE chy_bool_t
+cfish_XSBind_sv_defined(SV *sv) {
+ if (!sv || !SvANY(sv)) { return false; }
+ if (SvGMAGICAL(sv)) { mg_get(sv); }
+ return SvOK(sv);
+}
+
+/** If the SV contains a Clownfish object which passes an "isa" test against the
+ * passed-in VTable, return a pointer to it. If not, but
+ * <code>allocation</code> is non-NULL and a ZombieCharBuf would satisfy the
+ * "isa" test, stringify the SV, create a ZombieCharBuf using
+ * <code>allocation</code>, assign the SV's string to it, and return that
+ * instead. If all else fails, throw an exception.
+ */
+cfish_Obj*
+cfish_XSBind_sv_to_cfish_obj(SV *sv, cfish_VTable *vtable, void *allocation);
+
+/** As XSBind_sv_to_cfish_obj above, but returns NULL instead of throwing an
+ * exception.
+ */
+cfish_Obj*
+cfish_XSBind_maybe_sv_to_cfish_obj(SV *sv, cfish_VTable *vtable,
+ void *allocation);
+
+
+/** Derive an SV from a Clownfish object. If the Clownfish object is NULL, the SV
+ * will be undef.
+ *
+ * The new SV has single refcount for which the caller must take
+ * responsibility.
+ */
+static CHY_INLINE SV*
+cfish_XSBind_cfish_obj_to_sv(cfish_Obj *obj) {
+ return obj ? (SV*)Cfish_Obj_To_Host(obj) : newSV(0);
+}
+
+/** XSBind_cfish_obj_to_sv, with a cast.
+ */
+#define CFISH_OBJ_TO_SV(_obj) cfish_XSBind_cfish_obj_to_sv((cfish_Obj*)_obj)
+
+/** As XSBind_cfish_obj_to_sv above, except decrements the object's refcount
+ * after creating the SV. This is useful when the Clownfish expression creates a new
+ * refcount, e.g. a call to a constructor.
+ */
+static CHY_INLINE SV*
+cfish_XSBind_cfish_obj_to_sv_noinc(cfish_Obj *obj) {
+ SV *retval;
+ if (obj) {
+ retval = (SV*)Cfish_Obj_To_Host(obj);
+ Cfish_Obj_Dec_RefCount(obj);
+ }
+ else {
+ retval = newSV(0);
+ }
+ return retval;
+}
+
+/** XSBind_cfish_obj_to_sv_noinc, with a cast.
+ */
+#define CFISH_OBJ_TO_SV_NOINC(_obj) \
+ cfish_XSBind_cfish_obj_to_sv_noinc((cfish_Obj*)_obj)
+
+/** Deep conversion of Clownfish objects to Perl objects -- CharBufs to UTF-8
+ * SVs, ByteBufs to SVs, VArrays to Perl array refs, Hashes to Perl hashrefs,
+ * and any other object to a Perl object wrapping the Clownfish Obj.
+ */
+SV*
+cfish_XSBind_cfish_to_perl(cfish_Obj *obj);
+
+/** Deep conversion of Perl data structures to Clownfish objects -- Perl hash
+ * to Hash, Perl array to VArray, Clownfish objects stripped of their
+ * wrappers, and everything else stringified and turned to a CharBuf.
+ */
+cfish_Obj*
+cfish_XSBind_perl_to_cfish(SV *sv);
+
+/** Convert a ByteBuf into a new string SV.
+ */
+SV*
+cfish_XSBind_bb_to_sv(const cfish_ByteBuf *bb);
+
+/** Convert a CharBuf into a new UTF-8 string SV.
+ */
+SV*
+cfish_XSBind_cb_to_sv(const cfish_CharBuf *cb);
+
+/** Turn on overloading for the supplied Perl object and its class.
+ */
+void
+cfish_XSBind_enable_overload(void *pobj);
+
+/** Process hash-style params passed to an XS subroutine. The varargs must be
+ * a NULL-terminated series of ALLOT_ macros.
+ *
+ * cfish_XSBind_allot_params(stack, start, num_stack_elems,
+ * "Lucy::Search::TermQuery::new_PARAMS",
+ * ALLOT_OBJ(&field, "field", 5, LUCY_CHARBUF, true, alloca(cfish_ZCB_size()),
+ * ALLOT_OBJ(&term, "term", 4, LUCY_CHARBUF, true, alloca(cfish_ZCB_size()),
+ * NULL);
+ *
+ * The following ALLOT_ macros are available for primitive types:
+ *
+ * ALLOT_I8(ptr, key, keylen, required)
+ * ALLOT_I16(ptr, key, keylen, required)
+ * ALLOT_I32(ptr, key, keylen, required)
+ * ALLOT_I64(ptr, key, keylen, required)
+ * ALLOT_U8(ptr, key, keylen, required)
+ * ALLOT_U16(ptr, key, keylen, required)
+ * ALLOT_U32(ptr, key, keylen, required)
+ * ALLOT_U64(ptr, key, keylen, required)
+ * ALLOT_BOOL(ptr, key, keylen, required)
+ * ALLOT_CHAR(ptr, key, keylen, required)
+ * ALLOT_SHORT(ptr, key, keylen, required)
+ * ALLOT_INT(ptr, key, keylen, required)
+ * ALLOT_LONG(ptr, key, keylen, required)
+ * ALLOT_SIZE_T(ptr, key, keylen, required)
+ * ALLOT_F32(ptr, key, keylen, required)
+ * ALLOT_F64(ptr, key, keylen, required)
+ *
+ * The four arguments to these ALLOT_ macros have the following meanings:
+ *
+ * ptr -- A pointer to the variable to be extracted.
+ * key -- The name of the parameter as a C string.
+ * keylen -- The length of the parameter name in bytes.
+ * required -- A boolean indicating whether the parameter is required.
+ *
+ * If a required parameter is not present, allot_params() will immediately
+ * cease processing of parameters, set Err_error and return false.
+ *
+ * Use the following macro if a Clownfish object is desired:
+ *
+ * ALLOT_OBJ(ptr, key, keylen, required, vtable, allocation)
+ *
+ * The "vtable" argument must be the VTable corresponding to the class of the
+ * desired object. The "allocation" argument must be a blob of memory
+ * allocated on the stack sufficient to hold a ZombieCharBuf. (Use
+ * cfish_ZCB_size() to find the allocation size.)
+ *
+ * To extract a Perl scalar, use the following ALLOT_ macro:
+ *
+ * ALLOT_SV(ptr, key, keylen, required)
+ *
+ * @param stack The Perl stack.
+ * @param start Where on the Perl stack to start looking for params. For
+ * methods, this would typically be 1; for functions, most likely 0.
+ * @param num_stack_elems The number of arguments passed to the Perl function
+ * (generally, the XS variable "items").
+ * @param params_hash_name The name of a package global hash. Any param
+ * labels which are not present in this hash will trigger an exception.
+ * @return true on success, false on failure (sets Err_error).
+ */
+chy_bool_t
+cfish_XSBind_allot_params(SV** stack, int32_t start,
+ int32_t num_stack_elems,
+ char* params_hash_name, ...);
+
+#define XSBIND_WANT_I8 0x1
+#define XSBIND_WANT_I16 0x2
+#define XSBIND_WANT_I32 0x3
+#define XSBIND_WANT_I64 0x4
+#define XSBIND_WANT_U8 0x5
+#define XSBIND_WANT_U16 0x6
+#define XSBIND_WANT_U32 0x7
+#define XSBIND_WANT_U64 0x8
+#define XSBIND_WANT_BOOL 0x9
+#define XSBIND_WANT_F32 0xA
+#define XSBIND_WANT_F64 0xB
+#define XSBIND_WANT_OBJ 0xC
+#define XSBIND_WANT_SV 0xD
+
+#if (CHY_SIZEOF_CHAR == 1)
+ #define XSBIND_WANT_CHAR XSBIND_WANT_I8
+#else
+ #error Can't build unless sizeof(char) == 1
+#endif
+
+#if (CHY_SIZEOF_SHORT == 2)
+ #define XSBIND_WANT_SHORT XSBIND_WANT_I16
+#else
+ #error Can't build unless sizeof(short) == 2
+#endif
+
+#if (CHY_SIZEOF_INT == 4)
+ #define XSBIND_WANT_INT XSBIND_WANT_I32
+#else // sizeof(int) == 8
+ #define XSBIND_WANT_INT XSBIND_WANT_I64
+#endif
+
+#if (CHY_SIZEOF_LONG == 4)
+ #define XSBIND_WANT_LONG XSBIND_WANT_I32
+#else // sizeof(long) == 8
+ #define XSBIND_WANT_LONG XSBIND_WANT_I64
+#endif
+
+#if (CHY_SIZEOF_SIZE_T == 4)
+ #define XSBIND_WANT_SIZE_T XSBIND_WANT_U32
+#else // sizeof(long) == 8
+ #define XSBIND_WANT_SIZE_T XSBIND_WANT_U64
+#endif
+
+#define XSBIND_ALLOT_I8(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_I8, NULL, NULL
+#define XSBIND_ALLOT_I16(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_I16, NULL, NULL
+#define XSBIND_ALLOT_I32(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_I32, NULL, NULL
+#define XSBIND_ALLOT_I64(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_I64, NULL, NULL
+#define XSBIND_ALLOT_U8(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_U8, NULL, NULL
+#define XSBIND_ALLOT_U16(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_U16, NULL, NULL
+#define XSBIND_ALLOT_U32(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_U32, NULL, NULL
+#define XSBIND_ALLOT_U64(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_U64, NULL, NULL
+#define XSBIND_ALLOT_BOOL(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_BOOL, NULL, NULL
+#define XSBIND_ALLOT_CHAR(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_CHAR, NULL, NULL
+#define XSBIND_ALLOT_SHORT(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_SHORT, NULL, NULL
+#define XSBIND_ALLOT_INT(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_INT, NULL, NULL
+#define XSBIND_ALLOT_LONG(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_LONG, NULL, NULL
+#define XSBIND_ALLOT_SIZE_T(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_SIZE_T, NULL, NULL
+#define XSBIND_ALLOT_F32(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_F32, NULL, NULL
+#define XSBIND_ALLOT_F64(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_F64, NULL, NULL
+#define XSBIND_ALLOT_OBJ(ptr, key, keylen, required, vtable, allocation) \
+ ptr, key, keylen, required, XSBIND_WANT_OBJ, vtable, allocation
+#define XSBIND_ALLOT_SV(ptr, key, keylen, required) \
+ ptr, key, keylen, required, XSBIND_WANT_SV, NULL, NULL
+
+/* Define short names for most of the symbols in this file. Note that these
+ * short names are ALWAYS in effect, since they are only used for Perl and we
+ * can be confident they don't conflict with anything. (It's prudent to use
+ * full symbols nevertheless in case someone else defines e.g. a function
+ * named "XSBind_sv_defined".)
+ */
+#define XSBind_new_blank_obj cfish_XSBind_new_blank_obj
+#define XSBind_sv_defined cfish_XSBind_sv_defined
+#define XSBind_sv_to_cfish_obj cfish_XSBind_sv_to_cfish_obj
+#define XSBind_maybe_sv_to_cfish_obj cfish_XSBind_maybe_sv_to_cfish_obj
+#define XSBind_cfish_obj_to_sv cfish_XSBind_cfish_obj_to_sv
+#define XSBind_cfish_obj_to_sv_noinc cfish_XSBind_cfish_obj_to_sv_noinc
+#define XSBind_cfish_to_perl cfish_XSBind_cfish_to_perl
+#define XSBind_perl_to_cfish cfish_XSBind_perl_to_cfish
+#define XSBind_bb_to_sv cfish_XSBind_bb_to_sv
+#define XSBind_cb_to_sv cfish_XSBind_cb_to_sv
+#define XSBind_enable_overload cfish_XSBind_enable_overload
+#define XSBind_allot_params cfish_XSBind_allot_params
+#define ALLOT_I8 XSBIND_ALLOT_I8
+#define ALLOT_I16 XSBIND_ALLOT_I16
+#define ALLOT_I32 XSBIND_ALLOT_I32
+#define ALLOT_I64 XSBIND_ALLOT_I64
+#define ALLOT_U8 XSBIND_ALLOT_U8
+#define ALLOT_U16 XSBIND_ALLOT_U16
+#define ALLOT_U32 XSBIND_ALLOT_U32
+#define ALLOT_U64 XSBIND_ALLOT_U64
+#define ALLOT_BOOL XSBIND_ALLOT_BOOL
+#define ALLOT_CHAR XSBIND_ALLOT_CHAR
+#define ALLOT_SHORT XSBIND_ALLOT_SHORT
+#define ALLOT_INT XSBIND_ALLOT_INT
+#define ALLOT_LONG XSBIND_ALLOT_LONG
+#define ALLOT_SIZE_T XSBIND_ALLOT_SIZE_T
+#define ALLOT_F32 XSBIND_ALLOT_F32
+#define ALLOT_F64 XSBIND_ALLOT_F64
+#define ALLOT_OBJ XSBIND_ALLOT_OBJ
+#define ALLOT_SV XSBIND_ALLOT_SV
+
+/* Strip the prefix from some common ClownFish symbols where we know there's
+ * no conflict with Perl. It's a little inconsistent to do this rather than
+ * leave all symbols at full size, but the succinctness is worth it.
+ */
+#define THROW CFISH_THROW
+#define WARN CFISH_WARN
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // H_CFISH_XSBIND
+
+
