FAQ

svn commit: r884885 - in /lucene/lucy/trunk: core/Lucy/Store/ core/Lucy/Test/Store/ perl/lib/Lucy/ perl/t/core/

Marvin
Nov 27, 2009 at 2:48 pm
Author: marvin
Date: Fri Nov 27 14:47:39 2009
New Revision: 884885

URL: http://svn.apache.org/viewvc?rev=884885&view=rev
Log:
Commit LUCY-76, adding CompoundFileReader.

Added:
lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.bp (with props)
lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.c (with props)
lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.bp (with props)
lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.c (with props)
lucene/lucy/trunk/perl/t/core/113-cf_reader.t (with props)
Modified:
lucene/lucy/trunk/core/Lucy/Store/Folder.bp
lucene/lucy/trunk/core/Lucy/Store/Folder.c
lucene/lucy/trunk/core/Lucy/Store/RAMFolder.c
lucene/lucy/trunk/perl/lib/Lucy/Test.pm

Added: lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.bp
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.bp?rev=884885&view=auto
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.bp (added)
+++ lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.bp Fri Nov 27 14:47:39 2009
@@ -0,0 +1,112 @@
+parcel Lucy;
+
+/** Read from a compound file.
+ *
+ * A CompoundFileReader provides access to the files contained within the
+ * compound file format written by CompoundFileWriter. The InStream objects
+ * it spits out behave largely like InStreams opened against discrete files --
+ * e.g. Seek(0) seeks to the beginning of the sub-file, not the beginning of
+ * the compound file.
+ *
+ * Each of the InStreams spawned maintains its own memory buffer; however,
+ * they all share a single filehandle. This allows Lucy to get around
+ * the limitations that many operating systems place on the number of
+ * available filehandles.
+ */
+
+class Lucy::Store::CompoundFileReader cnick CFReader
+ extends Lucy::Store::Folder {
+
+ Folder *real_folder;
+ Hash *records;
+ InStream *instream;
+ i32_t format;
+
+ inert incremented CompoundFileReader*
+ open(Folder *folder);
+
+ /** Return a new CompoundFileReader or set Err_error and return NULL.
+ *
+ * @param folder A folder containing compound files.
+ */
+ inert CompoundFileReader*
+ do_open(CompoundFileReader *self, Folder *folder);
+
+ Folder*
+ Get_Real_Folder(CompoundFileReader *self);
+
+ void
+ Set_Path(CompoundFileReader *self, const CharBuf *path);
+
+ public void
+ Close(CompoundFileReader *self);
+
+ public void
+ Destroy(CompoundFileReader *self);
+
+ bool_t
+ Local_Delete(CompoundFileReader *self, const CharBuf *name);
+
+ bool_t
+ Local_Exists(CompoundFileReader *self, const CharBuf *name);
+
+ bool_t
+ Local_Is_Directory(CompoundFileReader *self, const CharBuf *name);
+
+ incremented FileHandle*
+ Local_Open_FileHandle(CompoundFileReader *self, const CharBuf *name,
+ u32_t flags);
+
+ incremented InStream*
+ Local_Open_In(CompoundFileReader *self, const CharBuf *name);
+
+ bool_t
+ Local_MkDir(CompoundFileReader *self, const CharBuf *name);
+
+ Folder*
+ Local_Find_Folder(CompoundFileReader *self, const CharBuf *name);
+
+ DirHandle*
+ Local_Open_Dir(CompoundFileReader *self);
+}
+
+/** DirHandle for CompoundFileReader.
+ */
+class Lucy::Store::CFReaderDirHandle cnick CFReaderDH
+ extends Lucy::Store::DirHandle {
+
+ CompoundFileReader *cf_reader;
+ VArray *elems;
+ i32_t tick;
+
+ inert incremented CFReaderDirHandle*
+ new(CompoundFileReader *cf_reader);
+
+ inert CFReaderDirHandle*
+ init(CFReaderDirHandle *self, CompoundFileReader *cf_reader);
+
+ bool_t
+ Next(CFReaderDirHandle *self);
+
+ bool_t
+ Entry_Is_Dir(CFReaderDirHandle *self);
+
+ bool_t
+ Close(CFReaderDirHandle *self);
+}
+
+/* Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+

Propchange: lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.bp
------------------------------------------------------------------------------
svn:eol-style = native

Added: lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.c
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.c?rev=884885&view=auto
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.c (added)
+++ lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.c Fri Nov 27 14:47:39 2009
@@ -0,0 +1,333 @@
+#define C_LUCY_COMPOUNDFILEREADER
+#define C_LUCY_CFREADERDIRHANDLE
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Util/Json.h"
+#include "Lucy/Util/StringHelper.h"
+
+static ZombieCharBuf cf_file = ZCB_LITERAL("cf.dat");
+static ZombieCharBuf cfmeta_file = ZCB_LITERAL("cfmeta.json");
+
+CompoundFileReader*
+CFReader_open(Folder *folder)
+{
+ CompoundFileReader *self
+ = (CompoundFileReader*)VTable_Make_Obj(COMPOUNDFILEREADER);
+ return CFReader_do_open(self, folder);
+}
+
+CompoundFileReader*
+CFReader_do_open(CompoundFileReader *self, Folder *folder)
+{
+ Hash *metadata = (Hash*)Json_slurp_json((Folder*)folder,
+ (CharBuf*)&cfmeta_file);
+ Err *error = NULL;
+
+ Folder_init((Folder*)self, Folder_Get_Path(folder));
+
+ /* Parse metadata file. */
+ if (!metadata || !Obj_Is_A(metadata, HASH)) {
+ error = Err_new(CB_newf("Can't read '%o' in '%o'", &cfmeta_file,
+ Folder_Get_Path(folder)));
+ }
+ else {
+ Obj *format = Hash_Fetch_Str(metadata, "format", 6);
+ self->format = format ? Obj_To_I64(format) : 0;
+ self->records = (Hash*)INCREF(Hash_Fetch_Str(metadata, "files", 5));
+ if (self->format < 1) {
+ error = Err_new(CB_newf(
+ "Corrupt %o file: Missing or invalid 'format'",
+ &cfmeta_file));
+ }
+ else if (self->format > CFWriter_current_file_format) {
+ error = Err_new(CB_newf("Unsupported compound file format: %i32 "
+ "(current = %i32", self->format,
+ CFWriter_current_file_format));
+ }
+ else if (!self->records) {
+ error = Err_new(CB_newf("Corrupt %o file: missing 'files' key",
+ &cfmeta_file));
+ }
+ }
+ DECREF(metadata);
+ if (error) {
+ Err_set_error(error);
+ DECREF(self);
+ return NULL;
+ }
+
+ /* Open an instream which we'll clone over and over. */
+ self->instream = Folder_Open_In(folder, (CharBuf*)&cf_file);
+ if(!self->instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ DECREF(self);
+ return NULL;
+ }
+
+ /* Assign. */
+ self->real_folder = (Folder*)INCREF(folder);
+
+ /* Strip directory name from filepaths for old format. */
+ if (self->format == 1) {
+ VArray *files = Hash_Keys(self->records);
+ ZombieCharBuf filename = ZCB_BLANK;
+ CharBuf *folder_path = Folder_Get_Path(folder);
+ size_t folder_path_len = CB_Length(folder_path);
+ u32_t i, max;
+
+ for (i = 0, max = VA_Get_Size(files); i < max; i++) {
+ CharBuf *orig = (CharBuf*)VA_Fetch(files, i);
+ if (CB_Starts_With(orig, folder_path)) {
+ Obj *record = Hash_Delete(self->records, (Obj*)orig);
+ ZCB_Assign(&filename, orig);
+ ZCB_Nip(&filename, folder_path_len + sizeof(DIR_SEP) - 1);
+ Hash_Store(self->records, (Obj*)&filename, (Obj*)record);
+ }
+ }
+ }
+
+ return self;
+}
+
+void
+CFReader_destroy(CompoundFileReader *self)
+{
+ DECREF(self->real_folder);
+ DECREF(self->instream);
+ DECREF(self->records);
+ SUPER_DESTROY(self, COMPOUNDFILEREADER);
+}
+
+Folder*
+CFReader_get_real_folder(CompoundFileReader *self) { return self->real_folder; }
+
+void
+CFReader_set_path(CompoundFileReader *self, const CharBuf *path)
+{
+ Folder_Set_Path(self->real_folder, path);
+ Folder_set_path((Folder*)self, path);
+}
+
+FileHandle*
+CFReader_local_open_filehandle(CompoundFileReader *self,
+ const CharBuf *name, u32_t flags)
+{
+ Hash *entry = (Hash*)Hash_Fetch(self->records, (Obj*)name);
+ FileHandle *fh = NULL;
+
+ if (entry) {
+ Err_set_error(Err_new(CB_newf(
+ "Can't open FileHandle for virtual file %o in '%o'", name,
+ self->path)));
+ }
+ else {
+ fh = Folder_Local_Open_FileHandle(self->real_folder, name, flags);
+ if (!fh) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ }
+
+ return fh;
+}
+
+bool_t
+CFReader_local_delete(CompoundFileReader *self, const CharBuf *name)
+{
+ Hash *record = (Hash*)Hash_Delete(self->records, (Obj*)name);
+ DECREF(record);
+
+ if (record == NULL) {
+ return Folder_Local_Delete(self->real_folder, name);
+ }
+ else {
+ /* Once the number of virtual files falls to 0, remove the compound
+ * files. */
+ if (Hash_Get_Size(self->records) == 0) {
+ if (!Folder_Delete(self->real_folder, (CharBuf*)&cf_file)) {
+ return false;
+ }
+ if (!Folder_Delete(self->real_folder, (CharBuf*)&cfmeta_file)) {
+ return false;
+
+ }
+ }
+ return true;
+ }
+}
+
+InStream*
+CFReader_local_open_in(CompoundFileReader *self, const CharBuf *name)
+{
+ Hash *entry = (Hash*)Hash_Fetch(self->records, (Obj*)name);
+
+ if (!entry) {
+ InStream *instream = Folder_Local_Open_In(self->real_folder, name);
+ if (!instream) {
+ ERR_ADD_FRAME(Err_get_error());
+ }
+ return instream;
+ }
+ else {
+ Obj *len = Hash_Fetch_Str(entry, "length", 6);
+ Obj *offset = Hash_Fetch_Str(entry, "offset", 6);
+ if (!len || !offset) {
+ Err_set_error(Err_new(CB_newf("Malformed entry for '%o' in '%o'",
+ name, Folder_Get_Path(self->real_folder))));
+ return NULL;
+ }
+ else if (CB_Get_Size(self->path)) {
+ CharBuf *fullpath = CB_newf("%o/%o", self->path, name);
+ InStream *instream = InStream_Reopen(self->instream, fullpath,
+ Obj_To_I64(offset), Obj_To_I64(len));
+ DECREF(fullpath);
+ return instream;
+ }
+ else {
+ return InStream_Reopen(self->instream, name,
+ Obj_To_I64(offset), Obj_To_I64(len));
+ }
+ }
+}
+
+bool_t
+CFReader_local_exists(CompoundFileReader *self, const CharBuf *name)
+{
+ if (Hash_Fetch(self->records, (Obj*)name)) { return true; }
+ if (Folder_Local_Exists(self->real_folder, name)) { return true; }
+ return false;
+}
+
+bool_t
+CFReader_local_is_directory(CompoundFileReader *self, const CharBuf *name)
+{
+ if (Hash_Fetch(self->records, (Obj*)name)) { return false; }
+ if (Folder_Local_Is_Directory(self->real_folder, name)) { return true; }
+ return false;
+}
+
+void
+CFReader_close(CompoundFileReader *self)
+{
+ InStream_Close(self->instream);
+}
+
+bool_t
+CFReader_local_mkdir(CompoundFileReader *self, const CharBuf *name)
+{
+ if (Hash_Fetch(self->records, (Obj*)name)) {
+ Err_set_error(Err_new(CB_newf("Can't MkDir: '%o' exists", name)));
+ return false;
+ }
+ else {
+ bool_t result = Folder_Local_MkDir(self->real_folder, name);
+ if (!result) { ERR_ADD_FRAME(Err_get_error()); }
+ return result;
+ }
+}
+
+Folder*
+CFReader_local_find_folder(CompoundFileReader *self, const CharBuf *name)
+{
+ if (Hash_Fetch(self->records, (Obj*)name)) { return false; }
+ return Folder_Local_Find_Folder(self->real_folder, name);
+}
+
+DirHandle*
+CFReader_local_open_dir(CompoundFileReader *self)
+{
+ return (DirHandle*)CFReaderDH_new(self);
+}
+
+/****************************************************************************/
+
+CFReaderDirHandle*
+CFReaderDH_new(CompoundFileReader *cf_reader)
+{
+ CFReaderDirHandle *self
+ = (CFReaderDirHandle*)VTable_Make_Obj(CFREADERDIRHANDLE);
+ return CFReaderDH_init(self, cf_reader);
+}
+
+CFReaderDirHandle*
+CFReaderDH_init(CFReaderDirHandle *self, CompoundFileReader *cf_reader)
+{
+ DH_init((DirHandle*)self, CFReader_Get_Path(cf_reader));
+ self->cf_reader = (CompoundFileReader*)INCREF(cf_reader);
+ self->elems = Hash_Keys(self->cf_reader->records);
+ self->tick = -1;
+ {
+ /* Accumulate entries from real Folder. */
+ DirHandle *dh = Folder_Local_Open_Dir(self->cf_reader->real_folder);
+ CharBuf *entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ VA_Push(self->elems, (Obj*)CB_Clone(entry));
+ }
+ DECREF(dh);
+ }
+ return self;
+}
+
+bool_t
+CFReaderDH_close(CFReaderDirHandle *self)
+{
+ if (self->elems) {
+ Obj_Dec_RefCount(self->elems);
+ self->elems = NULL;
+ }
+ if (self->cf_reader) {
+ Obj_Dec_RefCount(self->cf_reader);
+ self->cf_reader = NULL;
+ }
+ return true;
+}
+
+bool_t
+CFReaderDH_next(CFReaderDirHandle *self)
+{
+ if (self->elems) {
+ self->tick++;
+ if (self->tick < (i32_t)VA_Get_Size(self->elems)) {
+ CharBuf *path = (CharBuf*)CERTIFY(
+ VA_Fetch(self->elems, self->tick), CHARBUF);
+ CB_Mimic(self->entry, (Obj*)path);
+ return true;
+ }
+ else {
+ self->tick--;
+ return false;
+ }
+ }
+ return false;
+}
+
+bool_t
+CFReaderDH_entry_is_dir(CFReaderDirHandle *self)
+{
+ if (self->elems) {
+ CharBuf *name = (CharBuf*)VA_Fetch(self->elems, self->tick);
+ if (name) {
+ return CFReader_Local_Is_Directory(self->cf_reader, name);
+ }
+ }
+ return false;
+}
+
+/* Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+

Propchange: lucene/lucy/trunk/core/Lucy/Store/CompoundFileReader.c
------------------------------------------------------------------------------
svn:eol-style = native

Modified: lucene/lucy/trunk/core/Lucy/Store/Folder.bp
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Store/Folder.bp?rev=884885&r1=884884&r2=884885&view=diff
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Store/Folder.bp (original)
+++ lucene/lucy/trunk/core/Lucy/Store/Folder.bp Fri Nov 27 14:47:39 2009
@@ -140,6 +140,11 @@
public incremented ByteBuf*
Slurp_File(Folder *self, const CharBuf *path);

+ /** Collapse the contents of the directory into a compound file.
+ */
+ void
+ Consolidate(Folder *self, const CharBuf *path);
+
/** Given a filepath, return the Folder representing everything except
* the last component. E.g. the 'foo/bar' Folder for '/foo/bar/baz.txt',
* the 'foo' Folder for 'foo/bar', etc.

Modified: lucene/lucy/trunk/core/Lucy/Store/Folder.c
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Store/Folder.c?rev=884885&r1=884884&r2=884885&view=diff
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Store/Folder.c (original)
+++ lucene/lucy/trunk/core/Lucy/Store/Folder.c Fri Nov 27 14:47:39 2009
@@ -8,6 +8,8 @@
#endif

#include "Lucy/Store/Folder.h"
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
#include "Lucy/Store/DirHandle.h"
#include "Lucy/Store/FileHandle.h"
#include "Lucy/Store/InStream.h"
@@ -365,6 +367,32 @@
self->path = CB_Clone(path);
}

+void
+Folder_consolidate(Folder *self, const CharBuf *path)
+{
+ Folder *folder = Folder_Find_Folder(self, path);
+ Folder *enclosing_folder = Folder_Enclosing_Folder(self, path);
+ if (!folder) {
+ THROW(ERR, "Can't consolidate %o", path);
+ }
+ else if (Obj_Is_A(folder, COMPOUNDFILEREADER)) {
+ THROW(ERR, "Can't consolidate %o twice", path);
+ }
+ else {
+ CompoundFileWriter *cf_writer = CFWriter_new(folder);
+ CFWriter_Consolidate(cf_writer);
+ DECREF(cf_writer);
+ {
+ ZombieCharBuf name_zcb = ZCB_BLANK;
+ ZombieCharBuf *name = IxFileNames_local_part(path, &name_zcb);
+ CompoundFileReader *cf_reader = CFReader_open(folder);
+ if (!cf_reader) { RETHROW(INCREF(Err_get_error())); }
+ Hash_Store(enclosing_folder->entries, (Obj*)name,
+ (Obj*)cf_reader);
+ }
+ }
+}
+
static Folder*
S_enclosing_folder(Folder *self, ZombieCharBuf *path)
{

Modified: lucene/lucy/trunk/core/Lucy/Store/RAMFolder.c
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Store/RAMFolder.c?rev=884885&r1=884884&r2=884885&view=diff
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Store/RAMFolder.c (original)
+++ lucene/lucy/trunk/core/Lucy/Store/RAMFolder.c Fri Nov 27 14:47:39 2009
@@ -2,6 +2,7 @@
#include "Lucy/Util/ToolSet.h"

#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Store/CompoundFileReader.h"
#include "Lucy/Store/InStream.h"
#include "Lucy/Store/OutStream.h"
#include "Lucy/Store/RAMDirHandle.h"
@@ -149,16 +150,12 @@
}

/* Extract RAMFolders from compound reader wrappers, if necessary. */
- inner_from_folder = (RAMFolder*)from_folder;
- inner_to_folder = (RAMFolder*)to_folder;
- /*
inner_from_folder = Obj_Is_A(from_folder, COMPOUNDFILEREADER)
? (RAMFolder*)CFReader_Get_Real_Folder(from_folder)
: (RAMFolder*)from_folder;
inner_to_folder = Obj_Is_A(to_folder, COMPOUNDFILEREADER)
? (RAMFolder*)CFReader_Get_Real_Folder(to_folder)
: (RAMFolder*)to_folder;
- */
if (!Obj_Is_A(inner_from_folder, RAMFOLDER)) {
Err_set_error(Err_new(CB_newf("Not a RAMFolder, but a '%o'",
Obj_Get_Class_Name(inner_from_folder))));
@@ -173,15 +170,12 @@
/* Find the original element. */
elem = Hash_Fetch(inner_from_folder->entries, (Obj*)from_name);
if (!elem) {
- if (0) { }
- /*
if ( Obj_Is_A(from_folder, COMPOUNDFILEREADER)
&& CFReader_Local_Exists(from_folder, (CharBuf*)from_name)
) {
Err_set_error(Err_new(CB_newf("Source file '%o' is virtual",
from)));
}
- */
else {
Err_set_error(Err_new(CB_newf("File not found: '%o'", from)));
}
@@ -294,7 +288,10 @@
;
}
else if (Obj_Is_A(entry, FOLDER)) {
- RAMFolder *inner_folder = (RAMFolder*)CERTIFY(entry, RAMFOLDER);
+ RAMFolder *inner_folder = Obj_Is_A(entry, COMPOUNDFILEREADER)
+ ? (RAMFolder*)CERTIFY(
+ CFReader_Get_Real_Folder(entry), RAMFOLDER)
+ : (RAMFolder*)CERTIFY(entry, RAMFOLDER);
if (Hash_Get_Size(inner_folder->entries)) {
/* Can't delete non-empty dir. */
return false;

Added: lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.bp
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.bp?rev=884885&view=auto
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.bp (added)
+++ lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.bp Fri Nov 27 14:47:39 2009
@@ -0,0 +1,23 @@
+parcel Lucy;
+
+inert class Lucy::Test::Store::TestCompoundFileReader
+ cnick TestCFReader {
+ inert void
+ run_tests();
+}
+
+/* Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+

Propchange: lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.bp
------------------------------------------------------------------------------
svn:eol-style = native

Added: lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.c
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.c?rev=884885&view=auto
==============================================================================
--- lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.c (added)
+++ lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.c Fri Nov 27 14:47:39 2009
@@ -0,0 +1,355 @@
+#define C_LUCY_RAMFOLDER
+#include "Lucy/Util/ToolSet.h"
+
+#include "Lucy/Test.h"
+#include "Lucy/Test/Store/TestCompoundFileReader.h"
+#include "Lucy/Store/CompoundFileReader.h"
+#include "Lucy/Store/CompoundFileWriter.h"
+#include "Lucy/Store/FileHandle.h"
+#include "Lucy/Store/InStream.h"
+#include "Lucy/Store/OutStream.h"
+#include "Lucy/Store/RAMFolder.h"
+#include "Lucy/Util/Json.h"
+
+static ZombieCharBuf cfmeta_file = ZCB_LITERAL("cfmeta.json");
+static ZombieCharBuf cf_file = ZCB_LITERAL("cf.dat");
+static ZombieCharBuf foo = ZCB_LITERAL("foo");
+static ZombieCharBuf bar = ZCB_LITERAL("bar");
+static ZombieCharBuf baz = ZCB_LITERAL("baz");
+static ZombieCharBuf stuff = ZCB_LITERAL("stuff");
+
+static Folder*
+S_folder_with_contents()
+{
+ ZombieCharBuf seg_1 = ZCB_LITERAL("seg_1");
+ RAMFolder *folder = RAMFolder_new((CharBuf*)&seg_1);
+ OutStream *foo_out = RAMFolder_Open_Out(folder, (CharBuf*)&foo);
+ OutStream *bar_out = RAMFolder_Open_Out(folder, (CharBuf*)&bar);
+ OutStream_Write_Bytes(foo_out, "foo", 3);
+ OutStream_Write_Bytes(bar_out, "bar", 3);
+ OutStream_Close(foo_out);
+ OutStream_Close(bar_out);
+ DECREF(foo_out);
+ DECREF(bar_out);
+ Folder_Consolidate(folder, (CharBuf*)&EMPTY);
+ return (Folder*)folder;
+}
+
+static void
+test_open(TestBatch *batch)
+{
+ Folder *real_folder;
+ CompoundFileReader *cf_reader;
+ Hash *metadata;
+
+ Err_set_error(NULL);
+ real_folder = S_folder_with_contents();
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ ASSERT_TRUE(batch, cf_reader == NULL,
+ "Return NULL when cfmeta file missing");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when cfmeta file missing");
+ DECREF(real_folder);
+
+ Err_set_error(NULL);
+ real_folder = S_folder_with_contents();
+ Folder_Delete(real_folder, (CharBuf*)&cf_file);
+ cf_reader = CFReader_open(real_folder);
+ ASSERT_TRUE(batch, cf_reader == NULL,
+ "Return NULL when cf.dat file missing");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when cf.dat file missing");
+ DECREF(real_folder);
+
+ Err_set_error(NULL);
+ real_folder = S_folder_with_contents();
+ metadata = (Hash*)Json_slurp_json(real_folder, (CharBuf*)&cfmeta_file);
+ Hash_Store_Str(metadata, "format", 6, (Obj*)CB_newf("%i32", -1));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ ASSERT_TRUE(batch, cf_reader == NULL,
+ "Return NULL when format is invalid");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when format is invalid");
+
+ Err_set_error(NULL);
+ Hash_Store_Str(metadata, "format", 6, (Obj*)CB_newf("%i32", 1000));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ ASSERT_TRUE(batch, cf_reader == NULL,
+ "Return NULL when format is too recent");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when format too recent");
+
+ Err_set_error(NULL);
+ DECREF(Hash_Delete_Str(metadata, "format", 6));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ ASSERT_TRUE(batch, cf_reader == NULL,
+ "Return NULL when format key is missing");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when format key is missing");
+
+ Hash_Store_Str(metadata, "format", 6, (Obj*)CB_newf("%i32",
+ CFWriter_current_file_format));
+ DECREF(Hash_Delete_Str(metadata, "files", 5));
+ Folder_Delete(real_folder, (CharBuf*)&cfmeta_file);
+ Json_spew_json((Obj*)metadata, real_folder, (CharBuf*)&cfmeta_file);
+ cf_reader = CFReader_open(real_folder);
+ ASSERT_TRUE(batch, cf_reader == NULL,
+ "Return NULL when files key is missing");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Set Err_error when files key is missing");
+
+ DECREF(metadata);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_MkDir_and_Find_Folder(TestBatch *batch)
+{
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+
+ ASSERT_FALSE(batch,
+ Folder_Local_Is_Directory(cf_reader, (CharBuf*)&stuff),
+ "Local_Is_Directory returns false for non-existent entry");
+
+ ASSERT_TRUE(batch, CFReader_MkDir(cf_reader, (CharBuf*)&stuff),
+ "MkDir returns true");
+ ASSERT_TRUE(batch,
+ Folder_Find_Folder(real_folder, (CharBuf*)&stuff) != NULL,
+ "Local_MkDir pass-through");
+ ASSERT_TRUE(batch,
+ Folder_Find_Folder(real_folder, (CharBuf*)&stuff) ==
+ Folder_Find_Folder(cf_reader, (CharBuf*)&stuff),
+ "Local_Find_Folder pass-through");
+ ASSERT_TRUE(batch,
+ Folder_Local_Is_Directory(cf_reader, (CharBuf*)&stuff),
+ "Local_Is_Directory pass through");
+
+ Err_set_error(NULL);
+ ASSERT_FALSE(batch, CFReader_MkDir(cf_reader, (CharBuf*)&stuff),
+ "MkDir returns false when dir already exists");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "MkDir sets Err_error when dir already exists");
+
+ Err_set_error(NULL);
+ ASSERT_FALSE(batch, CFReader_MkDir(cf_reader, (CharBuf*)&foo),
+ "MkDir returns false when virtual file exists");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "MkDir sets Err_error when virtual file exists");
+
+ ASSERT_TRUE(batch,
+ CFReader_Find_Folder(cf_reader, (CharBuf*)&foo) == NULL,
+ "Virtual file not reported as directory");
+ ASSERT_FALSE(batch, Folder_Local_Is_Directory(cf_reader, (CharBuf*)&foo),
+ "Local_Is_Directory returns false for virtual file");
+
+ DECREF(real_folder);
+ DECREF(cf_reader);
+}
+
+static void
+test_Local_Delete_and_Exists(TestBatch *batch)
+{
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+
+ CFReader_MkDir(cf_reader, (CharBuf*)&stuff);
+ ASSERT_TRUE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&stuff),
+ "pass through for Local_Exists");
+ ASSERT_TRUE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&foo),
+ "Local_Exists returns true for virtual file");
+
+ ASSERT_TRUE(batch,
+ CFReader_Local_Exists(cf_reader, (CharBuf*)&cfmeta_file),
+ "cfmeta file exists");
+
+ ASSERT_TRUE(batch, CFReader_Local_Delete(cf_reader, (CharBuf*)&stuff),
+ "Local_Delete returns true when zapping real entity");
+ ASSERT_FALSE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&stuff),
+ "Local_Exists returns false after real entity zapped");
+
+ ASSERT_TRUE(batch, CFReader_Local_Delete(cf_reader, (CharBuf*)&foo),
+ "Local_Delete returns true when zapping virtual file");
+ ASSERT_FALSE(batch, CFReader_Local_Exists(cf_reader, (CharBuf*)&foo),
+ "Local_Exists returns false after virtual file zapped");
+
+ ASSERT_TRUE(batch, CFReader_Local_Delete(cf_reader, (CharBuf*)&bar),
+ "Local_Delete returns true when zapping last virtual file");
+ ASSERT_FALSE(batch,
+ CFReader_Local_Exists(cf_reader, (CharBuf*)&cfmeta_file),
+ "cfmeta file deleted when last virtual file deleted");
+ ASSERT_FALSE(batch,
+ CFReader_Local_Exists(cf_reader, (CharBuf*)&cf_file),
+ "compound data file deleted when last virtual file deleted");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_Open_Dir(TestBatch *batch)
+{
+
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+ DirHandle *dh;
+ CharBuf *entry;
+ bool_t saw_foo = false;
+ bool_t saw_stuff = false;
+ bool_t stuff_was_dir = false;
+
+ CFReader_MkDir(cf_reader, (CharBuf*)&stuff);
+
+ dh = CFReader_Local_Open_Dir(cf_reader);
+ entry = DH_Get_Entry(dh);
+ while (DH_Next(dh)) {
+ if (CB_Equals(entry, (Obj*)&foo)) {
+ saw_foo = true;
+ }
+ else if (CB_Equals(entry, (Obj*)&stuff)) {
+ saw_stuff = true;
+ stuff_was_dir = DH_Entry_Is_Dir(dh);
+ }
+ }
+
+ ASSERT_TRUE(batch, saw_foo, "DirHandle iterated over virtual file");
+ ASSERT_TRUE(batch, saw_stuff, "DirHandle iterated over real directory");
+ ASSERT_TRUE(batch, stuff_was_dir,
+ "DirHandle knew that real entry was dir");
+
+ DECREF(dh);
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_Open_FileHandle(TestBatch *batch)
+{
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+ FileHandle *fh;
+
+ {
+ OutStream *outstream = Folder_Open_Out(cf_reader, (CharBuf*)&baz);
+ OutStream_Write_Bytes(outstream, "baz", 3);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ }
+
+ fh = CFReader_Local_Open_FileHandle(cf_reader, (CharBuf*)&baz,
+ FH_READ_ONLY);
+ ASSERT_TRUE(batch, fh != NULL,
+ "Local_Open_FileHandle pass-through for real file");
+ DECREF(fh);
+
+ Err_set_error(NULL);
+ fh = CFReader_Local_Open_FileHandle(cf_reader, (CharBuf*)&stuff,
+ FH_READ_ONLY);
+ ASSERT_TRUE(batch, fh == NULL,
+ "Local_Open_FileHandle for non-existent file returns NULL");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Local_Open_FileHandle for non-existent file sets Err_error");
+
+ Err_set_error(NULL);
+ fh = CFReader_Local_Open_FileHandle(cf_reader, (CharBuf*)&foo,
+ FH_READ_ONLY);
+ ASSERT_TRUE(batch, fh == NULL,
+ "Local_Open_FileHandle for virtual file returns NULL");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Local_Open_FileHandle for virtual file sets Err_error");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Local_Open_In(TestBatch *batch)
+{
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+ InStream *instream;
+
+ instream = CFReader_Local_Open_In(cf_reader, (CharBuf*)&foo);
+ ASSERT_TRUE(batch, instream != NULL,
+ "Local_Open_In for virtual file");
+ ASSERT_TRUE(batch, CB_Starts_With(InStream_Get_Filename(instream),
+ Folder_Get_Path(cf_reader)), "InStream's path includes directory");
+ DECREF(instream);
+
+ {
+ OutStream *outstream = Folder_Open_Out(cf_reader, (CharBuf*)&baz);
+ OutStream_Write_Bytes(outstream, "baz", 3);
+ OutStream_Close(outstream);
+ DECREF(outstream);
+ instream = CFReader_Local_Open_In(cf_reader, (CharBuf*)&baz);
+ ASSERT_TRUE(batch, instream != NULL,
+ "Local_Open_In pass-through for real file");
+ DECREF(instream);
+ }
+
+ Err_set_error(NULL);
+ instream = CFReader_Local_Open_In(cf_reader, (CharBuf*)&stuff);
+ ASSERT_TRUE(batch, instream == NULL,
+ "Local_Open_In for non-existent file returns NULL");
+ ASSERT_TRUE(batch, Err_get_error() != NULL,
+ "Local_Open_In for non-existent file sets Err_error");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+static void
+test_Close(TestBatch *batch)
+{
+ Folder *real_folder = S_folder_with_contents();
+ CompoundFileReader *cf_reader = CFReader_open(real_folder);
+
+ CFReader_Close(cf_reader);
+ PASS(batch, "Close completes without incident");
+
+ CFReader_Close(cf_reader);
+ PASS(batch, "Calling Close() multiple times is ok");
+
+ DECREF(cf_reader);
+ DECREF(real_folder);
+}
+
+void
+TestCFReader_run_tests()
+{
+ TestBatch *batch = Test_new_batch("TestCompoundFileReader", 48, NULL);
+
+ PLAN(batch);
+ test_open(batch);
+ test_Local_MkDir_and_Find_Folder(batch);
+ test_Local_Delete_and_Exists(batch);
+ test_Local_Open_Dir(batch);
+ test_Local_Open_FileHandle(batch);
+ test_Local_Open_In(batch);
+ test_Close(batch);
+
+ batch->destroy(batch);
+}
+
+/* Copyright 2009 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+

Propchange: lucene/lucy/trunk/core/Lucy/Test/Store/TestCompoundFileReader.c
------------------------------------------------------------------------------
svn:eol-style = native

Modified: lucene/lucy/trunk/perl/lib/Lucy/Test.pm
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/perl/lib/Lucy/Test.pm?rev=884885&r1=884884&r2=884885&view=diff
==============================================================================
--- lucene/lucy/trunk/perl/lib/Lucy/Test.pm (original)
+++ lucene/lucy/trunk/perl/lib/Lucy/Test.pm Fri Nov 27 14:47:39 2009
@@ -37,6 +37,9 @@
lucy_TestVArray_run_tests();
}
/* Lucy::Store */
+ else if (strEQ(package, "TestCompoundFileReader")) {
+ lucy_TestCFReader_run_tests();
+ }
else if (strEQ(package, "TestCompoundFileWriter")) {
lucy_TestCFWriter_run_tests();
}

Added: lucene/lucy/trunk/perl/t/core/113-cf_reader.t
URL: http://svn.apache.org/viewvc/lucene/lucy/trunk/perl/t/core/113-cf_reader.t?rev=884885&view=auto
==============================================================================
--- lucene/lucy/trunk/perl/t/core/113-cf_reader.t (added)
+++ lucene/lucy/trunk/perl/t/core/113-cf_reader.t Fri Nov 27 14:47:39 2009
@@ -0,0 +1,6 @@
+use strict;
+use warnings;
+
+use Lucy::Test;
+Lucy::Test::run_tests("TestCompoundFileReader");
+

Propchange: lucene/lucy/trunk/perl/t/core/113-cf_reader.t
------------------------------------------------------------------------------
svn:eol-style = native
reply

Search Discussions

Related Discussions

Discussion Navigation
viewthread | post

1 user in discussion

Marvin: 1 post