Menu
▾
▴
[Sc2-cvs] SF.net SVN: sc2:[3086] trunk/sc2
|
From: <Mee...@us...> - 2008-10-12 21:47:06
|
Revision: 3086
http://sc2.svn.sourceforge.net/sc2/?rev=3086&view=rev
Author: Meep-Eep
Date: 2008-10-12 21:46:57 +0000 (Sun, 12 Oct 2008)
Log Message:
-----------
Added read-ahead buffering when reading zip index files.
Modified Paths:
--------------
trunk/sc2/ChangeLog
trunk/sc2/src/sc2code/libs/uio/fileblock.c
trunk/sc2/src/sc2code/libs/uio/fileblock.h
trunk/sc2/src/sc2code/libs/uio/zip/zip.c
Modified: trunk/sc2/ChangeLog
===================================================================
--- trunk/sc2/ChangeLog 2008-10-12 01:34:39 UTC (rev 3085)
+++ trunk/sc2/ChangeLog 2008-10-12 21:46:57 UTC (rev 3086)
@@ -1,4 +1,5 @@
Changes towards version 0.7:
+- Added read-ahead buffering when reading zip index files. - SvdB
- Added support for packed ani and font files - Mika
- DrawTracedText abstraction (bug #1029), from Nic
- Experimental support for Symbian S60 3rd edition - Mika & SvdB
Modified: trunk/sc2/src/sc2code/libs/uio/fileblock.c
===================================================================
--- trunk/sc2/src/sc2code/libs/uio/fileblock.c 2008-10-12 01:34:39 UTC (rev 3085)
+++ trunk/sc2/src/sc2code/libs/uio/fileblock.c 2008-10-12 21:46:57 UTC (rev 3086)
@@ -18,6 +18,8 @@
*
*/
+#define uio_INTERNAL_FILEBLOCK
+
#include "iointrn.h"
#include "fileblock.h"
#include "uioport.h"
@@ -25,11 +27,42 @@
#include <errno.h>
static uio_FileBlock *uio_FileBlock_new(uio_Handle *handle, int flags,
- off_t offset, size_t blockSize, char *buffer, size_t bufSize);
+ off_t offset, size_t blockSize, char *buffer, size_t bufSize,
+ off_t bufOffset, size_t bufFill, size_t readAheadBufSize);
static inline uio_FileBlock *uio_FileBlock_alloc(void);
static void uio_FileBlock_free(uio_FileBlock *block);
+// caller should uio_Handle_ref(handle) (unless it doesn't need it's own
+// reference anymore).
+static uio_FileBlock *
+uio_FileBlock_new(uio_Handle *handle, int flags, off_t offset,
+ size_t blockSize, char *buffer, size_t bufSize, off_t bufOffset,
+ size_t bufFill, size_t readAheadBufSize) {
+ uio_FileBlock *result;
+
+ result = uio_FileBlock_alloc();
+ result->handle = handle;
+ result->flags = flags;
+ result->offset = offset;
+ result->blockSize = blockSize;
+ result->buffer = buffer;
+ result->bufSize = bufSize;
+ result->bufOffset = bufOffset;
+ result->bufFill = bufFill;
+ result->readAheadBufSize = readAheadBufSize;
+ return result;
+}
+static inline uio_FileBlock *
+uio_FileBlock_alloc(void) {
+ return uio_malloc(sizeof (uio_FileBlock));
+}
+
+static inline void
+uio_FileBlock_free(uio_FileBlock *block) {
+ uio_free(block);
+}
+
uio_FileBlock *
uio_openFileBlock(uio_Handle *handle) {
// TODO: if mmap support is available, and it is available natively
@@ -44,7 +77,7 @@
return NULL;
}
uio_Handle_ref(handle);
- return uio_FileBlock_new(handle, 0, 0, statBuf.st_size, NULL, 0);
+ return uio_FileBlock_new(handle, 0, 0, statBuf.st_size, NULL, 0, 0, 0, 0);
}
uio_FileBlock *
@@ -58,52 +91,153 @@
// errno is set
return NULL;
}
- if (statBuf.st_size >
+ if (statBuf.st_size > offset || (statBuf.st_size - offset > size)) {
+ // NOT: 'if (statBuf.st_size > offset + size)', to protect
+ // against overflow.
+
+ }
#endif
uio_Handle_ref(handle);
- return uio_FileBlock_new(handle, 0, offset, size, NULL, 0);
+ return uio_FileBlock_new(handle, 0, offset, size, NULL, 0, 0, 0, 0);
}
+static inline ssize_t
+uio_accessFileBlockMmap(uio_FileBlock *block, off_t offset, size_t length,
+ char **buffer) {
+ // TODO
+ errno = ENOSYS;
+ (void) block;
+ (void) offset;
+ (void) length;
+ (void) buffer;
+ return -1;
+}
+
+static inline ssize_t
+uio_accessFileBlockNoMmap(uio_FileBlock *block, off_t offset, size_t length,
+ char **buffer) {
+
+ // TODO: add read-ahead buffering
+ ssize_t numRead;
+ off_t start;
+ off_t end;
+ size_t bufSize;
+ char *oldBuffer;
+ //size_t oldBufSize;
+
+ // Don't go beyond the end of the block.
+ if (offset > (off_t) block->blockSize) {
+ *buffer = block->buffer;
+ return 0;
+ }
+ if (length > block->blockSize - offset)
+ length = block->blockSize - offset;
+
+ if (block->buffer != NULL) {
+ // Check whether the requested data is already in the buffer.
+ if (offset >= block->bufOffset &&
+ (offset - block->bufOffset) + length < block->bufFill) {
+ *buffer = block->buffer + (offset - block->bufOffset);
+ return length;
+ }
+ }
+
+ if (length < block->readAheadBufSize &&
+ (block->flags & uio_FB_USAGE_MASK) != 0) {
+ // We can buffer more data.
+ switch (block->flags & uio_FB_USAGE_MASK) {
+ case uio_FB_USAGE_FORWARD:
+ // Read extra data after the requested data.
+ start = offset;
+ end = (block->readAheadBufSize > block->blockSize - offset) ?
+ block->blockSize : offset + block->readAheadBufSize;
+ break;
+ case uio_FB_USAGE_BACKWARD:
+ // Read extra data before the requested data.
+ end = offset + length;
+ start = (end <= (off_t) block->blockSize) ?
+ 0 : end - block->bufSize;
+ break;
+ case uio_FB_USAGE_FORWARD | uio_FB_USAGE_BACKWARD: {
+ // Read extra data both before and after the requested data.
+ off_t extraBefore = (block->readAheadBufSize - length) / 2;
+ start = (offset < extraBefore) ? 0 : offset - extraBefore;
+
+ end = (block->readAheadBufSize > block->blockSize - start) ?
+ block->blockSize : start + block->readAheadBufSize;
+ break;
+ }
+ }
+ } else {
+ start = offset;
+ end = offset + length;
+ }
+ bufSize = (length > block->readAheadBufSize) ?
+ length : block->readAheadBufSize;
+
+ // Start contains the start index in the block of the data we're going
+ // to read.
+ // End contains the end index.
+ // bufSize contains the size of the buffer. bufSize >= end - start.
+
+ oldBuffer = block->buffer;
+ //oldBufSize = block->bufSize;
+ if (block->buffer != NULL || block->bufSize != bufSize) {
+ // We don't have a buffer, or we have one, but of the wrong size.
+ block->buffer = uio_malloc(bufSize);
+ block->bufSize = bufSize;
+ }
+
+ if (oldBuffer != NULL) {
+ // TODO: If we have part of the data still in the old buffer, we
+ // can keep that.
+ // memmove(...)
+
+ if (oldBuffer != block->buffer)
+ uio_free(oldBuffer);
+ }
+ block->bufFill = 0;
+ block->bufOffset = start;
+
+ // TODO: lock handle
+ if (uio_lseek(block->handle, block->offset + start, SEEK_SET) ==
+ (off_t) -1) {
+ // errno is set
+ return -1;
+ }
+
+ numRead = uio_read(block->handle, block->buffer, end - start);
+ if (numRead == -1) {
+ // errno is set
+ // TODO: unlock handle
+ return -1;
+ }
+ // TODO: unlock handle
+
+ block->bufFill = numRead;
+ *buffer = block->buffer + (offset - block->bufOffset);
+ if (numRead <= (offset - block->bufOffset))
+ return 0;
+ if ((size_t) numRead >= length)
+ return length;
+ return numRead - offset;
+}
+
// block remains usable until the next call to uio_accessFileBlock
// with the same block as argument, or until uio_closeFileBlock with
// that block as argument.
+// The 'offset' parameter is wrt. the start of the block.
+// Requesting access to data beyond the file block is not an error. The
+// FileBlock is meant to be used as a replacement of seek() and read(), and
+// as with those functions, trying to go beyond the end of a file just
+// goes to the end. The return value is the number of bytes in the buffer.
ssize_t
uio_accessFileBlock(uio_FileBlock *block, off_t offset, size_t length,
char **buffer) {
if (block->flags & uio_FB_USE_MMAP) {
- // TODO
- errno = ENOSYS;
- return -1;
+ return uio_accessFileBlockMmap(block, offset, length, buffer);
} else {
- // TODO: add read-ahead buffering
- ssize_t numRead;
-
- if (length > block->blockSize - offset)
- length = block->blockSize - offset;
-
- if (block->buffer != NULL && length != block->bufSize) {
- uio_free(block->buffer);
- block->buffer = NULL;
- }
- if (block->buffer == NULL)
- block->buffer = uio_malloc(length);
-
- // TODO: lock handle
- if (uio_lseek(block->handle, block->offset + offset, SEEK_SET) ==
- (off_t) -1) {
- // errno is set
- return -1;
- }
-
- numRead = uio_read(block->handle, block->buffer, length);
- if (numRead == -1) {
- // errno is set
- // TODO: unlock handle
- return -1;
- }
- // TODO: unlock handle
- *buffer = block->buffer;
- return numRead;
+ return uio_accessFileBlockNoMmap(block, offset, length, buffer);
}
}
@@ -115,10 +249,31 @@
errno = ENOSYS;
return -1;
} else {
- ssize_t numRead;
+ ssize_t numCopied = 0;
+ ssize_t readResult;
+ // Don't go beyond the end of the block.
+ if (offset > (off_t) block->blockSize)
+ return 0;
if (length > block->blockSize - offset)
length = block->blockSize - offset;
+
+ // Check whether (part of) the requested data is already in our
+ // own buffer.
+ if (block->buffer != NULL && offset >= block->bufOffset
+ && offset < block->bufOffset + (off_t) block->bufFill) {
+ size_t toCopy = block->bufFill - offset;
+ if (toCopy > length)
+ toCopy = length;
+ memcpy(buffer, block->buffer + (offset - block->bufOffset),
+ toCopy);
+ numCopied += toCopy;
+ length -= toCopy;
+ if (length == 0)
+ return numCopied;
+ buffer += toCopy;
+ offset += toCopy;
+ }
// TODO: lock handle
if (uio_lseek(block->handle, block->offset + offset, SEEK_SET) ==
@@ -127,14 +282,15 @@
return -1;
}
- numRead = uio_read(block->handle, buffer, length);
- if (numRead == -1) {
+ readResult = uio_read(block->handle, buffer, length);
+ // TODO: unlock handle
+ if (readResult == -1) {
// errno is set
- // TODO: unlock handle
return -1;
}
- // TODO: unlock handle
- return numRead;
+ numCopied += readResult;
+
+ return numCopied;
}
}
@@ -154,31 +310,25 @@
return 0;
}
-// caller should uio_Handle_ref(handle) (unless it doesn't need it's own
-// reference anymore).
-static uio_FileBlock *
-uio_FileBlock_new(uio_Handle *handle, int flags, off_t offset,
- size_t blockSize, char *buffer, size_t bufSize) {
- uio_FileBlock *result;
-
- result = uio_FileBlock_alloc();
- result->handle = handle;
- result->flags = flags;
- result->offset = offset;
- result->blockSize = blockSize;
- result->buffer = buffer;
- result->bufSize = bufSize;
- return result;
+// Usage is the or'ed value of zero or more of uio_FB_USAGE_FORWARD,
+// and uio_FB_USAGE_BACKWARD.
+void
+uio_setFileBlockUsageHint(uio_FileBlock *block, int usage,
+ size_t readAheadBufSize) {
+ block->flags = (block->flags & ~uio_FB_USAGE_MASK) |
+ (usage & uio_FB_USAGE_MASK);
+ block->readAheadBufSize = readAheadBufSize;
}
-static inline uio_FileBlock *
-uio_FileBlock_alloc(void) {
- return uio_malloc(sizeof (uio_FileBlock));
+// Call if you want the memory used by the fileblock to be released, but
+// still want to use the fileblock later. If you don't need the fileblock,
+// call uio_closeFileBlock() instead.
+void
+uio_clearFileBlockBuffers(uio_FileBlock *block) {
+ if (block->buffer != NULL) {
+ uio_free(block->buffer);
+ block->buffer = NULL;
+ }
}
-static void
-uio_FileBlock_free(uio_FileBlock *block) {
- uio_free(block);
-}
-
Modified: trunk/sc2/src/sc2code/libs/uio/fileblock.h
===================================================================
--- trunk/sc2/src/sc2code/libs/uio/fileblock.h 2008-10-12 01:34:39 UTC (rev 3085)
+++ trunk/sc2/src/sc2code/libs/uio/fileblock.h 2008-10-12 21:46:57 UTC (rev 3086)
@@ -21,23 +21,55 @@
#ifndef _FILEBLOCK_H
#define _FILEBLOCK_H
+typedef struct uio_FileBlock uio_FileBlock;
+
#include "io.h"
#include "uioport.h"
#include <sys/types.h>
-typedef struct uio_FileBlock {
+#define uio_FB_USAGE_FORWARD 1
+#define uio_FB_USAGE_BACKWARD 2
+#define uio_FB_USAGE_MASK (uio_FB_USAGE_FORWARD | uio_FB_USAGE_BACKWARD)
+
+#ifdef uio_INTERNAL_FILEBLOCK
+
+// A fileblock represents a contiguous block of data from a file.
+// It's purpose is to avoid needless copying of data, while enabling
+// buffering.
+
+struct uio_FileBlock {
uio_Handle *handle;
int flags;
-#define uio_FB_USE_MMAP 1
+ // See above for uio_FB_USAGE_FORWARD, uio_FB_USAGE_BACKWARD.
+#define uio_FB_USE_MMAP 4
off_t offset;
// Offset to the start of the block in the file.
size_t blockSize;
+ // Size of the block of data represented by this FileBlock.
char *buffer;
// either allocated buffer, or buffer to mmap'ed area.
size_t bufSize;
-} uio_FileBlock;
+ // Size of the buffer.
+ off_t bufOffset;
+ // Offset of the start of the buffer into the block.
+ size_t bufFill;
+ // Part of 'buffer' which is in use.
+ size_t readAheadBufSize;
+ // Try to read up to this many bytes at a time, even when less
+ // is immediately needed.
+};
+// INV: The FileBlock represents 'fileData[offset..(offset + blockSize - 1)]'
+// where 'fileData' is the contents of the file.
+// INV: If buf != NULL then:
+// bufFill <= bufSize
+// bufFill <= blockSize
+// buffer[0..bufFill - 1] == fileData[
+// (offset + bufOffset)..(offset + bufOffset + bufFill - 1)]
+
+#endif /* uio_INTERNAL_FILEBLOCK */
+
uio_FileBlock *uio_openFileBlock(uio_Handle *handle);
uio_FileBlock *uio_openFileBlock2(uio_Handle *handle, off_t offset,
size_t size);
@@ -46,6 +78,10 @@
int uio_copyFileBlock(uio_FileBlock *block, off_t offset, char *buffer,
size_t length);
int uio_closeFileBlock(uio_FileBlock *block);
+#define uio_FB_READAHEAD_BUFSIZE_MAX ((size_t) -1)
+void uio_setFileBlockUsageHint(uio_FileBlock *block, int usage,
+ size_t readAheadBufSize);
+void uio_clearFileBlockBuffers(uio_FileBlock *block);
#endif /* _FILEBLOCK_H */
Modified: trunk/sc2/src/sc2code/libs/uio/zip/zip.c
===================================================================
--- trunk/sc2/src/sc2code/libs/uio/zip/zip.c 2008-10-12 01:34:39 UTC (rev 3085)
+++ trunk/sc2/src/sc2code/libs/uio/zip/zip.c 2008-10-12 21:46:57 UTC (rev 3086)
@@ -43,6 +43,8 @@
#endif
+#define DIR_STRUCTURE_READ_BUFSIZE 0x10000
+
static int zip_badFile(zip_GPFileData *gPFileData, char *fileName);
static int zip_fillDirStructure(uio_GPDir *top, uio_Handle *handle);
#if zip_USE_HEADERS == zip_USE_LOCAL_HEADERS
@@ -583,7 +585,7 @@
int savedErrno = errno;
#ifdef DEBUG
fprintf(stderr, "Error: failed to read the zip directory "
- "structure - %d.\n", errno);
+ "structure - %s.\n", strerror(errno));
#endif
uio_GPRoot_umount(result);
errno = savedErrno;
@@ -647,7 +649,11 @@
}
startCentralDir = makeUInt32(buf[16], buf[17], buf[18], buf[19]);
-
+
+ // Enable read-ahead buffering, for speed.
+ uio_setFileBlockUsageHint(fileBlock, uio_FB_USAGE_FORWARD,
+ DIR_STRUCTURE_READ_BUFSIZE);
+
pos = startCentralDir;
while (numEntries--) {
if (zip_fillDirStructureCentralProcessEntry(top, fileBlock, &pos)
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|