Thread: [Getdata-commits] SF.net SVN: getdata:[854] trunk/getdata (Page 8)

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Revision: 854
          http://sourceforge.net/p/getdata/code/854
Author:   ketiltrout
Date:     2013-10-22 20:55:24 +0000 (Tue, 22 Oct 2013)
Log Message:
-----------
Merge OOP write and temp file code in the encoding bookkeeping layer.

Modified Paths:
--------------
    trunk/getdata/ChangeLog
    trunk/getdata/src/add.c
    trunk/getdata/src/encoding.c
    trunk/getdata/src/fpos.c
    trunk/getdata/src/getdata.c
    trunk/getdata/src/mod.c
    trunk/getdata/src/move.c
    trunk/getdata/src/parse.c
    trunk/getdata/src/putdata.c
    trunk/getdata/test/Makefile.am

Added Paths:
-----------
    trunk/getdata/test/gzip_put_endian.c

Property Changed:
----------------
    trunk/getdata/test/

Modified: trunk/getdata/ChangeLog
===================================================================

--- trunk/getdata/ChangeLog	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/ChangeLog	2013-10-22 20:55:24 UTC (rev 854)
@@ -1,3 +1,12 @@
+2013-10-19   D. V. Wiebe <ge...@ke...> svn:854
+	* src/add.c (_GD_Add) src/fpos.c (_GD_GetFilePos _GD_WriteSeek _GD_Seek)
+	src/getdata.c (_GD_DoRaw) src/mod.c (_GD_Change) src/move.c
+	(_GD_MogrifyFile) src/parse.c (_GD_ParseFieldSpec) src/putdata.c
+	(_GD_DoRawOut): Compute the swap argument correctly in _GD_InitRawIO calls.
+	* src/encoding.c (_GD_InitRawIO): Merge oop_write and temp file creation
+	code; also, assume that the caller has figured out the swap parameter.
+	* test/gzip_put_endian.c: Added.
+
 2013-10-19   D. V. Wiebe <ge...@ke...> svn:852
 	
 	_GD_MakeTempFile patch from Seth:

Modified: trunk/getdata/src/add.c
===================================================================
--- trunk/getdata/src/add.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/add.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -301,7 +301,8 @@
             GD_SIZE(E->EN(raw,data_type))) == 0)
         _GD_SetError(D, GD_E_BAD_TYPE, entry->EN(raw,data_type), NULL, 0, NULL);
       else if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0,
-            GD_FILE_WRITE | GD_FILE_TOUCH, 0))
+            GD_FILE_WRITE | GD_FILE_TOUCH,
+            _GD_FileSwapBytes(D, E->fragment_index)))
       {
         ;
       } else if (D->fragment[E->fragment_index].ref_name == NULL) {

Modified: trunk/getdata/src/encoding.c
===================================================================
--- trunk/getdata/src/encoding.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/encoding.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -1,4 +1,4 @@
-/* Copyright (C) 2008-2012 D. V. Wiebe
+/* Copyright (C) 2008-2013 D. V. Wiebe
  *
  ***************************************************************************
  *
@@ -486,7 +486,6 @@
     int fragment, const struct encoding_t *enc, unsigned int funcs,
     unsigned int mode, int swap)
 {
-  int temp_fd = -1;
   const int touch = mode & GD_FILE_TOUCH;
   int oop_write = 0;
 
@@ -538,59 +537,41 @@
   if (fragment == -1)
     fragment = E->fragment_index;
 
-  if (mode & GD_FILE_TEMP) {
+  if (oop_write || mode & GD_FILE_TEMP) {
     /* create temporary file in file[1] */
-    if ((*enc->name)(D, (const char*)D->fragment[E->fragment_index].enc_data,
+    if ((*enc->name)(D, (const char*)D->fragment[fragment].enc_data,
           E->e->u.raw.file + 1, filebase, 1, 0))
     {
       ; /* error already set */
-    } else if ((*enc->open)(D->fragment[fragment].dirfd, E->e->u.raw.file + 1, swap,
-          GD_FILE_WRITE | GD_FILE_TEMP))
+      dreturn("%i", 1);
+      return 1;
+    } else if ((*enc->open)(D->fragment[fragment].dirfd, E->e->u.raw.file + 1,
+          swap, GD_FILE_WRITE | GD_FILE_TEMP))
     {
       _GD_SetError(D, GD_E_RAW_IO, 0, E->e->u.raw.file[1].name, errno, NULL);
-    }
-
-    if (D->error) {
       dreturn("%i", 1);
       return 1;
     }
-    dreturn("%i", 0);
-    return 0;
-  }
 
-  if (oop_write) {
-    /* an out-of-place write requires us to open a temporary file and pass
-     * in its fd */
-    if ((*enc->name)(D, (const char*)D->fragment[E->fragment_index].enc_data,
-          E->e->u.raw.file + 1, filebase, 1, 0))
-    {
-      dreturn("%i", 1);
-      return 1;
-    } else if ((temp_fd = _GD_MakeTempFile(D,
-            D->fragment[E->fragment_index].dirfd, E->e->u.raw.file[1].name))
-          < 0)
-    {
-      dreturn("%i", 1);
-      return 1;
-    } else if ((*enc->open)(temp_fd, E->e->u.raw.file + 1, _GD_FileSwapBytes(D,
-            E->fragment_index), GD_FILE_WRITE)) {
-      _GD_SetError(D, GD_E_RAW_IO, 0, E->e->u.raw.file[0].name, errno, NULL);
-      dreturn("%i", 1);
-      return 1;
+    if (oop_write) {
+      /* The read file in OOP mode is flagged as RW. */
+      mode = GD_FILE_RDWR;
+    } else {
+      /* Temp file creation complete */
+      dreturn("%i", 0);
+      return 0;
     }
-    /* The read file in OOP mode is flagged as RW. */
-    mode = GD_FILE_RDWR;
   }
 
   /* open a regular file, if necessary */
   if (E->e->u.raw.file[0].idata < 0) {
-    if ((*enc->name)(D, (const char*)D->fragment[E->fragment_index].enc_data,
+    if ((*enc->name)(D, (const char*)D->fragment[fragment].enc_data,
           E->e->u.raw.file, filebase, 0, 0))
     {
       dreturn("%i", 1);
       return 1;
-    } else if ((*enc->open)(D->fragment[E->fragment_index].dirfd,
-          E->e->u.raw.file, _GD_FileSwapBytes(D, E->fragment_index), mode))
+    } else if ((*enc->open)(D->fragment[fragment].dirfd, E->e->u.raw.file, swap,
+          mode))
     {
       /* In oop_write mode, it doesn't matter if the old file doesn't exist */
       if (!oop_write || errno != ENOENT) {
@@ -603,7 +584,7 @@
   }
 
   if (touch)
-    _GD_FiniRawIO(D, E, E->fragment_index, GD_FINIRAW_KEEP);
+    _GD_FiniRawIO(D, E, fragment, GD_FINIRAW_KEEP);
 
   dreturn("%i", 0);
   return 0;

Modified: trunk/getdata/src/fpos.c
===================================================================
--- trunk/getdata/src/fpos.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/fpos.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -1,4 +1,4 @@
-/* Copyright (C) 2011-2012 D. V. Wiebe
+/* Copyright (C) 2011-2013 D. V. Wiebe
  *
  ***************************************************************************
  *
@@ -38,8 +38,11 @@
     case GD_RAW_ENTRY:
       /* We must open the file to know its starting offset */
       if (E->e->u.raw.file[0].idata < 0)
-        if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_READ, 0))
+        if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_READ,
+              _GD_FileSwapBytes(D, E->fragment_index)))
+        {
           break;
+        }
       pos = E->e->u.raw.file[0].pos + E->EN(raw,spf) *
         D->fragment[E->fragment_index].frame_offset;
       break;
@@ -171,7 +174,8 @@
     if (_GD_FiniRawIO(D, E, E->fragment_index, GD_FINIRAW_KEEP)) {
       dreturn("%i", -1);
       return -1;
-    } else if (_GD_InitRawIO(D, E, NULL, 0, NULL, GD_EF_SEEK, GD_FILE_WRITE, 0))
+    } else if (_GD_InitRawIO(D, E, NULL, 0, NULL, GD_EF_SEEK, GD_FILE_WRITE,
+          _GD_FileSwapBytes(D, E->fragment_index)))
     {
       dreturn("%i", -1);
       return -1;
@@ -250,8 +254,11 @@
   switch (E->field_type) {
     case GD_RAW_ENTRY:
       /* open/create the file, if necessary */
-      if (_GD_InitRawIO(D, E, NULL, 0, NULL, GD_EF_SEEK, mode, 0))
+      if (_GD_InitRawIO(D, E, NULL, 0, NULL, GD_EF_SEEK, mode,
+            _GD_FileSwapBytes(D, E->fragment_index)))
+      {
         break;
+      }
 
       /* The requested offset is before the start of the file, so I guess
        * pretend we've repositioned it...

Modified: trunk/getdata/src/getdata.c
===================================================================
--- trunk/getdata/src/getdata.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/getdata.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -286,7 +286,7 @@
   if (ns > 0) {
     /** open the file (and cache the fp) if it hasn't been opened yet. */
     if (_GD_InitRawIO(D, E, NULL, 0, NULL, GD_EF_SEEK | GD_EF_READ,
-          GD_FILE_READ, 0))
+          GD_FILE_READ, _GD_FileSwapBytes(D, E->fragment_index)))
     {
       free(databuffer);
       dreturn("%i", 0);

Modified: trunk/getdata/src/mod.c
===================================================================
--- trunk/getdata/src/mod.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/mod.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -323,9 +323,11 @@
         enc = gd_ef_ + E->e->u.raw.file[0].subenc;
 
         /* open the old file */
-        if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_READ, 0))
+        if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_READ,
+              _GD_FileSwapBytes(D, E->fragment_index)))
+        {
           break;
-        else if ((*enc->seek)(E->e->u.raw.file, 0, E->EN(raw,data_type),
+        } else if ((*enc->seek)(E->e->u.raw.file, 0, E->EN(raw,data_type),
               GD_FILE_READ) == -1)
         {
           _GD_SetError(D, GD_E_RAW_IO, 0, E->e->u.raw.file[0].name, errno,

Modified: trunk/getdata/src/move.c
===================================================================
--- trunk/getdata/src/move.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/move.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -1,4 +1,4 @@
-/* Copyright (C) 2008-2011 D. V. Wiebe
+/* Copyright (C) 2008-2013 D. V. Wiebe
  *
  ***************************************************************************
  *
@@ -138,7 +138,9 @@
   }
 
   /* Open the input file, if necessary */
-  if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_READ, 0)) {
+  if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_READ,
+        _GD_FileSwapBytes(D, E->fragment_index)))
+  {
     free(new_filebase);
     dreturn("%i", -1);
     return -1;

Modified: trunk/getdata/src/parse.c
===================================================================
--- trunk/getdata/src/parse.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/parse.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -1545,7 +1545,8 @@
         /* If the encoding scheme is unsupported, we can't add the field */
         _GD_SetError(D, GD_E_UNSUPPORTED, 0, NULL, 0, NULL);
       else
-        _GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_WRITE | GD_FILE_TOUCH, 0);
+        _GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_WRITE | GD_FILE_TOUCH,
+            _GD_FileSwapBytes(D, E->fragment_index));
     }
 
     /* Is this the first raw field ever defined? */

Modified: trunk/getdata/src/putdata.c
===================================================================
--- trunk/getdata/src/putdata.c	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/src/putdata.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -1,6 +1,6 @@
 /* Copyright (C) 2003-2005 C. Barth Netterfield
  * Copyright (C) 2003-2005 Theodore Kisner
- * Copyright (C) 2005-2012 D. V. Wiebe
+ * Copyright (C) 2005-2013 D. V. Wiebe
  *
  ***************************************************************************
  *
@@ -93,7 +93,9 @@
   }
   /* write data to file. */
 
-  if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_WRITE, 0)) {
+  if (_GD_InitRawIO(D, E, NULL, 0, NULL, 0, GD_FILE_WRITE,
+        _GD_FileSwapBytes(D, E->fragment_index)))
+  {
     free(databuffer);
     dreturn("%i", 0);
     return 0;

Index: trunk/getdata/test
===================================================================
--- trunk/getdata/test	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/test	2013-10-22 20:55:24 UTC (rev 854)

Property changes on: trunk/getdata/test
___________________________________________________________________
Modified: svn:ignore
## -485,6 +485,7 ##
 gzip_move_to
 gzip_nframes
 gzip_put
+gzip_put_endian
 gzip_put_get
 header_complex
 hide
Modified: trunk/getdata/test/Makefile.am
===================================================================
--- trunk/getdata/test/Makefile.am	2013-10-19 03:12:50 UTC (rev 853)
+++ trunk/getdata/test/Makefile.am	2013-10-22 20:55:24 UTC (rev 854)
@@ -190,7 +190,8 @@
 GLOBAL_TESTS=global_flags global_name global_ref global_ref_empty global_ref_set
 
 GZIP_TESTS=gzip_add gzip_get gzip_get_get gzip_get_get2 gzip_get_put \
-					 gzip_move_from gzip_move_to gzip_nframes gzip_put gzip_put_get
+					 gzip_move_from gzip_move_to gzip_nframes gzip_put gzip_put_endian \
+					 gzip_put_get
 
 HEADER_TESTS=header_complex
 

Added: trunk/getdata/test/gzip_put_endian.c
===================================================================
--- trunk/getdata/test/gzip_put_endian.c	                        (rev 0)
+++ trunk/getdata/test/gzip_put_endian.c	2013-10-22 20:55:24 UTC (rev 854)
@@ -0,0 +1,125 @@
+/* Copyright (C) 2013 D. V. Wiebe
+ *
+ ***************************************************************************
+ *
+ * This file is part of the GetData project.
+ *
+ * GetData is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU Lesser General Public License as published by the
+ * Free Software Foundation; either version 2.1 of the License, or (at your
+ * option) any later version.
+ *
+ * GetData is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public
+ * License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with GetData; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ */
+#include "test.h"
+
+#include <inttypes.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <string.h>
+#include <errno.h>
+#include <stdio.h>
+
+int main(void)
+{
+#ifndef TEST_GZIP
+  return 77;
+#else
+  const char *filedir = "dirfile";
+  const char *format = "dirfile/format";
+  const char *data_gz = "dirfile/data.gz";
+  const char *data = "dirfile/data";
+  const char *format_data = "data RAW UINT16 8\n"
+#ifdef WORDS_BIGENDIAN
+  "ENDIAN little\n";
+#else
+  "ENDIAN big\n";
+#endif
+  uint16_t c[8];
+#ifdef USE_GZIP
+  char command[4096];
+  uint16_t d;
+#endif
+  struct stat buf;
+  int fd, i, n, error, stat_data, unlink_data, r = 0;
+  DIRFILE *D;
+
+  memset(c, 0, 8);
+  rmdirfile();
+  mkdir(filedir, 0777);
+
+  for (i = 0; i < 8; ++i)
+    c[i] = (uint16_t)(0x102 * i);
+
+  fd = open(format, O_CREAT | O_EXCL | O_WRONLY, 0666);
+  write(fd, format_data, strlen(format_data));
+  close(fd);
+
+#ifdef USE_GZIP
+  D = gd_open(filedir, GD_RDWR | GD_GZIP_ENCODED | GD_VERBOSE);
+#else
+  D = gd_open(filedir, GD_RDWR | GD_GZIP_ENCODED);
+#endif
+  n = gd_putdata(D, "data", 5, 0, 1, 0, GD_UINT16, c);
+  error = gd_error(D);
+
+  gd_discard(D);
+
+  stat_data = stat(data_gz, &buf);
+#ifdef USE_GZIP
+  if (stat_data) {
+    perror("stat");
+  }
+  CHECKI(stat_data, 0);
+#else
+  CHECKI(stat_data, -1);
+#endif
+
+#ifdef USE_GZIP
+  /* uncompress */
+  snprintf(command, 4096, "%s -f %s > /dev/null", GUNZIP, data);
+  if (gd_system(command)) {
+    r = 1;
+  } else {
+    fd = open(data, O_RDONLY | O_BINARY);
+    if (fd >= 0) {
+      i = 0;
+      while (read(fd, &d, sizeof(uint16_t))) {
+        if (i < 40 || i > 48) {
+          CHECKIi(i, d, 0);
+        } else
+          CHECKIi(i, d, 0x201 * (i - 40));
+        i++;
+      }
+      CHECKI(i, 48);
+      close(fd);
+    }
+  }
+#endif
+
+  unlink_data = unlink(data);
+  unlink(format);
+  rmdir(filedir);
+
+#ifdef USE_GZIP
+  CHECKI(unlink_data, 0);
+  CHECKI(error, GD_E_OK);
+  CHECKI(n, 8);
+#else
+  CHECKI(unlink_data, -1);
+  CHECKI(error, GD_E_UNSUPPORTED);
+  CHECKI(n, 0);
+#endif
+
+  return r;
+#endif
+}

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





Thread: [Getdata-commits] SF.net SVN: getdata:[854] trunk/getdata (Page 8)

Scientific Database Format

getdata-commits