[Getdata-commits] SF.net SVN: getdata:[942] trunk/defile

SourceForge Headquarters 1320 Columbia Street Suite 310 San Diego, CA 92101 +1 (858) 422-6466

Revision: 942
          http://sourceforge.net/p/getdata/code/942
Author:   ketiltrout
Date:     2015-02-20 19:39:20 +0000 (Fri, 20 Feb 2015)
Log Message:
-----------
Disentangle appending vs. resuming

Modified Paths:
--------------
    trunk/defile/bin/defile.c
    trunk/defile/bin/net.c
    trunk/defile/doc/defile.1.in
    trunk/defile/lib/defile.h
    trunk/defile/lib/internal.h
    trunk/defile/lib/libdefile.c

Modified: trunk/defile/bin/defile.c
===================================================================

--- trunk/defile/bin/defile.c	2015-02-20 16:50:05 UTC (rev 941)
+++ trunk/defile/bin/defile.c	2015-02-20 19:39:20 UTC (rev 942)
@@ -945,7 +945,7 @@
   DF_WOPT_NOLISTEN, DF_WOPT_RCMD, DF_WOPT_RDEBUG, DF_WOPT_NORDEBUG,
   DF_WOPT_SRVRATE, DF_WOPT_DAEMON, DF_WOPT_CURFILE, DF_WOPT_SYMLINK,
   DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE, DF_WOPT_CLOBBERONE, DF_WOPT_FOFFS,
-  DF_WOPT_NFRAMES, DF_WOPT_ALLFRAMES
+  DF_WOPT_NFRAMES, DF_WOPT_ALLFRAMES, DF_WOPT_RESUME
 };
 
 static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
@@ -1072,6 +1072,7 @@
 
 
         SETMODE(APPEND, DF_MODE_APPEND);
+        SETMODE(RESUME, DF_MODE_APPEND | DF_MODE_RESUME);
         CLRMODE(NOAPPEND, DF_MODE_APPEND);
 
         SETULONG(BHEAD, bufhead);
@@ -1097,7 +1098,7 @@
         SETLINT(QUIET, quiet, 1);
         SETLINT(VERBOSE, quiet, 0);
 
-        SETULLONG(SKIP, offset);
+        SETULLONG(SKIP, skip);
 
         SETMODE(ASYNC, DF_MODE_ASYNC);
         CLRMODE(SYNC, DF_MODE_ASYNC);
@@ -1251,11 +1252,10 @@
   if (df->rateout == 2)
     putchar('\n');
   if (df->nframes_in > 0)
-    printf("R:[%llu of %llu] W:[%llu] %.*f %cHz   \r", nr + df->offset,
-        df->nframes_in, nw + df->offset,
+    printf("R:[%llu of %llu] W:[%llu] %.*f %cHz   \r", nr, df->nframes_in, nw,
         (drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si);
   else
-    printf("R:[%llu] W:[%llu] %.*f %cHz   \r", nr + df->offset, nw + df->offset,
+    printf("R:[%llu] W:[%llu] %.*f %cHz   \r", nr, nw,
         (drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si);
   fflush(stdout);
 
@@ -1303,7 +1303,7 @@
   return 0;
 }
 
-static int DF_CreateOutput(int clobber, int append)
+static int DF_CreateOutput(int clobber, int append, int resume)
 {
   int new = 1;
 
@@ -1329,10 +1329,11 @@
       return 1;
     }
   } else if (gd_error(df->D) == GD_E_OK) {
+    /* dirfile doesn't exist.  Turn off append mode */
     pthread_mutex_lock(&df_config_mx);
-    df->mode_flags &= ~DF_MODE_APPEND;
+    df->mode_flags &= ~(DF_MODE_APPEND | DF_MODE_RESUME);
     pthread_mutex_unlock(&df_config_mx);
-    append = 0;
+    append = resume = 0;
   }
 
   if (gd_error(df->D)) {
@@ -1347,17 +1348,20 @@
   gd_flags(df->D, GD_VERBOSE, 0);
   gd_verbose_prefix(df->D, "defile: ");
 
+  /* append mode offset */
   if (append) {
     long long nf = gd_nframes(df->D);
     pthread_mutex_lock(&df_config_mx);
-    if (df->offset < nf)
-      df->offset = nf;
+    df->output_offset = nf;
     pthread_mutex_unlock(&df_config_mx);
-  }
+    if (resume)
+      df->skip = nf;
+  } else
+    df->output_offset = 0;
 
   if (!(df->mode_flags & DF_MODE_SERVER))
-    df_dprintf(DF_PRN_INFO, "%s %s\n", append ? "appending to" :
-        new ? "created" : "overwriting", df->dirfile);
+    df_dprintf(DF_PRN_INFO, "%s %s\n", resume ? "resuming" :
+        append ? "appending to" : new ? "created" : "overwriting", df->dirfile);
 
 
   return 0;
@@ -1600,8 +1604,11 @@
   if (!df->dirfile)
     return 1;
 
-  if (DF_CreateOutput(df->clobber, df_mode() & DF_MODE_APPEND))
+  if (DF_CreateOutput(df->clobber, df_mode() & DF_MODE_APPEND,
+        df_mode() & DF_MODE_RESUME))
+  {
     return 1;
+  }
 
   if (df->output_state == DF_ST_ABORT)
     return 1;
@@ -1922,7 +1929,7 @@
               df_rpc_push_uint8(&data, DF_RPC_FRAME);
 
               /* frame number */
-              df_rpc_push_uint64(&data, df->nwrote + df->offset);
+              df_rpc_push_uint64(&data, df->nwrote + df->skip);
 
               /* buffer some frames */
               if (nf_out + df->srv_base > df->buflen)
@@ -2424,12 +2431,6 @@
   return h;
 }
 
-static void DF_ShutdownLTDL(void)
-{
-  lt_dlclose(df->inhandle);
-  lt_dlexit();
-}
-
 void df_discard_metadata(int reset)
 {
   int i;
@@ -2502,6 +2503,9 @@
     { DF_WOPT_NOAPPEND, DF_OPT_NO_ARG, 'A', "no-append", "NoAppend", NULL,
       "don't append data (default)"
     },
+    { DF_WOPT_RESUME, DF_OPT_NO_ARG, 'r', "resume", "Resume", NULL,
+      "resume an interrupted conversion (implies --append)"
+    },
     { DF_WOPT_ASYNC, DF_OPT_NO_ARG, 'a', "async", "Asynchronous",
       NULL, "turn on asynchronous mode"
     },
@@ -2691,7 +2695,6 @@
 
   /* initialise LTDL and register its cleanup function */
   lt_dlinit();
-  atexit(DF_ShutdownLTDL);
 
   /* initialise and parse command line */
   df->ofl = df_string_subset(DF_SUBSET_FULL);

Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c	2015-02-20 16:50:05 UTC (rev 941)
+++ trunk/defile/bin/net.c	2015-02-20 19:39:20 UTC (rev 942)
@@ -1172,8 +1172,7 @@
     + ((df->mode_flags & DF_MODE_RDEBUG_O) ? sizeof(" -d o") : 0)
     + ((df->mode_flags & DF_MODE_RDEBUG_T) ? sizeof(" -d t") : 0)
     + (!(df->mode_flags & DF_MODE_FOLLOW) ? sizeof(" +f") : 0)
-    + ((df->offset > 0) ? sizeof(" -s ") + 1
-        + (int)(log(df->offset) * M_LOG10E) : 0)
+    + ((df->skip > 0) ? sizeof(" -s ") + 1 + (int)(log(df->skip) * M_LOG10E) : 0)
     + (df->type ? sizeof(" -t ") + strlen(df->type) : 0)
     ;
   if (df->iargc > 1) {
@@ -1197,8 +1196,8 @@
     wordlen += sprintf(cmd[nwords] + wordlen, " -d t");
   if (!(df->mode_flags & DF_MODE_FOLLOW))
     wordlen += sprintf(cmd[nwords] + wordlen, " +f");
-  if (df->offset)
-    wordlen += sprintf(cmd[nwords] + wordlen, " -s %lli", df->offset);
+  if (df->skip)
+    wordlen += sprintf(cmd[nwords] + wordlen, " -s %lli", df->skip);
   if (df->type)
     wordlen += sprintf(cmd[nwords] + wordlen, " -t %s", df->type);
 

Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in	2015-02-20 16:50:05 UTC (rev 941)
+++ trunk/defile/doc/defile.1.in	2015-02-20 19:39:20 UTC (rev 942)
@@ -180,9 +180,9 @@
 has not been specified, and the input plugin provides a new name, a new output
 dirfile will be created in the directory specified with
 .BR \-\-directory ,
-if possible.  If you want to clobber a pre-existing, explicitly named output
-dirfile, but not allow the input plugin to cause defile to overwrite its output,
-use
+if possible.  If you want to overwrite a pre-existing dirfile when defile first
+starts up, but not allow the input plugin to subsequently cause defile to
+overwrite its newly-created output, use
 .BR \-\-clobber\-once .
 
 .TP
@@ -302,29 +302,16 @@
 .BR \-s ", " \-\-skip=\fINUM
 Skip the first
 .I NUM
-frames of the input; equivalently: request the input start with the frame
-numbered
+frames of the input.  Even when this option is specified, defile may start on a
+later frame, if the first frame available in the input is greater than
 .IR NUM .
-Even when this option is specified, defile may start on a later frame, if the
-first frame available in the input is greater than
-.IR NUM .
 Only the last
 .B \-\-skip
 specified is honoured.  The default skip is zero (i.e. start with the first
-frame).
-
-Using
-.B \-\-append
-(q.v.) to append to a pre-existing output dirfile has the effect of specifying
-an implicit
-.B \-\-skip
-with
-.I NUM
-equal to the number of frames currently in the output dirfile.  A
-.B \-\-skip
-which specifies a smaller number is ignored, but this option can be used to
-advance the starting frame number further (leaving a hole in the outpu
-dirfile). (Directive:
+available frame).  In some cases, this option also affect the starting frame
+number of the output dirfile; see
+.B \-\-offset\-adjust
+below. (Directive:
 .BR Skip )
 
 .TP
@@ -365,7 +352,10 @@
 adds 100 to the default frame offset.  The default is, effectively, "+0" (i.e.,
 no adjustment).  In
 .I Append Mode\fR,
-an adjustment is relative to the current end of the output dirfile.  (Directive:
+an adjustment is relative to the current end of the output dirfile, and is
+unaffected by a
+.B \-\-skip
+option.  (Directive:
 .BR OutputAdjust )
 
 .TP
@@ -423,16 +413,23 @@
 .TP
 .BR \-A ", " \-\-append
 .I Append Mode on\fR:
-if the output dirfile already exists, append data to it.  Unless a
-.B \-\-skip
-follows to set the starting frame number, this option will ask the input to
-start on the frame following the last frame in the existing output, making it
-suitable for resuming an interrupted conversion.
+if the output dirfile already exists, append data to it.  See also
+.BR \-\-resume ,
+which provides a convenient way to use append mode to resume an interrupted
+conversion.  To adjust the frame number where the appended data start, see
+.BR \-\-offset\-adjust .
 .TP
 .BR +A ", " \-\-no\-append
 .I Append Mode off\fR:
 don't append data to an existing dirfile.  This is the default mode.
 .TP
+.BR \-r ", " \-\-resume
+Equivalent to
+.B \-\-append \-\-skip=\fIN
+where
+.I N
+is the current number of frames in the output dirfile.
+.TP
 .BR \-d ", " \-\-debug [\fB=\fIWHAT\fR]
 .I Debug Mode on\fR:
 write extra debugging messages to standard error.  This is primarily useful

Modified: trunk/defile/lib/defile.h
===================================================================
--- trunk/defile/lib/defile.h	2015-02-20 16:50:05 UTC (rev 941)
+++ trunk/defile/lib/defile.h	2015-02-20 19:39:20 UTC (rev 942)
@@ -84,12 +84,12 @@
 };
 
 /* mode flags */
-#define DF_MODE_APPEND  0x001 /* append mode */
-#define DF_MODE_ASYNC   0x002 /* asynchronous writing */
-#define DF_MODE_DEBUG   0x004 /* input plugin debug mode */
-#define DF_MODE_FOLLOW  0x008 /* follow mode */
-#define DF_MODE_SERVER  0x010 /* server mode */
-/*                0x020-0x800 are reserved */
+#define DF_MODE_APPEND  0x0001 /* append mode */
+#define DF_MODE_ASYNC   0x0002 /* asynchronous writing */
+#define DF_MODE_DEBUG   0x0004 /* input plugin debug mode */
+#define DF_MODE_FOLLOW  0x0008 /* follow mode */
+#define DF_MODE_SERVER  0x0010 /* server mode */
+/*                0x020-0x1000 are reserved */
 
 
 /* defile configuration */

Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h	2015-02-20 16:50:05 UTC (rev 941)
+++ trunk/defile/lib/internal.h	2015-02-20 19:39:20 UTC (rev 942)
@@ -125,13 +125,14 @@
 
 /* private mode flags */
 #define DF_MODE_DEBUG_I   DF_MODE_DEBUG /* alias for consistency */
-#define DF_MODE_DEBUG_O   0x020 /* output debug */
-#define DF_MODE_DEBUG_T   0x040 /* transfer debug */
-#define DF_MODE_RDEBUG_I  0x080 /* remote input debug */
-#define DF_MODE_RDEBUG_O  0x100 /* remote output debug */
-#define DF_MODE_RDEBUG_T  0x200 /* transfer debug */
-#define DF_MODE_CLOBBER   0x400 /* overwrite the output dirfile */
-#define DF_MODE_NO_DAEMON 0x800 /* don't daemonise servers */
+#define DF_MODE_DEBUG_O   0x0020 /* output debug */
+#define DF_MODE_DEBUG_T   0x0040 /* transfer debug */
+#define DF_MODE_RDEBUG_I  0x0080 /* remote input debug */
+#define DF_MODE_RDEBUG_O  0x0100 /* remote output debug */
+#define DF_MODE_RDEBUG_T  0x0200 /* transfer debug */
+#define DF_MODE_CLOBBER   0x0400 /* overwrite the output dirfile */
+#define DF_MODE_NO_DAEMON 0x0800 /* don't daemonise servers */
+#define DF_MODE_RESUME    0x1000 /* resume mode */
 
 /* long long integer conversion */
 #ifndef HAVE_STRTOULL
@@ -295,9 +296,10 @@
 
   /* operations */
   struct df_config config;
-  long long offset;
-  long long input_offset;
-  long long output_adjust;
+  long long output_offset; /* EOF for append mode, zero otherwise */
+  long long skip; /* specified by --skip or --resume */
+  long long input_offset; /* specified by input plugin */
+  long long output_adjust; /* specified by --output-adjust */
   long long nframes_out;
   int output_adjust_set;
   unsigned long buflen;

Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c	2015-02-20 16:50:05 UTC (rev 941)
+++ trunk/defile/lib/libdefile.c	2015-02-20 19:39:20 UTC (rev 942)
@@ -1229,9 +1229,13 @@
 
     /* handle a starting offset */
     pthread_mutex_lock(&df_config_mx);
-    if (df->input_offset > df->offset)
-      df->offset = df->input_offset;
-    offset = df->offset;
+    if (df->output_offset > 0)
+      offset = df->output_offset;
+    else {
+      if (df->input_offset > df->skip)
+        df->skip = df->input_offset;
+      offset = df->skip;
+    }
     pthread_mutex_unlock(&df_config_mx);
 
     /* output frameoffset */
@@ -1477,7 +1481,7 @@
 
   /* restart */
   pthread_mutex_lock(&df_config_mx);
-  df->input_offset = df->offset = 0;
+  df->input_offset = df->skip = 0;
   df->mode_flags &= ~DF_MODE_APPEND;
   if (df->clobber > 1) /* --clobber-once */
     df->clobber = 0;
@@ -2113,7 +2117,7 @@
   DF_CHECK_SEQUENCE2(DF_ST_BUILD, DF_ST_RUN);
 
   pthread_mutex_lock(&df_config_mx);
-  offset = (df->input_offset > df->offset) ? df->input_offset : df->offset;
+  offset = (df->input_offset > df->skip) ? df->input_offset : df->skip;
   pthread_mutex_unlock(&df_config_mx);
 
   return offset;

This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.





[Getdata-commits] SF.net SVN: getdata:[942] trunk/defile

Scientific Database Format

[Getdata-commits] SF.net SVN: getdata:[942] trunk/defile