[Getdata-commits] SF.net SVN: getdata:[924] trunk/defile
Scientific Database Format
Brought to you by:
ketiltrout
Menu
▾
▴
[Getdata-commits] SF.net SVN: getdata:[924] trunk/defile
|
From: <ket...@us...> - 2014-10-01 00:04:56
|
Revision: 924
http://sourceforge.net/p/getdata/code/924
Author: ketiltrout
Date: 2014-10-01 00:04:49 +0000 (Wed, 01 Oct 2014)
Log Message:
-----------
--num-frames; plus network client bugfix.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/bin/net.c
trunk/defile/doc/defile.1.in
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2014-09-30 02:40:40 UTC (rev 923)
+++ trunk/defile/bin/defile.c 2014-10-01 00:04:49 UTC (rev 924)
@@ -122,6 +122,7 @@
static struct df_defile the_defile = {
.buflen = DF_BUFLEN,
.iargc = 1, /* like argv, iargv[0] is the program name */
+ .nframes_out = -1, /* write all frames */
.srv_rate = 10 /* megabits per second */
};
struct df_defile *const df = &the_defile;
@@ -481,7 +482,7 @@
if (str == NULL)
return df_rpc_push_uint32(slug, 0xFFFFFFFF);
- /* yeah, that's right: we silently limit strings to 4Gb */
+ /* yeah, that's right: we silently limit strings to 4GB */
len = (uint32_t)strlen(str);
if (len == 0xFFFFFFFF)
len--;
@@ -943,7 +944,8 @@
DF_WOPT_VERBOSE, DF_WOPT_REMOTE, DF_WOPT_LOCAL, DF_WOPT_RSH, DF_WOPT_LISTEN,
DF_WOPT_NOLISTEN, DF_WOPT_RCMD, DF_WOPT_RDEBUG, DF_WOPT_NORDEBUG,
DF_WOPT_SRVRATE, DF_WOPT_DAEMON, DF_WOPT_CURFILE, DF_WOPT_SYMLINK,
- DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE, DF_WOPT_CLOBBERONE, DF_WOPT_FOFFS
+ DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE, DF_WOPT_CLOBBERONE, DF_WOPT_FOFFS,
+ DF_WOPT_NFRAMES, DF_WOPT_ALLFRAMES
};
static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
@@ -1048,7 +1050,6 @@
df->mode_flags &=
~(DF_MODE_RDEBUG_I | DF_MODE_RDEBUG_O | DF_MODE_RDEBUG_T);
break;
-
case DF_WOPT_FOFFS: /* --output-offset */
/* a sign means relative adjustment */
if (config->argument[0] != '-' && config->argument[0] != '+')
@@ -1057,6 +1058,14 @@
df->output_adjust_set = 0;
df->output_adjust = strtoll(config->argument, NULL, 0);
break;
+ case DF_WOPT_ALLFRAMES: /* --all-frames */
+ df->nframes_out = -1;
+ break;
+ case DF_WOPT_NFRAMES: /* --num-frames */
+ df->nframes_out = strtoll(config->argument, NULL, 0);
+ if (df->nframes_out < 0)
+ df->nframes_out = 0;
+ break;
@@ -1241,9 +1250,9 @@
}
if (df->rateout == 2)
putchar('\n');
- if (df->nframes > 0)
+ if (df->nframes_in > 0)
printf("R:[%llu of %llu] W:[%llu] %.*f %cHz \r", nr + df->offset,
- df->nframes, nw + df->offset,
+ df->nframes_in, nw + df->offset,
(drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si);
else
printf("R:[%llu] W:[%llu] %.*f %cHz \r", nr + df->offset, nw + df->offset,
@@ -2161,7 +2170,7 @@
/* loop through raws, writing ones that have full buffer heads */
while (!last_pass) {
wait = 1;
- if (finalise || df->input_state != DF_ST_RUN)
+ if (finalise || df->input_state != DF_ST_RUN || df->nraw_active == 0)
last_pass = 1;
for (i = 0; i < df->nraw; ++i) {
@@ -2182,6 +2191,10 @@
nf = (r->in == out) ? df->buflen - 1 :
((r->in + r->buflen - out) % r->buflen) / r->framesize - 1;
+ /* deal with nframes_out */
+ if (r->nframes_out >= 0 && nf > r->nframes_out)
+ nf = r->nframes_out;
+
if (nf == 0 || (!last_pass && nf <= df->bufhead)) {
pthread_mutex_unlock(&r->mx);
continue;
@@ -2241,6 +2254,14 @@
} else
r->ahead += nf;
+ /* update nframes_out */
+ if (r->nframes_out > 0) {
+ r->nframes_out -= nf;
+
+ if (r->nframes_out == 0)
+ df->nraw_active--;
+ }
+
if (nwrote > 0) {
last_pass = 0;
wait = 0;
@@ -2314,7 +2335,11 @@
DF_OutputFini(0);
/* done */
- DF_OUTSTATE_MX(DONE);
+ if (df->nframes_out >= 0 && df->nraw_active == 0)
+ /* Force input to abort. This is inelegant but seems to work. */
+ DF_OUTSTATE_MX(ERROR);
+ else
+ DF_OUTSTATE_MX(DONE);
return NULL;
}
@@ -2543,6 +2568,13 @@
NULL,
"ignore all previous options which modifiy the ouput field list (-F, +F)"
},
+ { DF_WOPT_NFRAMES, DF_OPT_ARG_RQ, 'n', "num-frames", "NumFrames",
+ "NUM", "write at most NUM frames to the output dirfile"
+ },
+ { DF_WOPT_ALLFRAMES, DF_OPT_PLUS | DF_OPT_NO_ARG, 'n', "all-frames",
+ "AllFrames", NULL,
+ "write all available frames to the output dirfile (default)"
+ },
{ DF_WOPT_OUTPUT, DF_OPT_EXPAND | DF_OPT_ARG_RQ, 'o', "output", "Output",
"PATH", "write data to a dirfile called PATH"
},
Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c 2014-09-30 02:40:40 UTC (rev 923)
+++ trunk/defile/bin/net.c 2014-10-01 00:04:49 UTC (rev 924)
@@ -211,7 +211,7 @@
static void df_rpc_get_cdouble(double v[2], int sock)
{
uint64_t u[2];
- if (df_rpc_get_mem(sock, sizeof(v), u, NULL))
+ if (df_rpc_get_mem(sock, 2 * sizeof(u[0]), u, NULL))
return;
u[0] = df_ntohll(u[0]);
@@ -317,7 +317,7 @@
if (ioerr == DFIO_NO_DATA) return 0; \
if (ioerr) return 1; \
pos++; \
- remaining -= sizeof(2 * sizeof(double)); \
+ remaining -= 2 * sizeof(double); \
} while(0)
#define DF_GET_STRING(n) DF_GET_SDATUM(string[n])
@@ -521,7 +521,7 @@
}
if (pos > 3) {
lastpos = E.u.lincom.n_fields * 7 + 3;
- for (i = 0; i < E.u.lincom.n_fields; ++i)
+ for (i = 0; i < E.u.lincom.n_fields; ++i)
switch ((pos - 4) % 7) {
case 0:
DF_GET_SDATUM(E.in_fields[i]);
@@ -1430,13 +1430,27 @@
}
}
client_ioerr:
- if (ioerr == DFIO_HANG_UP) {
- df_dprintf(DF_PRN_NOTE, "connection to server closed.\n");
- break;
- } else if (ioerr)
- goto client_drop;
+ switch (ioerr) {
+ case DFIO_HANG_UP:
+ df_dprintf(DF_PRN_NOTE, "connection to server closed.\n");
+ goto client_hup;
+ case DFIO_READ_ERR:
+ df_dprintf(DF_PRN_ERR, "read error from server\n");
+ goto client_drop;
+ case DFIO_NO_DATA:
+ df_dprintf(DF_PRN_ERR, "missing data from server\n");
+ goto client_drop;
+ case DFIO_PROTO_ERR:
+ df_dprintf(DF_PRN_ERR, "protocol error from server\n");
+ goto client_drop;
+ case DFIO_CALL_ERR:
+ df_dprintf(DF_PRN_ERR, "bad request from server\n");
+ goto client_drop;
+ }
}
+ /* normal clean up */
+client_hup:
close(sock);
free(iobuf);
DF_INSTATE_MX(DONE);
Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in 2014-09-30 02:40:40 UTC (rev 923)
+++ trunk/defile/doc/defile.1.in 2014-10-01 00:04:49 UTC (rev 924)
@@ -296,7 +296,8 @@
.BR NoCurFile )
.SS Frame Selection Options
-The following options affect the starting frame number for input and output.
+The following options affect the starting frame number for input and output as
+well as the maximum number of frames output.
.TP
.BR \-s ", " \-\-skip=\fINUM
Skip the first
@@ -367,6 +368,32 @@
this option is ignored. (Directive:
.BR OutputAdjust )
+.TP
+.BR \-n ", " \-\-num\-frames=\fINUM
+Write at most
+.I NUM
+frames to the output dirfile. After writing that many frames, defile will exit,
+regardless of
+.I Follow Mode\fR,
+even if more data are available from the input. Fewer frames may be written if
+the input ends before
+.I NUM
+frames have been produced. In
+.I Append Mode\fR,
+the number of frames already in the output dirfile is not considered: that is,
+defile will attempt to add
+.I NUM
+additional frames to the dirfile. Ignored in
+.I Server Mode\fR.
+(Directive:
+.BR NumFrames )
+
+.TP
+.BR +n ", " \-\-all\-frames
+Write all available data to the output dirfile. This is the default behaviour.
+(Directive:
+.BR AllFrames )
+
.SS Running Mode Options
The following options affect the modes in which defile runs.
.TP
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2014-09-30 02:40:40 UTC (rev 923)
+++ trunk/defile/lib/internal.h 2014-10-01 00:04:49 UTC (rev 924)
@@ -208,6 +208,7 @@
unsigned long in, out; /* read/write pointers */
size_t framesize; /* GD_SIZE(type) * spf */
unsigned long buflen; /* r->framesize * df->buflen */
+ long long nframes_out; /* number of frames left to write */
gd_type_t type; /* data type */
unsigned int spf; /* samples-per-frame */
unsigned long ahead; /* difference between the number of frames written from
@@ -289,7 +290,7 @@
double rate, tc;
long long wpartial, rpartial;
long long nwrote, nread;
- long long nframes;
+ long long nframes_in;
double srv_rate; /* in megabits per second */
/* operations */
@@ -297,6 +298,7 @@
long long offset;
long long input_offset;
long long output_adjust;
+ long long nframes_out;
int output_adjust_set;
unsigned long buflen;
unsigned int mode_flags;
@@ -307,7 +309,7 @@
int n_metafiles;
/* output buffering */
- int nraw;
+ int nraw, nraw_active;
struct df_raw *raw;
unsigned long srv_base;
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2014-09-30 02:40:40 UTC (rev 923)
+++ trunk/defile/lib/libdefile.c 2014-10-01 00:04:49 UTC (rev 924)
@@ -754,6 +754,9 @@
df->raw[r].spf = f->spf * count;
df->raw[r].in = 0;
df->raw[r].b = NULL;
+ df->raw[r].nframes_out = df->nframes_out;
+ if (df->nframes_out)
+ df->nraw_active++;
pthread_mutex_init(&df->raw[r].mx, NULL);
} else {
/* increase the samples per frame of this raw */
@@ -1070,9 +1073,9 @@
if (!(df->mode_flags & DF_MODE_FOLLOW)) {
pthread_mutex_lock(&df_config_mx);
if (add)
- df->nframes += length;
+ df->nframes_in += length;
else
- df->nframes = length;
+ df->nframes_in = length;
pthread_mutex_unlock(&df_config_mx);
}
@@ -1095,7 +1098,7 @@
pthread_mutex_lock(&df_config_mx);
if (rate >= 0)
df->rate = rate;
- df->nframes = length;
+ df->nframes_in = length;
pthread_mutex_unlock(&df_config_mx);
/* wait for the output thread to create the dirfile */
@@ -1506,7 +1509,7 @@
df->nwrote = 0;
df->wpartial = 0;
df->nread = 0;
- df->nframes = (uint64_t)length;
+ df->nframes_in = (uint64_t)length;
df->wframesize = df->rframesize = df->sframesize = 0;
pthread_mutex_unlock(&df_rate_mx);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
