Thread: [Getdata-commits] SF.net SVN: getdata:[662] trunk/defile
Scientific Database Format
Brought to you by:
ketiltrout
Menu
▾
▴
getdata-commits
|
From: <ket...@us...> - 2012-02-20 22:35:43
|
Revision: 662
http://getdata.svn.sourceforge.net/getdata/?rev=662&view=rev
Author: ketiltrout
Date: 2012-02-20 22:35:37 +0000 (Mon, 20 Feb 2012)
Log Message:
-----------
svn:ignore
Property Changed:
----------------
trunk/defile/
trunk/defile/bin/
trunk/defile/input/
trunk/defile/lib/
trunk/defile/m4/
trunk/defile/test/
Property changes on: trunk/defile
___________________________________________________________________
Added: svn:ignore
+ Makefile
Makefile.in
aclocal.m4
autom4te.cache
config.guess
config.log
config.lt
config.status
config.sub
configure
depcomp
install-sh
libtool
ltmain.sh
missing
Property changes on: trunk/defile/bin
___________________________________________________________________
Added: svn:ignore
+ .deps
Makefile
Makefile.in
defile
Property changes on: trunk/defile/input
___________________________________________________________________
Added: svn:ignore
+ .deps
Makefile
Makefile.in
Property changes on: trunk/defile/lib
___________________________________________________________________
Added: svn:ignore
+ .deps
Makefile
Makefile.in
config.h
config.h.in
stamp-h1
Property changes on: trunk/defile/m4
___________________________________________________________________
Added: svn:ignore
+ libtool.m4
ltoptions.m4
ltsugar.m4
ltversion.m4
lt~obsolete.m4
Property changes on: trunk/defile/test
___________________________________________________________________
Added: svn:ignore
+ .deps
Makefile
Makefile.in
madedata
makedata
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2012-02-21 01:23:52
|
Revision: 663
http://getdata.svn.sourceforge.net/getdata/?rev=663&view=rev
Author: ketiltrout
Date: 2012-02-21 01:23:45 +0000 (Tue, 21 Feb 2012)
Log Message:
-----------
Input description
Modified Paths:
--------------
trunk/defile/bin/cli.c
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/defile.h
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-02-20 22:35:37 UTC (rev 662)
+++ trunk/defile/bin/cli.c 2012-02-21 01:23:45 UTC (rev 663)
@@ -153,8 +153,12 @@
fprintf(stream, " v%s", fwk->version);
if (fwk->copyright)
fprintf(stream, " %s", fwk->copyright);
- if (path)
+ if (path) {
+ if (fwk->description)
+ fprintf(stream, "\n%s [type: %s]", prefix ? prefix : "",
+ fwk->description);
fprintf(stream, "\n%s [path: %s]", prefix ? prefix : "", path);
+ }
putc('\n', stream);
}
Modified: trunk/defile/input/ascii.c
===================================================================
--- trunk/defile/input/ascii.c 2012-02-20 22:35:37 UTC (rev 662)
+++ trunk/defile/input/ascii.c 2012-02-21 01:23:45 UTC (rev 663)
@@ -39,6 +39,7 @@
#define ASCII_COPYRIGHT "Copyright (C) 2012 D. V. Wiebe"
#define ASCII_CONTACT \
"Please send reports of bugs and other communication to:\n " PACKAGE_BUGREPORT
+#define ASCII_DESCRIPTION "ASCII tabular data"
/* the probe function: given the name of the input, returns non-zero if this
* module is willing to attempt to decode it */
@@ -343,6 +344,7 @@
.licence = DF_LICENCE_SAME,
.contact = ASCII_CONTACT,
.copyright = ASCII_COPYRIGHT,
+ .description = ASCII_DESCRIPTION,
.name = "ascii",
.version = PACKAGE_VERSION,
.preamble = NULL,
Modified: trunk/defile/input/dirfile.c
===================================================================
--- trunk/defile/input/dirfile.c 2012-02-20 22:35:37 UTC (rev 662)
+++ trunk/defile/input/dirfile.c 2012-02-21 01:23:45 UTC (rev 663)
@@ -32,6 +32,7 @@
#define DIRFILE_COPYRIGHT "Copyright (C) 2012 D. V. Wiebe"
#define DIRFILE_CONTACT \
"Please send reports of bugs and other communication to:\n " PACKAGE_BUGREPORT
+#define DIRFILE_DESCRIPTION "dirfile databases"
#define CHUNK_SIZE 65535 /* in bytes, the largest amount of data we're willing
to read in one go */
@@ -338,6 +339,7 @@
.licence = DF_LICENCE_SAME,
.contact = DIRFILE_CONTACT,
.copyright = DIRFILE_COPYRIGHT,
+ .description = DIRFILE_DESCRIPTION,
.name = "dirfile",
.version = PACKAGE_VERSION,
.preamble = NULL,
Modified: trunk/defile/lib/defile.h
===================================================================
--- trunk/defile/lib/defile.h 2012-02-20 22:35:37 UTC (rev 662)
+++ trunk/defile/lib/defile.h 2012-02-21 01:23:45 UTC (rev 663)
@@ -88,7 +88,7 @@
};
/* The input plugin description */
-#define DF_LICENCE_OTHER 0
+#define DF_LICENCE_OTHER 0 /* no automatic licence boilerplate */
#define DF_LICENCE_SAME 1 /* the same license as defile (whatver that is) */
#define DF_LICENCE_GPL 2 /* GPL v2+ */
#define DF_LICENCE_LGPL 3 /* LGPL v2.1+ */
@@ -98,6 +98,7 @@
const char *version; /* the version string for the plugin */
const char *copyright; /* the copyright notice for the plugin */
const char *contact; /* a contact (and perhaps licence) notice */
+ const char *description; /* description of the file format read */
const char *preamble; /* the --help preamble */
const char *postamble; /* the --help postamble */
const struct df_optdef *opts; /* the plugin command-line options */
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2012-03-07 04:48:10
|
Revision: 668
http://getdata.svn.sourceforge.net/getdata/?rev=668&view=rev
Author: ketiltrout
Date: 2012-03-07 04:48:00 +0000 (Wed, 07 Mar 2012)
Log Message:
-----------
Bug fixes, extra functionality, docs.
Modified Paths:
--------------
trunk/defile/Makefile.am
trunk/defile/bin/cli.c
trunk/defile/bin/rc.c
trunk/defile/configure.ac
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/defile.h
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Added Paths:
-----------
trunk/defile/doc/
trunk/defile/doc/Makefile.am
trunk/defile/doc/defile-input.7
trunk/defile/doc/defile.1.in
trunk/defile/doc/df_add_alias.3
trunk/defile/doc/df_add_entry.3
trunk/defile/doc/df_add_framedef.3
trunk/defile/doc/df_bsearch.3
trunk/defile/doc/df_check_abort.3
trunk/defile/doc/df_field_in_input.3
trunk/defile/doc/df_init.3
trunk/defile/doc/df_input_name.3
trunk/defile/doc/df_nframes_allowed.3
trunk/defile/doc/df_on_abort.3
trunk/defile/doc/df_push_frame.3
trunk/defile/doc/df_ready.3
trunk/defile/doc/df_set_const.3
trunk/defile/doc/df_shell_expand.3
Modified: trunk/defile/Makefile.am
===================================================================
--- trunk/defile/Makefile.am 2012-03-06 01:45:18 UTC (rev 667)
+++ trunk/defile/Makefile.am 2012-03-07 04:48:00 UTC (rev 668)
@@ -22,4 +22,4 @@
ACLOCAL_AMFLAGS = -I m4
-SUBDIRS = lib bin input test
+SUBDIRS = lib bin input doc test
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-03-06 01:45:18 UTC (rev 667)
+++ trunk/defile/bin/cli.c 2012-03-07 04:48:00 UTC (rev 668)
@@ -92,23 +92,37 @@
#define SETULONG(x,y) SETSIMPLE(x,y,strtoul(config->argument, NULL, 0),,\
"%lu")
+#define ABORT_DELAY 60000 /* sixty seconds; this is less than the maximum TCP
+ * round-trip timeout. Do we care? */
+
+/* the very defile object */
+static struct df_defile the_defile;
+struct df_defile *const df = &the_defile;
+
+/* state mutex */
+pthread_mutex_t state_mx = PTHREAD_MUTEX_INITIALIZER;
+
static void DF_SignalHandler(int sig)
{
if (!df->quiet)
switch (sig) {
case SIGHUP:
- puts("Aborting due to hang up");
+ puts("Aborting due to hang up.");
break;
case SIGINT:
- puts("Aborting due to interruption");
+ puts("Aborting due to interruption.");
break;
case SIGTERM:
- puts("Aborting due to unexpected termination");
+ puts("Aborting due to unexpected termination.");
break;
}
- df->input_state = DF_ST_ABORT;
- df->output_state = DF_ST_ABORT;
+ pthread_mutex_lock(&state_mx);
+ if (df->output_state != DF_ST_DONE && df->output_state != DF_ST_ERROR)
+ DF_OUTSTATE(ABORT);
+ if (df->input_state != DF_ST_DONE && df->input_state != DF_ST_ERROR)
+ DF_INSTATE(ABORT);
+ pthread_mutex_unlock(&state_mx);
}
static void *DF_Embiggen(void *old, size_t size)
@@ -250,6 +264,10 @@
result = 4;
lt_dlclose(lib);
}
+ if (df->config.mode_flags & DF_MODE_DEBUG) {
+ fprintf(stderr, "defile: probing %s; result: %s\n", filename,
+ (result) ? "fail" : "pass");
+ }
}
/* success, add it to the list */
@@ -469,46 +487,24 @@
/* remove */
free(df->ofl[r]);
memmove(df->ofl + r, df->ofl + r + 1, sizeof(char*) * (df->nofl-- - r));
+ /* an empty OFL always means write everything */
+ if (df->nofl == 0)
+ df->ofl_incl = 0;
}
}
return 0;
}
-static int DF_FileToOFL(const char *name, int incl)
-{
- abort();
-
- return 0;
-}
-
enum {
- DF_WOPT_ADD = 1,
- DF_WOPT_ADDFROM,
- DF_WOPT_ASYNC,
- DF_WOPT_AUTOTYPE,
- DF_WOPT_BHEAD,
- DF_WOPT_BLEN,
- DF_WOPT_CLOBBER,
- DF_WOPT_DEL,
- DF_WOPT_DELFROM,
- DF_WOPT_DIR,
- DF_WOPT_GZIP,
- DF_WOPT_INPUT,
- DF_WOPT_LIBDIR,
- DF_WOPT_LIST,
- DF_WOPT_NOCLOBBER,
- DF_WOPT_OFLCLR,
- DF_WOPT_OUTPUT,
- DF_WOPT_PERSIST,
- DF_WOPT_QUIET,
- DF_WOPT_SYNC,
- DF_WOPT_TERMEOF,
- DF_WOPT_VERBOSE,
- DF_WOPT_TYPE,
+ DF_WOPT_ADD = 1, DF_WOPT_ASYNC, DF_WOPT_AUTOTYPE, DF_WOPT_BHEAD, DF_WOPT_BLEN,
+ DF_WOPT_CLOBBER, DF_WOPT_DEBUG, DF_WOPT_DEL, DF_WOPT_DIR, DF_WOPT_GZIP,
+ DF_WOPT_INPUT, DF_WOPT_LIBDIR, DF_WOPT_LIST, DF_WOPT_NOCLOBBER,
+ DF_WOPT_NODEBUG, DF_WOPT_NOFOLLOW, DF_WOPT_OFLCLR, DF_WOPT_OUTPUT,
+ DF_WOPT_FOLLOW, DF_WOPT_QUIET, DF_WOPT_SYNC, DF_WOPT_TYPE, DF_WOPT_VERBOSE
};
-static int DF_OutputOpt(df_defiler df, struct df_rc_config *config,
+static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
struct df_rc_data *rcd)
{
int i;
@@ -542,18 +538,12 @@
case DF_RC_MODE_SW:
df->mode_switch = 1;
break;
- case DF_WOPT_ADD: /* -f */
+ case DF_WOPT_ADD: /* -F */
DF_UpdateOFL(config->argument, 1);
break;
- case DF_WOPT_DEL: /* +f */
+ case DF_WOPT_DEL: /* +F */
DF_UpdateOFL(config->argument, 0);
break;
- case DF_WOPT_ADDFROM: /* -F */
- DF_FileToOFL(config->argument, 1);
- break;
- case DF_WOPT_DELFROM: /* +F */
- DF_FileToOFL(config->argument, 0);
- break;
case DF_WOPT_OFLCLR: /* --reset-field-list */
for (i = 0; i < df->nofl; ++i)
free(df->ofl[i]);
@@ -577,7 +567,7 @@
SETULONG(BHEAD, bufhead);
- SETULONG(BLEN, buflen);
+ SETULONG(BLEN, config.buflen);
SETLINT(CLOBBER, clobber, 1);
SETLINT(NOCLOBBER, clobber, 0);
@@ -585,12 +575,15 @@
SETSTRING(DIR, dir);
SETSTRING(OUTPUT, output);
- SETBIT(PERSIST, config.mode_flags, DF_MODE_PERSIST);
- CLRBIT(TERMEOF, config.mode_flags, DF_MODE_PERSIST);
+ SETBIT(FOLLOW, config.mode_flags, DF_MODE_FOLLOW);
+ CLRBIT(NOFOLLOW, config.mode_flags, DF_MODE_FOLLOW);
SETLINT(QUIET, quiet, 1);
SETLINT(VERBOSE, quiet, 0);
+ SETBIT(DEBUG, config.mode_flags, DF_MODE_DEBUG);
+ CLRBIT(NODEBUG, config.mode_flags, DF_MODE_DEBUG);
+
SETBIT(ASYNC, config.mode_flags, DF_MODE_ASYNC);
CLRBIT(SYNC, config.mode_flags, DF_MODE_ASYNC);
@@ -629,60 +622,152 @@
return ptr;
}
-static void DF_OutputFini(int delete)
+static int DF_OutputFini(int delete)
{
gd_close(df->D);
if (delete) {
pid_t pid = fork();
+ int status;
+
if (pid > 0)
waitpid(pid, NULL, 0);
else if (pid == 0) {
execl(PATH_RM, PATH_RM, "-rf", df->output, (char*)NULL);
+ /* can't get here */
+ fputs("defile: exec failure while deleting discarded output\n", stderr);
exit(1);
}
+ waitpid(pid, &status, 0);
+ if (!WIFEXITED(status) || WEXITSTATUS(status)) {
+ fputs("defile: error deleting discarded output\n", stderr);
+ return 1;
+ }
}
free(df->output);
+
+ return 0;
}
-static void DF_CreateOutput(unsigned long flags)
+static int DF_CreateOutput(unsigned long flags)
{
+ int gderr;
+
df->flags = flags;
df->D = gd_open(df->dirfile, GD_RDWR | GD_CREAT | ((flags & DF_CLOBBER) ?
GD_TRUNC : GD_EXCL) | GD_VERBOSE);
- if (gd_error(df->D)) {
+ if ((gderr = gd_error(df->D))) {
char *err = gd_error_string(df->D, NULL, 0);
fprintf(stderr, "defile: getdata error: %s\n", err);
free(err);
- DF_OutputFini(0);
- exit(1);
+ if (gderr == GD_E_CREAT) { /* no clobber: allow the input to try again */
+ pthread_mutex_lock(&state_mx);
+ DF_OUTSTATE(INIT);
+ pthread_mutex_unlock(&state_mx);
+ } else
+ return 1;
}
+
+ return 0;
}
-/* check thread status; returns zero when we're done */
-static int DF_ThreadCheck(df_defiler df)
+/* check thread status; returns zero when defile's done */
+static int DF_ThreadCheck(struct df_defile *df, const struct timeval *time)
{
- /* error */
- if (df->input_state < 0 || df->output_state < 0)
- return 0;
+ /* this will overflow in the year 294,247 (Y294k bug) */
+ const long long now = time->tv_sec * 1000000 + time->tv_usec;
/* done */
if (df->input_state == DF_ST_DONE && df->output_state == DF_ST_DONE)
return 0;
+ /* abort the other on error */
+ if (df->abort_timer == 0) {
+ pthread_mutex_lock(&state_mx);
+ if (df->input_state == DF_ST_ERROR) {
+ DF_INSTATE(DONE);
+ if (df->output_state != DF_ST_DONE && df->output_state != DF_ST_ERROR)
+ DF_OUTSTATE(ABORT);
+ }
+ if (df->output_state == DF_ST_ERROR) {
+ DF_OUTSTATE(DONE);
+ if (df->input_state != DF_ST_DONE && df->input_state != DF_ST_ERROR)
+ DF_INSTATE(ABORT);
+ }
+
+ if (df->output_state == DF_ST_ABORT || df->input_state == DF_ST_ABORT)
+ df->abort_timer = now + ABORT_DELAY;
+ pthread_mutex_unlock(&state_mx);
+ } else if (df->abort_timer < now) {
+ if (df->output_state != DF_ST_DONE) {
+ fputs("defile: cancelling unresponsive output\n", stderr);
+ pthread_cancel(df->output_id);
+ }
+ if (df->input_state != DF_ST_DONE) {
+ fputs("defile: cancelling unresponsive input\n", stderr);
+ pthread_cancel(df->input_id);
+ }
+ return 0;
+ }
+
/* still going */
return 1;
}
/* wait while the input does something */
-static inline void DF_WaitWhileInputState(int state)
+static inline void DF_WaitWhileInputStates(int state1, int state2, int quiet)
{
+ if (!quiet && df->config.mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: output waiting on input for end of %s and %s\n",
+ DF_STATENAME(state1), DF_STATENAME(state2));
+ while (df->input_state == state1 || df->input_state == state2)
+ usleep(10000);
+}
+
+/* wait while the input does something */
+static inline void DF_WaitWhileInputState(int state, int quiet)
+{
+ if (!quiet && df->config.mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: output waiting on input for end of %s\n",
+ DF_STATENAME(state));
while (df->input_state == state)
usleep(10000);
}
+static int DF_MakeOutput(void)
+{
+ /* figure out the name of the output */
+ df->dirfile = DF_OutputName(df->dir, df->output, df->input);
+
+ if (!df->dirfile)
+ return 1;
+
+ if (DF_CreateOutput(df->clobber ? DF_CLOBBER : 0))
+ return 1;
+
+ if (df->output_state == DF_ST_ABORT)
+ return 1;
+
+ /* signal we're ready for metadata */
+ DF_OUTSTATE_MX(BUILD);
+
+ /* wait for the input to build the dirfile */
+ DF_WaitWhileInputStates(DF_ST_BUILD, DF_ST_INIT, 0);
+
+ if (df->output_state == DF_ST_ABORT)
+ return 1;
+
+ /* write metadata to disk */
+ gd_metaflush(df->D);
+
+ /* signal we're ready for data */
+ DF_OUTSTATE_MX(RUN);
+
+ return 0;
+}
+
/* output thread */
static void *DF_OutputThread(void)
{
@@ -692,37 +777,52 @@
unsigned long in, out, nf;
/* wait for the input to initialise */
- DF_WaitWhileInputState(DF_ST_STARTUP);
+ DF_WaitWhileInputState(DF_ST_START, 0);
/* initialise the output */
- df->dirfile = DF_OutputName(df->dir, df->output, df->input);
- if (!df->dirfile)
- exit(1);
+ if (DF_MakeOutput()) {
+ DF_OUTSTATE_MX(ERROR);
+ return NULL;
+ }
- DF_CreateOutput(df->clobber ? DF_CLOBBER : 0);
-
- /* signal we're ready for metadata */
- df->output_state = DF_ST_BUILDING;
-
- /* wait for the input to build the dirfile */
- DF_WaitWhileInputState(DF_ST_BUILDING);
-
- /* write metadata to disk */
- gd_metaflush(df->D);
-
- /* signal we're ready to for data */
- df->output_state = DF_ST_RUNNING;
-
/* loop through raws, writing ones that have full buffer heads */
while (!last_pass) {
- if (df->input_state != DF_ST_RUNNING)
+ if (df->input_state != DF_ST_RUN)
last_pass = 1;
for (i = 0; i < df->nraw; ++i) {
+ /* check for unexpected state change */
+ if (df->output_state != DF_ST_RUN) {
+ int delete = 0;
+ switch(df->output_state) {
+ case DF_ST_DROPSTART:
+ delete = 1;
+ /* fallthrough */
+ case DF_ST_SAVESTART:
+ if (DF_OutputFini(delete)) {
+ DF_OUTSTATE_MX(ERROR);
+ return NULL;
+ } else if (DF_MakeOutput()) {
+ DF_OUTSTATE_MX(ERROR);
+ return NULL;
+ }
+ break;
+ case DF_ST_ABORT:
+ DF_OutputFini(0);
+ DF_OUTSTATE_MX(DONE);
+ return NULL;
+ default:
+ fputs("defile: unexpected traversal in output state map.\n",
+ stderr);
+ DF_OutputFini(0);
+ DF_OUTSTATE_MX(ERROR);
+ return NULL;
+ }
+ }
r = df->raw + i;
in = r->in;
out = r->out;
- nf = (in == out) ? df->buflen - 1 :
+ nf = (in == out) ? df->config.buflen - 1 :
((in + r->buflen - out) % r->buflen) / r->framesize - 1;
if (nf == 0 || (!last_pass && nf <= df->bufhead))
@@ -759,13 +859,10 @@
/* bail on error */
if (gd_error(df->D)) {
- df->output_state = DF_ST_ERROR;
- exit(1);
+ DF_OUTSTATE_MX(ERROR);
+ return NULL;
}
- if (nwrote > 0)
- last_pass = 0;
-
/* update ahead counts */
nf = nwrote / r->spf;
if (i == df->ref_ind) {
@@ -775,6 +872,20 @@
} else
r->ahead += nwrote / r->spf;
+ if (nwrote > 0) {
+ last_pass = 0;
+ if (df->config.mode_flags & DF_MODE_DEBUG) {
+ if (i != df->ref_ind && !(df->config.mode_flags & DF_MODE_ASYNC))
+ fprintf(stderr, "defile: output %lu %s of field %s "
+ "(lead is %lu %s)\n",
+ nf, (nf == 1) ? "frame" : "frames", r->name, r->ahead,
+ (r->ahead == 1) ? "frame" : "frames");
+ else
+ fprintf(stderr, "defile: output %lu %s of field %s\n", nf,
+ (nf == 1) ? "frame" : "frames", r->name);
+ }
+ }
+
/* update write pointer */
r->out = (r->out + nwrote * GD_SIZE(r->type)) % r->buflen;
@@ -789,23 +900,25 @@
}
/* finished */
- gd_close(df->D);
+ DF_OutputFini(0);
/* done */
- df->output_state = DF_ST_DONE;
+ DF_OUTSTATE_MX(DONE);
return NULL;
}
/* input thread container. */
static void *DF_InputThread(void)
{
- int status;
-
/* Execute the input and wait for it to finish */
- status = df->input_framework->entry(&df->config);
+ int ret = df->input_framework->entry(&df->config);
- /* report status */
- df->input_state = status ? DF_ST_ERROR : DF_ST_DONE;
+ pthread_mutex_lock(&state_mx);
+ if (ret)
+ DF_INSTATE(ERROR);
+ else
+ DF_INSTATE(DONE);
+ pthread_mutex_unlock(&state_mx);
/* done */
return NULL;
@@ -818,6 +931,8 @@
static unsigned long long lw = 0;
static int init = 2;
+ const char eol = (df->config.mode_flags & DF_MODE_DEBUG) ? '\n' : '\r';
+
double delta, drate;
unsigned long long nf;
const unsigned long long nw = df->nwrote;
@@ -876,13 +991,11 @@
si = 'm';
}
if (df->nframes > 0)
- printf("R:[%llu(%llu) of %llu] W:[%llu(%llu)] %.*f %cHz \r", nr,
- (unsigned long long)df->rpartial, df->nframes, nw,
- (unsigned long long)df->wpartial,
- (drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si);
+ printf("R:[%llu of %llu] W:[%llu] %.*f %cHz %c", nr, df->nframes, nw,
+ (drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si, eol);
else
- printf("R:[%llu] W:[%llu] %.*f %cHz \r", nr, nw,
- (drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si);
+ printf("R:[%llu] W:[%llu] %.*f %cHz %c", nr, nw,
+ (drate >= 100) ? 1 : (drate >= 10) ? 2 : 3, drate, si, eol);
fflush(stdout);
return rate;
@@ -925,8 +1038,6 @@
int main(int argc, char **argv)
{
- pthread_t input_thread, output_thread;
-
/* rate stuff */
double rate;
struct timeval then;
@@ -937,77 +1048,45 @@
const struct df_optdef options[] = {
{ DF_WOPT_ASYNC, DF_OPT_NO_ARG, 'a', "asyncrhonous", "Asynchronous", 0,
- NULL,
- "allow field in the output dirfile to be written to disk independantly "
- "of each other; for some input types, this can greatly increase the "
- "efficiency of the conversion"
+ NULL, "operate in asynchronous mode"
},
{ DF_WOPT_SYNC, DF_OPT_NO_ARG, 'a', "synchronous", "Synchronous",
- DF_OPT_PLUS, NULL,
- "ensure that the output dirfile's reference field never has more data "
- "any other raw field (the default behaviour)"
+ DF_OPT_PLUS, NULL, "operate in synchronous mode"
},
{ DF_WOPT_CLOBBER, DF_OPT_NO_ARG, 0, "clobber", "Clobber", 0, NULL,
- "if the output dirfile already exists, overwrite it; this will delete "
- "every regular file in the target directory; use with caution"
+ "if the output dirfile already exists, overwrite it; use with caution"
},
{ DF_WOPT_NOCLOBBER, DF_OPT_NO_ARG, 0, "no-clobber", "NoClobber", 0, NULL,
- "don't overwrite an existing dirfile (the default behaviour)"
+ "don't overwrite an existing dirfile (default)"
},
- { DF_WOPT_DIR, DF_OPT_ARG_RQ, 'd', "directory", "Directory", DF_OPT_EXPAND,
- "DIR",
- "if -o is not specified, create the output dirfile as a subdirectory "
- "under DIR, with a name based on the name of the input file; if -o is "
- "used, but does not specify an absolute path, create the specified "
- "output dirfile under DIR; if -o specifies an absolute path, this "
- "option is ignored"
+ { DF_WOPT_DIR, DF_OPT_ARG_RQ, 'D', "directory", "Directory", DF_OPT_EXPAND,
+ "DIR", "write the output dirfile as a subdirectory under DIR"
},
- { DF_WOPT_ADD, DF_OPT_ARG_RQ, 'f', "include", "WriteField", 0, "FIELD",
- "add FIELD to the output field list; this option may be given multiple "
- "times to build up a list of fields to include, and may be used in "
- "conjunction with -F; however, mixing -[fF] with +[fF] may not work as "
- "expected; inserting --reset-field-list before switching from -[fF] to "
- "+[fF] is recommended"
+ { DF_WOPT_DEBUG, DF_OPT_NO_ARG, 'd', "debug", "Debug", 0, NULL,
+ "print debugging messages"
},
- { DF_WOPT_DEL, DF_OPT_ARG_RQ, 'f', "exclude", "IgnoreField", DF_OPT_PLUS,
- "FIELD",
- "remove FIELD from the output field list; this option may be given "
- "multiple times to build up a list of fields to exclude, and may be "
- "used in conjunction with +F; however, mixing +[fF] with -[fF] may not "
- "work as expected; inserting --reset-field-list before switching from "
- "+[fF] to -[fF] is recommended"
+ { DF_WOPT_NODEBUG, DF_OPT_NO_ARG, 'd', "no-debug", "NoDebug", DF_OPT_PLUS,
+ NULL, "suppress printing debugging messages (default)"
},
- { DF_WOPT_ADDFROM, DF_OPT_ARG_RQ, 'F', "include-from", "WriteAllFrom",
- DF_OPT_EXPAND, "FILE",
- "add all fields given in FILE (one field name per line) to the output "
- "field list; this option may be given multiple times; see -f for "
- "caveats concerning combinding this option with +f or +F"
+ { DF_WOPT_ADD, DF_OPT_ARG_RQ, 'F', "include", "IncludeField", 0, "FIELD",
+ "include FIELD in the output dirfile, if available"
},
- { DF_WOPT_DELFROM, DF_OPT_ARG_RQ, 'F', "exclude-from", "IgnoreAllFrom",
- DF_OPT_PLUS | DF_OPT_EXPAND, "FILE",
- "remove all fields given in FILE (one field name per line) from the "
- "output field list; this option may be given multiple times; see +f "
- "for caveats concerning combinding this option with -f or -F"
+ { DF_WOPT_DEL, DF_OPT_ARG_RQ, 'F', "exclude", "ExcludeField", DF_OPT_PLUS,
+ "FIELD", "exclude FIELD from the output dirfile"
},
{ DF_WOPT_BHEAD, DF_OPT_ARG_RQ, 'h', "buf-head", "BufferHead", 0, "N",
- "set the length of the output buffer head to N frames; defile waits for "
- "the buffer head to fill up before writing the accumulated buffer to "
- "disk; the default value is zero, meaning data are written as soon as "
- "possible; setting this to a non-zero number may improve performance, "
- "at the cost of latency and memory usage; see also -l below"
+ "set the length of the output buffer head to N frames; the default "
+ "value is zero"
},
- { DF_WOPT_INPUT, DF_OPT_ARG_RQ, 'i', "input", "Input", 0, "PATH",
- "read input data from PATH; typically the input PATH can be specified as "
+ { DF_WOPT_INPUT, DF_OPT_ARG_RQ, 'i', "input", "Input", 0, "NAME",
+ "read input data from NAME; typically the NAME can be specified as "
"a bare argument, without the need for -i; this option is provided "
- "only for use in situations where PATH begins with - or + (and would, "
+ "only for use in situations where NAME begins with - or + (and would, "
"therefore, be mistaken for an option)"
},
{ DF_WOPT_BLEN, DF_OPT_ARG_RQ, 'l', "buf-len", "BufferLen", 0, "N",
"set the length of the output buffer to N frames; the default value "
- "is " DF_STRINGIFY(DF_BUFLEN) " frames; decreasing this will reduce "
- "memory usage at the expense of performance; if this is smaller than "
- "the buffer header (see -h above), the output buffer will be expanded "
- "to accommodate it"
+ "is " DF_STRINGIFY(DF_BUFLEN) ""
},
{ DF_WOPT_LIBDIR, DF_OPT_NO_ARG, 0, "lib-dir", NULL, 0, NULL,
"print the default input plugin path and exit"
@@ -1015,25 +1094,20 @@
{ DF_WOPT_LIST, DF_OPT_NO_ARG, 0, "list-types", NULL, 0, NULL,
"print a list of available input plugins and exit"
},
- { DF_WOPT_OFLCLR, DF_OPT_NO_ARG, 0, "reset-field-list", "ResetFieldList", 0,
+ { DF_WOPT_OFLCLR, DF_OPT_NO_ARG, 0, "clear-field-list", "ClearFieldList", 0,
NULL,
- "ignore all previous options which modifiy the ouput field list (-f, +f, "
- "-F, +F); this is typically not needed, but can be used if the current "
- "contents of the output field list is unknown"
+ "ignore all previous options which modifiy the ouput field list (-F, +F)"
},
{ DF_WOPT_OUTPUT, DF_OPT_ARG_RQ, 'o', "output", "Output", DF_OPT_EXPAND,
- "PATH", "write data to a dirfile called PATH; if PATH is relative, it is "
- "created relative the current directory, unless -d is also specified"
+ "PATH", "write data to a dirfile called PATH"
},
- { DF_WOPT_PERSIST, DF_OPT_NO_ARG, 'p', "persistent", "Persistent", 0, NULL,
- "upon reaching the end of the input, do not exit, but monitor the input "
- "for additional data, and keep writing to the output as new data "
- "appears; implies -s"
+ { DF_WOPT_FOLLOW, DF_OPT_NO_ARG, 'f', "follow", "Follow", 0, NULL,
+ "keep monitoring the input and write data to the outpu as it becomes "
+ "available"
},
- { DF_WOPT_TERMEOF, DF_OPT_NO_ARG, 'p', "terminate-on-eof", "TerminateOnEOF",
+ { DF_WOPT_NOFOLLOW, DF_OPT_NO_ARG, 'f', "no-follow", "NoFollow",
DF_OPT_PLUS, NULL,
- "finalise the output and exit upon reaching the end of the input (the "
- "default behaviour)"
+ "exit successfully upon reaching the end of the input (default)"
},
{ DF_WOPT_QUIET, DF_OPT_NO_ARG, 'q', "quiet", "Quiet", 0, NULL,
"be less verbose"
@@ -1047,10 +1121,10 @@
{ DF_WOPT_AUTOTYPE, DF_OPT_NO_ARG, 't', "autotype", "AutoType", DF_OPT_PLUS,
NULL, "attempt to automatically determine the data type of the input"
},
- { DF_WOPT_GZIP, DF_OPT_NO_ARG, 'z', "gzip", "GZip", 0, NULL,
- "gzip compress the output raw data"
- },
- { DF_OPT_ENDOPT }
+ // { DF_WOPT_GZIP, DF_OPT_NO_ARG, 'z', "gzip", "GZip", 0, NULL,
+ // "gzip compress the output raw data"
+ // },
+ DF_OPT_ENDOPT
};
struct df_rc_data rcd = {
@@ -1073,7 +1147,7 @@
df->iargv = malloc(sizeof(char*));
df->iargv[0] = argv[0];
df->iargc = 1;
- df->buflen = DF_BUFLEN;
+ df->config.buflen = DF_BUFLEN;
/* initialise LTDL and register its cleanup function */
lt_dlinit();
@@ -1082,8 +1156,8 @@
DF_Parse(argc, argv, &rcd, df);
/* sanitise */
- if (df->buflen < df->bufhead)
- df->buflen = df->bufhead;
+ if (df->config.buflen < df->bufhead)
+ df->config.buflen = df->bufhead;
if (!df->quiet)
puts(PACKAGE_STRING " " DEFILE_COPYRIGHT);
@@ -1137,8 +1211,9 @@
pthread_sigmask(SIG_BLOCK, &signals, NULL);
/* start the threads */
- df->input_state = DF_ST_STARTUP;
- if (pthread_create(&input_thread, NULL, (void*(*)(void*))DF_InputThread,
+ DF_INSTATE(START);
+ DF_OUTSTATE(START);
+ if (pthread_create(&df->input_id, NULL, (void*(*)(void*))DF_InputThread,
NULL))
{
perror("pthread_create");
@@ -1146,8 +1221,7 @@
}
/* start the output thread */
- df->output_state = DF_ST_STARTUP;
- if (pthread_create(&output_thread, NULL, (void*(*)(void*))DF_OutputThread,
+ if (pthread_create(&df->output_id, NULL, (void*(*)(void*))DF_OutputThread,
NULL))
{
perror("pthread_create");
@@ -1158,8 +1232,8 @@
pthread_sigmask(SIG_UNBLOCK, &signals, NULL);
/* wait until the initialisation is done */
- DF_WaitWhileInputState(DF_ST_STARTUP);
- DF_WaitWhileInputState(DF_ST_BUILDING);
+ DF_WaitWhileInputState(DF_ST_START, 1);
+ DF_WaitWhileInputState(DF_ST_BUILD, 1);
/* record our "start" time */
gettimeofday(&then, &tz);
@@ -1167,18 +1241,21 @@
/* status loop */
df->tc = 1.;
- while (DF_ThreadCheck(df)) {
+ do {
/* print the status line, if necessary */
rate = DF_PrintRate(&then, rate);
usleep(500000); /* ~2Hz refresh rate */
- }
+ } while (DF_ThreadCheck(df, &then));
DF_PrintRate(&then, rate);
if (!df->quiet)
putchar('\n');
- pthread_join(input_thread, NULL);
- pthread_join(output_thread, NULL);
+ /* one last check */
+ if (pthread_kill(df->input_id, 0) == 0)
+ fputs("defile: terminating with input still active!\n", stderr);
+ if (pthread_kill(df->output_id, 0) == 0)
+ fputs("defile: terminating with output still active!\n", stderr);
return 0;
}
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2012-03-06 01:45:18 UTC (rev 667)
+++ trunk/defile/bin/rc.c 2012-03-07 04:48:00 UTC (rev 668)
@@ -63,7 +63,7 @@
"51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA"
static const struct df_optdef df_no_opts[] = {
- { DF_OPT_ENDOPT }
+ DF_OPT_ENDOPT
};
struct shortarg_s {
@@ -348,6 +348,21 @@
}
}
+static void DF_DoRunControl (const char *val, const char *cwd,
+ struct df_rc_data *rcd)
+{
+ if (val == NULL)
+ logprintf(" Skipping unspecified configuration file.\n");
+ else {
+ char file[4096];
+ if (val[0] != '/' && val[0] != '~' && val[0] != '$')
+ snprintf(file, 4096, "%s/%s", cwd, val);
+ else
+ strcpy(file, val);
+ DF_ReadConfigFile(file, 0, val, rcd);
+ }
+}
+
static void DF_PushConfig(int i, const char* val, int error, int no_dup,
const char* cwd, const char* label, int line, int sopt,
struct df_rc_data *rcd)
@@ -361,22 +376,16 @@
rcd->options[i].value, rcd->confc, line);
if (opt) {
if (opt->type == DF_OPT_CONFIG) {
- if (val == NULL)
- logprintf(" Skipping unspecified configuration file.\n");
- else {
- char file[4096];
- if (val[0] != '/' && val[0] != '~' && val[0] != '$')
- snprintf(file, 4096, "%s/%s", cwd, val);
- else
- strcpy(file, val);
- DF_ReadConfigFile(file, 0, val, rcd);
- }
+ DF_DoRunControl(val, cwd, rcd);
return;
}
if (val && opt->flags & DF_OPT_EXPAND) {
- val = df_shell_expand(val);
- no_dup = 1;
+ char *expanded = df_shell_expand(val);
+ if (expanded) {
+ val = expanded;
+ no_dup = 1;
+ }
}
}
@@ -421,6 +430,14 @@
/* expand */
file = df_shell_expand(name);
+ if (file == NULL) {
+ if (skip_ok)
+ return;
+
+ fprintf(stderr, "Could not read %s: %s\n", file, strerror(errno));
+ exit(1);
+ }
+
strcpy(cwd, dirname(file));
free(file);
file = df_shell_expand(name);
@@ -472,6 +489,12 @@
logprintf(" Found directive `%s' with argument `%s' on line %i\n", opt,
val, ln);
+ /* Include */
+ if (strcasecmp(opt, "Include") == 0) {
+ DF_DoRunControl(val, cwd, rcd);
+ continue;
+ }
+
/* match directive */
found = 0;
for (i = 0; rcd->options[i].type != DF_OPT_END_OO; ++i)
@@ -620,6 +643,12 @@
DF_Version(rcd, argv[0]);
}
+ /* conf */
+ if (strcmp(&argv[i][2], "conf") == 0) {
+ DF_DoRunControl(value, ".", rcd);
+ continue;
+ }
+
/* match long option */
found = 0;
for (k = 0; rcd->options[k].type != DF_OPT_END_OO; ++k) {
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2012-03-06 01:45:18 UTC (rev 667)
+++ trunk/defile/configure.ac 2012-03-07 04:48:00 UTC (rev 668)
@@ -165,11 +165,29 @@
dnl`
fi
+dnl Calculate a few absolute directory paths (needed by the man pages)
+cat <<EOF > conftest.sh
+prefix=$prefix
+test "x$prefix" = xNONE && prefix=$ac_default_prefix
+exec_prefix=$exec_prefix
+test "x$exec_prefix" = xNONE && exec_prefix=\$prefix
+moduledir=$moduledir
+PACKAGE_TARNAME=$PACKAGE_TARNAME
+echo absolute_moduledir=$moduledir
+echo absolute_sysconfdir=$sysconfdir
+EOF
+
+eval `$SHELL conftest.sh`
+AC_SUBST([absolute_moduledir])
+AC_SUBST([absolute_sysconfdir])
+
echo
echo "*** Writing configure output"
echo
AC_CONFIG_FILES([Makefile])
AC_CONFIG_FILES([bin/Makefile])
+AC_CONFIG_FILES([doc/Makefile])
+AC_CONFIG_FILES([doc/defile.1])
AC_CONFIG_FILES([input/Makefile])
AC_CONFIG_FILES([lib/Makefile])
AC_CONFIG_FILES([test/Makefile])
Added: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am (rev 0)
+++ trunk/defile/doc/Makefile.am 2012-03-07 04:48:00 UTC (rev 668)
@@ -0,0 +1,24 @@
+# ...
[truncated message content] |
|
From: <ket...@us...> - 2012-03-14 20:25:48
|
Revision: 673
http://getdata.svn.sourceforge.net/getdata/?rev=673&view=rev
Author: ketiltrout
Date: 2012-03-14 20:25:41 +0000 (Wed, 14 Mar 2012)
Log Message:
-----------
Some legitimacy for neo-defile: add stuff needed for CHIME data conversion (input plugin elsewhere).
Modified Paths:
--------------
trunk/defile/bin/cli.c
trunk/defile/bin/rc.c
trunk/defile/doc/Makefile.am
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/Makefile.am
trunk/defile/lib/defile.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/bin/cli.c 2012-03-14 20:25:41 UTC (rev 673)
@@ -92,8 +92,8 @@
#define SETULONG(x,y) SETSIMPLE(x,y,strtoul(config->argument, NULL, 0),,\
"%lu")
-#define ABORT_DELAY 60000 /* sixty seconds; this is less than the maximum TCP
- * round-trip timeout. Do we care? */
+#define ABORT_DELAY 60000000 /* sixty seconds; this is less than the maximum TCP
+ * round-trip timeout. Do we care? */
/* the very defile object */
static struct df_defile the_defile;
@@ -107,13 +107,13 @@
if (!df->quiet)
switch (sig) {
case SIGHUP:
- puts("Aborting due to hang up.");
+ fputs("\ndefile: aborting due to hang up.\n", stderr);
break;
case SIGINT:
- puts("Aborting due to interruption.");
+ fputs("\ndefile: aborting due to interruption.\n", stderr);
break;
case SIGTERM:
- puts("Aborting due to unexpected termination.");
+ fputs("\ndefile: aborting due to unexpected termination.\n", stderr);
break;
}
@@ -332,13 +332,19 @@
l->r = 3;
/* run through the search path looking for candidate libraries */
- if (dlp)
+ if (dlp) {
+ if (df->config.mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: searching local paths: %s\n", dlp);
lt_dlforeachfile(dlp, DF_LTDLSearchFunc, l);
+ }
/* the default plugin dir if the above didn't work; or if we're compiling a
* list */
- if ((type == NULL && input == NULL) || l->r != 0)
+ if (type == NULL || l->r != 0) {
+ if (df->config.mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: searching default path: %s\n", DEFILE_MODULEDIR);
lt_dlforeachfile(DEFILE_MODULEDIR, DF_LTDLSearchFunc, l);
+ }
if (print) {
for (i = 0; i < l->n; ++i) {
@@ -608,9 +614,12 @@
} else if (output == NULL) {
char *ptr2, *ptr3;
ptr2 = strdup(input);
- ptr3 = basename(ptr2);
- ptr = malloc(strlen(dir) + strlen(ptr3) + 2);
- strcat(strcat(strcpy(ptr, dir), "/"), ptr3);
+ /* remove forbidden characters */
+ for (ptr3 = ptr2; *ptr3; ++ptr3)
+ if (*ptr3 == '/')
+ *ptr3 = '_';
+ ptr = malloc(strlen(dir) + strlen(ptr2) + 2);
+ sprintf(ptr, "%s/%s", dir, ptr2);
free(ptr2);
} else if (output[0] == '/') {
ptr = strdup(output);
@@ -656,7 +665,7 @@
df->flags = flags;
df->D = gd_open(df->dirfile, GD_RDWR | GD_CREAT | ((flags & DF_CLOBBER) ?
- GD_TRUNC : GD_EXCL) | GD_VERBOSE);
+ (GD_TRUNC | GD_TRUNCSUB) : GD_EXCL) | GD_VERBOSE);
if ((gderr = gd_error(df->D))) {
char *err = gd_error_string(df->D, NULL, 0);
@@ -670,6 +679,9 @@
return 1;
}
+ if (!df->quiet)
+ printf("defile: writing to %s\n\n", df->dirfile);
+
return 0;
}
@@ -778,6 +790,10 @@
/* wait for the input to initialise */
DF_WaitWhileInputState(DF_ST_START, 0);
+ if (df->input_state == DF_ST_DONE || df->input_state == DF_ST_ERROR) {
+ DF_OUTSTATE_MX(DONE);
+ return NULL;
+ }
/* initialise the output */
if (DF_MakeOutput()) {
@@ -1158,6 +1174,8 @@
/* sanitise */
if (df->config.buflen < df->bufhead)
df->config.buflen = df->bufhead;
+ if (df->config.buflen < 2)
+ df->config.buflen = 2;
if (!df->quiet)
puts(PACKAGE_STRING " " DEFILE_COPYRIGHT);
@@ -1168,10 +1186,6 @@
else /* Load the input plugin */
df->input_framework = DF_ResolveInput(df->type);
- if (!df->quiet)
- DF_InputBanner(stdout, NULL, df->input_framework, NULL);
- putchar('\n');
-
/* parse input options */
memset(&rcd, 0, sizeof(rcd));
rcd.contact = df->input_framework->contact;
@@ -1195,6 +1209,10 @@
exit(1);
}
+ if (!df->quiet)
+ DF_InputBanner(stdout, NULL, df->input_framework, NULL);
+ putchar('\n');
+
/* set up signal masks and handlers */
sigemptyset(&signals);
sigaddset(&signals, SIGHUP);
@@ -1232,7 +1250,8 @@
pthread_sigmask(SIG_UNBLOCK, &signals, NULL);
/* wait until the initialisation is done */
- DF_WaitWhileInputState(DF_ST_START, 1);
+ fflush(stdout);
+ DF_WaitWhileInputStates(DF_ST_START, DF_ST_INIT, 1);
DF_WaitWhileInputState(DF_ST_BUILD, 1);
/* record our "start" time */
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/bin/rc.c 2012-03-14 20:25:41 UTC (rev 673)
@@ -225,6 +225,15 @@
int nopt = 0;
const struct df_optdef **opt;
+ if (rcd->name)
+ printf("defile-%s input plugin v%s ", rcd->name, rcd->version);
+ else
+ printf("%s ", PACKAGE_STRING);
+
+ if (rcd->copyright)
+ puts(rcd->copyright);
+ putchar('\n');
+
if (rcd->usage_preamble) {
if (rcd->name == NULL)
printf(rcd->usage_preamble, argv0, argv0);
@@ -271,7 +280,7 @@
static void DF_Version(const struct df_rc_data *rcd, const char *argv0)
{
if (rcd->name)
- printf("defile-%s input plugin v%s\n", rcd->name, rcd->version);
+ printf("\ndefile-%s input plugin v%s\n", rcd->name, rcd->version);
else
puts(PACKAGE_STRING);
Modified: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/doc/Makefile.am 2012-03-14 20:25:41 UTC (rev 673)
@@ -20,5 +20,9 @@
#
AUTOMAKE_OPTIONS = foreign
-man_MANS = defile.1 df_bsearch.3 df_check_abort.3 df_field_in_input.3 \
- df_init.3 df_on_abort.3 defile-input.7
+BUILT_SOURCES=defile.1
+
+man_MANS = defile.1 defile-input.7 df_add_alias.3 df_add_entry.3 \
+ df_add_framedef.3 df_bsearch.3 df_check_abort.3 df_field_in_input.3 \
+ df_init.3 df_input_name.3 df_nframes_allowed.3 df_on_abort.3 \
+ df_push_frame.3 df_ready.3 df_set_const.3 df_shell_expand.3
Modified: trunk/defile/input/ascii.c
===================================================================
--- trunk/defile/input/ascii.c 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/input/ascii.c 2012-03-14 20:25:41 UTC (rev 673)
@@ -279,7 +279,7 @@
}
/* add it */
- n = df_add_framedef(&fd, 1);
+ n = df_add_framedef(&fd, 1, 0);
if (n < 0)
exit(1);
Modified: trunk/defile/input/dirfile.c
===================================================================
--- trunk/defile/input/dirfile.c 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/input/dirfile.c 2012-03-14 20:25:41 UTC (rev 673)
@@ -210,7 +210,7 @@
/* add it and remember its index */
raw[r].name = fd_field.name;
- raw[r].index = df_add_framedef(&fd, 1);
+ raw[r].index = df_add_framedef(&fd, 1, 0);
if (raw[r].index < 0) {
dirfile_cleanup();
return 1;
Modified: trunk/defile/lib/Makefile.am
===================================================================
--- trunk/defile/lib/Makefile.am 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/lib/Makefile.am 2012-03-14 20:25:41 UTC (rev 673)
@@ -21,6 +21,7 @@
AUTOMAKE_OPTIONS = foreign
lib_LTLIBRARIES = libdefile.la
+include_HEADERS = defile.h
AM_CPPFLAGS = ${GETDATA_CFLAGS}
libdefile_la_SOURCES = libdefile.c defile.h internal.h
libdefile_la_LDFLAGS = -export-dynamic -version-info \
Modified: trunk/defile/lib/defile.h
===================================================================
--- trunk/defile/lib/defile.h 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/lib/defile.h 2012-03-14 20:25:41 UTC (rev 673)
@@ -127,10 +127,12 @@
const char *name);
/* can only be used while building the dirfile (BUILD state) */
-int df_add_alias(const char *name, const char *target);
-int df_add_framedef(const struct df_fdef *fdef, int count);
+int df_add_alias(const char *name, const char *target, int fragment);
+int df_add_fragment(const char *subdir, const char *name, unsigned int encoding,
+ unsigned long byte_sex, int parent);
+int df_add_framedef(const struct df_fdef *fdef, int count, int parent);
int df_add_entry(const gd_entry_t *entry);
-int df_add_spec(const char *spec);
+int df_add_spec(const char *spec, int parent);
int df_ready(const char *reference);
/* can only be used while pushing data (RUN state) */
@@ -139,7 +141,7 @@
int df_reinit(unsigned long long nframes, unsigned long long offset,
double rate, const char *name, int mode);
-/* can only be used while building or pushing data (BUILD or RUN state */
+/* can only be used while building or pushing data (BUILD or RUN state) */
int df_set_const(const char *name, gd_type_t type, const void *data_in);
int df_set_string(const char *name, const char *string);
int df_set_carray(const char *name, unsigned int start, size_t len,
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2012-03-14 20:01:01 UTC (rev 672)
+++ trunk/defile/lib/libdefile.c 2012-03-14 20:25:41 UTC (rev 673)
@@ -23,6 +23,8 @@
#include "config.h"
#endif
#include "internal.h"
+#include <sys/types.h>
+#include <sys/stat.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
@@ -127,13 +129,13 @@
return ((missing && !df->ofl_incl) || (!missing && df->ofl_incl));
}
-int df_add_alias(const char *name, const char *target)
+int df_add_alias(const char *name, const char *target, int fragment)
{
DF_CHECK_SEQUENCE(DF_ST_BUILD);
/* skip fields we've been asked to exclude */
if (df_field_in_output(name))
- if (gd_add_alias(df->D, name, target, 0))
+ if (gd_add_alias(df->D, name, target, fragment))
return DF_OUTPUT;
return DF_SUCCESS;
@@ -145,7 +147,8 @@
/* skip RAW fields */
if (E->field_type == GD_RAW_ENTRY) {
- fputs("libdefile: Attempt to define RAW field via df_add_entry().", stderr);
+ fputs("libdefile: Attempt to define RAW field via df_add_entry().\n",
+ stderr);
return DF_INPUT;
}
@@ -157,7 +160,7 @@
return DF_SUCCESS;
}
-int df_add_spec(const char *spec)
+int df_add_spec(const char *spec, int fragment)
{
char *ptr;
DF_CHECK_SEQUENCE(DF_ST_BUILD);
@@ -181,14 +184,15 @@
/* check the second token to see if it's a RAW field */
ptr = gd_tokenise(df->D, NULL);
- if (ptr && strcmp(ptr, "RAW")) {
- fputs("libdefile: Attempt to define RAW field via df_add_spec().", stderr);
+ if (ptr && strcmp(ptr, "RAW") == 0) {
+ fputs("libdefile: Attempt to define RAW field via df_add_spec().\n",
+ stderr);
return DF_INPUT;
}
free(ptr);
/* add it */
- if (gd_add_spec(df->D, spec, 0))
+ if (gd_add_spec(df->D, spec, fragment))
return DF_OUTPUT;
return DF_SUCCESS;
@@ -245,7 +249,8 @@
}
#define DF_NO_RAW (-9999)
-static int df_set_raw(const struct df_fdef_field *f, int i, int count)
+static int df_set_raw(const struct df_fdef_field *f, int i, int count,
+ int fragment)
{
int r, is_new = 0;
void *ptr;
@@ -262,17 +267,17 @@
char *name = strdup(f->name);
if (name == NULL) {
- perror("libdefile");
+ perror("libdefile: strdup");
return DF_SYSTEM;
}
/* try adding it to the dirfile */
- if (gd_add_raw(df->D, f->name, f->type, f->spf, 0))
+ if (gd_add_raw(df->D, f->name, f->type, f->spf, fragment))
return DF_OUTPUT;
/* add a new record */
ptr = realloc(df->raw, sizeof(struct df_raw) * (df->nraw + 1));
if (ptr == NULL) {
- perror("libdefile");
+ perror("libdefile: realloc");
return DF_SYSTEM;
}
df->raw = (struct df_raw*)ptr;
@@ -305,7 +310,7 @@
df->raw[r].out = df->raw[r].buflen - df->raw[r].framesize;
ptr = realloc(df->raw[r].b, df->raw[r].buflen);
if (ptr == NULL) {
- perror("libdefile");
+ perror("libdefile: realloc");
return DF_SYSTEM;
}
df->raw[r].b = ptr;
@@ -313,7 +318,7 @@
return r;
}
-int df_add_framedef(const struct df_fdef *fd, int count)
+int df_add_framedef(const struct df_fdef *fd, int count, int fragment)
{
size_t c;
unsigned int i;
@@ -367,7 +372,7 @@
/* copy the framedef into an internal framedef record */
ptr = realloc(df->fd, sizeof(struct df_infdef) * (df->nfd + 1));
if (ptr == NULL) {
- perror("libdefile");
+ perror("libdefile: realloc");
return DF_SYSTEM;
}
df->fd = (struct df_infdef*)ptr;
@@ -381,11 +386,11 @@
(struct df_infdef_field *)malloc(sizeof(struct df_infdef_field) *
fd->n_fields);
if (field == NULL) {
- perror("libdefile");
+ perror("libdefile: malloc");
return DF_SYSTEM;
}
for (i = 0; i < fd->n_fields; ++i) {
- field[i].index = df_set_raw(fd->field + i, i, count);
+ field[i].index = df_set_raw(fd->field + i, i, count, fragment);
if (field[i].index < 0) {
/* check for error */
if (field[i].index != DF_NO_RAW) {
@@ -454,11 +459,12 @@
free(df->input);
df->input = strdup(name);
if (df->input == NULL) {
- perror("libdefile");
+ perror("libdefile: strdup");
return DF_SYSTEM;
}
}
- df->rate = rate;
+ if (rate >= 0)
+ df->rate = rate;
df->nframes = (uint64_t)nframes;
df->offset = (uint64_t)offset;
@@ -687,7 +693,7 @@
free(df->input);
df->input = strdup(name);
if (df->input == NULL) {
- perror("libdefile");
+ perror("libdefile: strdup");
return DF_SYSTEM;
}
}
@@ -758,3 +764,55 @@
return DF_SUCCESS;
}
+
+int df_add_fragment(const char *subdir, const char *name, unsigned int encoding,
+ unsigned long byte_sex, int parent)
+{
+ int ind;
+ struct stat stat_buf;
+ char *subdirpath;
+
+ DF_CHECK_SEQUENCE(DF_ST_BUILD);
+
+ /* make the subdir, if needed */
+ if (subdir) {
+ subdirpath = malloc(strlen(df->dirfile) + strlen(subdir) + 2);
+ if (subdirpath == NULL) {
+ perror("libdefile: malloc");
+ return DF_SYSTEM;
+ }
+ sprintf(subdirpath, "%s/%s", df->dirfile, subdir);
+ if (stat(subdirpath, &stat_buf) == 0)
+ if (S_ISDIR(stat_buf.st_mode))
+ goto DF_HAVE_DIR;
+
+ if (mkdir(subdirpath, 0777) == 0)
+ goto DF_HAVE_DIR;
+ free(subdirpath);
+
+ fprintf(stderr, "libdefile: unable to create output subdirectory %s\n",
+ subdir);
+ return DF_OUTPUT;
+
+DF_HAVE_DIR:
+ free(subdirpath);
+ subdirpath = malloc(strlen(subdir) + strlen(name) + 2);
+ sprintf(subdirpath, "%s/%s", subdir, name);
+ } else
+ subdirpath = strdup(name);
+
+ if (subdirpath == NULL) {
+ perror("libdefile: strdup");
+ return DF_SYSTEM;
+ }
+
+ /* attempt to add the new fragment */
+ ind = gd_include(df->D, subdirpath, 0,
+ GD_CREAT | GD_EXCL | encoding | byte_sex);
+ free(subdirpath);
+
+ if (ind < 0)
+ return DF_OUTPUT;
+
+ return ind;
+}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2012-03-22 23:01:58
|
Revision: 684
http://getdata.svn.sourceforge.net/getdata/?rev=684&view=rev
Author: ketiltrout
Date: 2012-03-22 23:01:51 +0000 (Thu, 22 Mar 2012)
Log Message:
-----------
Append mode. Also, be threadsafe.
Modified Paths:
--------------
trunk/defile/bin/cli.c
trunk/defile/input/Makefile.am
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/defile.h
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/bin/cli.c 2012-03-22 23:01:51 UTC (rev 684)
@@ -25,6 +25,7 @@
#include "internal.h"
+#include <errno.h>
#include <signal.h>
#include <stdlib.h>
#include <stdio.h>
@@ -87,20 +88,23 @@
#define CLRSTRING(x,y) SETSTATE(x,y,free(df-> y),df ->y = NULL,"%s")
#define SETFLOAT(x,y) SETSIMPLE(x,y,atof(config->argument),,"%g")
#define SETLFLOAT(x,y,b) SETSIMPLE(x,y,b,,"%g" )
-#define SETINT(x,y) SETSIMPLE(x,y,strol(config->argument, NULL, 0),,"%i")
+#define SETINT(x,y) SETSIMPLE(x,y,strtol(config->argument, NULL, 0),,"%i")
#define SETLINT(x,y,b) SETSIMPLE(x,y,b,,"%i" )
#define SETULONG(x,y) SETSIMPLE(x,y,strtoul(config->argument, NULL, 0),,\
"%lu")
+#define SETULLONG(x,y) SETSIMPLE(x,y,strtoull(config->argument, NULL, 0),,\
+ "%llu")
-#define ABORT_DELAY 60000000 /* sixty seconds; this is less than the maximum TCP
- * round-trip timeout. Do we care? */
+#define ABORT_DELAY 10000000 /* ten seconds */
/* the very defile object */
static struct df_defile the_defile;
struct df_defile *const df = &the_defile;
-/* state mutex */
+/* mutices */
pthread_mutex_t state_mx = PTHREAD_MUTEX_INITIALIZER;
+pthread_mutex_t config_mx = PTHREAD_MUTEX_INITIALIZER;
+pthread_mutex_t rate_mx = PTHREAD_MUTEX_INITIALIZER;
static void DF_SignalHandler(int sig)
{
@@ -264,7 +268,7 @@
result = 4;
lt_dlclose(lib);
}
- if (df->config.mode_flags & DF_MODE_DEBUG) {
+ if (df->mode_flags & DF_MODE_DEBUG) {
fprintf(stderr, "defile: probing %s; result: %s\n", filename,
(result) ? "fail" : "pass");
}
@@ -333,7 +337,7 @@
/* run through the search path looking for candidate libraries */
if (dlp) {
- if (df->config.mode_flags & DF_MODE_DEBUG)
+ if (df->mode_flags & DF_MODE_DEBUG)
fprintf(stderr, "defile: searching local paths: %s\n", dlp);
lt_dlforeachfile(dlp, DF_LTDLSearchFunc, l);
}
@@ -341,7 +345,7 @@
/* the default plugin dir if the above didn't work; or if we're compiling a
* list */
if (type == NULL || l->r != 0) {
- if (df->config.mode_flags & DF_MODE_DEBUG)
+ if (df->mode_flags & DF_MODE_DEBUG)
fprintf(stderr, "defile: searching default path: %s\n", DEFILE_MODULEDIR);
lt_dlforeachfile(DEFILE_MODULEDIR, DF_LTDLSearchFunc, l);
}
@@ -503,11 +507,13 @@
}
enum {
- DF_WOPT_ADD = 1, DF_WOPT_ASYNC, DF_WOPT_AUTOTYPE, DF_WOPT_BHEAD, DF_WOPT_BLEN,
- DF_WOPT_CLOBBER, DF_WOPT_DEBUG, DF_WOPT_DEL, DF_WOPT_DIR, DF_WOPT_GZIP,
- DF_WOPT_INPUT, DF_WOPT_LIBDIR, DF_WOPT_LIST, DF_WOPT_NOCLOBBER,
- DF_WOPT_NODEBUG, DF_WOPT_NOFOLLOW, DF_WOPT_OFLCLR, DF_WOPT_OUTPUT,
- DF_WOPT_FOLLOW, DF_WOPT_QUIET, DF_WOPT_SYNC, DF_WOPT_TYPE, DF_WOPT_VERBOSE
+ DF_WOPT_ADD = 1, DF_WOPT_APPEND, DF_WOPT_ASYNC, DF_WOPT_AUTOTYPE,
+ DF_WOPT_BHEAD, DF_WOPT_BLEN, DF_WOPT_CLOBBER, DF_WOPT_CUTDIR, DF_WOPT_DEBUG,
+ DF_WOPT_DEL, DF_WOPT_DIR, DF_WOPT_FOLLOW, DF_WOPT_GZIP, DF_WOPT_INPUT,
+ DF_WOPT_LIBDIR, DF_WOPT_LIST, DF_WOPT_NOAPPEND, DF_WOPT_NOCLOBBER,
+ DF_WOPT_NOCUTDIR, DF_WOPT_NODEBUG, DF_WOPT_NOFOLLOW, DF_WOPT_OFLCLR,
+ DF_WOPT_OUTPUT, DF_WOPT_QUIET, DF_WOPT_SKIP, DF_WOPT_SYNC, DF_WOPT_TYPE,
+ DF_WOPT_VERBOSE
};
static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
@@ -572,27 +578,35 @@
return 0;
+ SETBIT(APPEND, mode_flags, DF_MODE_APPEND);
+ CLRBIT(NOAPPEND, mode_flags, DF_MODE_APPEND);
+
SETULONG(BHEAD, bufhead);
- SETULONG(BLEN, config.buflen);
+ SETULONG(BLEN, buflen);
SETLINT(CLOBBER, clobber, 1);
SETLINT(NOCLOBBER, clobber, 0);
+ SETINT(CUTDIR, cutdir);
+ SETLINT(NOCUTDIR, cutdir, 0);
+
SETSTRING(DIR, dir);
SETSTRING(OUTPUT, output);
- SETBIT(FOLLOW, config.mode_flags, DF_MODE_FOLLOW);
- CLRBIT(NOFOLLOW, config.mode_flags, DF_MODE_FOLLOW);
+ SETBIT(FOLLOW, mode_flags, DF_MODE_FOLLOW);
+ CLRBIT(NOFOLLOW, mode_flags, DF_MODE_FOLLOW);
SETLINT(QUIET, quiet, 1);
SETLINT(VERBOSE, quiet, 0);
- SETBIT(DEBUG, config.mode_flags, DF_MODE_DEBUG);
- CLRBIT(NODEBUG, config.mode_flags, DF_MODE_DEBUG);
+ SETBIT(DEBUG, mode_flags, DF_MODE_DEBUG);
+ CLRBIT(NODEBUG, mode_flags, DF_MODE_DEBUG);
- SETBIT(ASYNC, config.mode_flags, DF_MODE_ASYNC);
- CLRBIT(SYNC, config.mode_flags, DF_MODE_ASYNC);
+ SETULLONG(SKIP, offset);
+ SETBIT(ASYNC, mode_flags, DF_MODE_ASYNC);
+ CLRBIT(SYNC, mode_flags, DF_MODE_ASYNC);
+
SETSTRING(TYPE, type);
CLRSTRING(AUTOTYPE, type);
}
@@ -612,15 +626,27 @@
} else
ptr = strdup(output);
} else if (output == NULL) {
- char *ptr2, *ptr3;
- ptr2 = strdup(input);
+ int cutleft = df->cutdir;
+ char *ptr_base, *ptr2, *ptr3;
+ ptr_base = strdup(input);
+
+ /* cut directories */
+ if (cutleft == 0)
+ ptr2 = ptr_base;
+ else {
+ cutleft = df->cutdir;
+ for (ptr2 = ptr_base + strlen(ptr_base); ptr2 > ptr_base; --ptr2)
+ if (*(ptr2 - 1) == '/')
+ if (--cutleft == 0)
+ break;
+ }
/* remove forbidden characters */
for (ptr3 = ptr2; *ptr3; ++ptr3)
if (*ptr3 == '/')
*ptr3 = '_';
ptr = malloc(strlen(dir) + strlen(ptr2) + 2);
sprintf(ptr, "%s/%s", dir, ptr2);
- free(ptr2);
+ free(ptr_base);
} else if (output[0] == '/') {
ptr = strdup(output);
} else {
@@ -658,29 +684,61 @@
return 0;
}
-static int DF_CreateOutput(unsigned long flags)
+static int DF_CreateOutput(int clobber, int append)
{
- int gderr;
+ int new = 1;
- df->flags = flags;
+ /* check wether it already exists */
+ df->D = gd_open(df->dirfile, GD_RDWR | GD_CREAT | GD_EXCL);
- df->D = gd_open(df->dirfile, GD_RDWR | GD_CREAT | ((flags & DF_CLOBBER) ?
- (GD_TRUNC | GD_TRUNCSUB) : GD_EXCL) | GD_VERBOSE);
+ /* abort on error */
+ if (df->D == NULL) {
+ fprintf(stderr, "defile: unexplained catastrophe attempting to create %s\n",
+ df->dirfile);
+ return 1;
+ } else if (gd_error(df->D) == GD_E_EXISTS && (clobber || append)) {
+ new = 0;
+ /* destroy invalid object and reopen */
+ gd_discard(df->D);
+ df->D = gd_open(df->dirfile, GD_RDWR |
+ (append ? 0 : GD_TRUNC | GD_TRUNCSUB));
- if ((gderr = gd_error(df->D))) {
- char *err = gd_error_string(df->D, NULL, 0);
- fprintf(stderr, "defile: getdata error: %s\n", err);
- free(err);
- if (gderr == GD_E_CREAT) { /* no clobber: allow the input to try again */
- pthread_mutex_lock(&state_mx);
- DF_OUTSTATE(INIT);
- pthread_mutex_unlock(&state_mx);
- } else
+ if (df->D == NULL) {
+ fprintf(stderr,
+ "defile: unexplained catastrophe attempting to create %s\n",
+ df->dirfile);
return 1;
+ }
+ } else if (gd_error(df->D) == GD_E_OK) {
+ pthread_mutex_lock(&config_mx);
+ df->mode_flags &= ~DF_MODE_APPEND;
+ pthread_mutex_unlock(&config_mx);
+ append = 0;
}
+ if (gd_error(df->D)) {
+ char *estring = gd_error_string(df->D, NULL, 0);
+ fprintf(stderr, "defile: libgetdata: %s\n", estring);
+ gd_discard(df->D);
+ free(estring);
+ return 1;
+ }
+
+ /* set up GetData's verbose mode */
+ gd_flags(df->D, GD_VERBOSE, 0);
+ gd_verbose_prefix(df->D, "defile: ");
+
+ if (append) {
+ unsigned long long nf = gd_nframes(df->D);
+ pthread_mutex_lock(&config_mx);
+ if (df->offset < nf)
+ df->offset = nf;
+ pthread_mutex_unlock(&config_mx);
+ }
+
if (!df->quiet)
- printf("defile: writing to %s\n\n", df->dirfile);
+ printf("defile: %s %s\n\n", append ? "appending to" : new ? "created" :
+ "overwriting", df->dirfile);
return 0;
}
@@ -731,7 +789,7 @@
/* wait while the input does something */
static inline void DF_WaitWhileInputStates(int state1, int state2, int quiet)
{
- if (!quiet && df->config.mode_flags & DF_MODE_DEBUG)
+ if (!quiet && df_mode() & DF_MODE_DEBUG)
fprintf(stderr, "defile: output waiting on input for end of %s and %s\n",
DF_STATENAME(state1), DF_STATENAME(state2));
while (df->input_state == state1 || df->input_state == state2)
@@ -741,7 +799,7 @@
/* wait while the input does something */
static inline void DF_WaitWhileInputState(int state, int quiet)
{
- if (!quiet && df->config.mode_flags & DF_MODE_DEBUG)
+ if (!quiet && df_mode() & DF_MODE_DEBUG)
fprintf(stderr, "defile: output waiting on input for end of %s\n",
DF_STATENAME(state));
while (df->input_state == state)
@@ -756,7 +814,7 @@
if (!df->dirfile)
return 1;
- if (DF_CreateOutput(df->clobber ? DF_CLOBBER : 0))
+ if (DF_CreateOutput(df->clobber, df_mode() & DF_MODE_APPEND))
return 1;
if (df->output_state == DF_ST_ABORT)
@@ -766,7 +824,7 @@
DF_OUTSTATE_MX(BUILD);
/* wait for the input to build the dirfile */
- DF_WaitWhileInputStates(DF_ST_BUILD, DF_ST_INIT, 0);
+ DF_WaitWhileInputState(DF_ST_BUILD, 0);
if (df->output_state == DF_ST_ABORT)
return 1;
@@ -783,10 +841,11 @@
/* output thread */
static void *DF_OutputThread(void)
{
- int i, j, last_pass = 0;
+ int i, j, last_pass = 0, wait = 1;
size_t nwrote;
struct df_raw *r;
- unsigned long in, out, nf;
+ unsigned long nf, out;
+ const int debug = df_mode() & DF_MODE_DEBUG;
/* wait for the input to initialise */
DF_WaitWhileInputState(DF_ST_START, 0);
@@ -803,6 +862,7 @@
/* loop through raws, writing ones that have full buffer heads */
while (!last_pass) {
+ wait = 1;
if (df->input_state != DF_ST_RUN)
last_pass = 1;
@@ -818,7 +878,8 @@
if (DF_OutputFini(delete)) {
DF_OUTSTATE_MX(ERROR);
return NULL;
- } else if (DF_MakeOutput()) {
+ }
+ if (DF_MakeOutput()) {
DF_OUTSTATE_MX(ERROR);
return NULL;
}
@@ -836,26 +897,45 @@
}
}
r = df->raw + i;
- in = r->in;
+
+ /* skip this field for now, if we can't lock it */
+ if (pthread_mutex_trylock(&r->mx) == EBUSY) {
+ if (debug)
+ fprintf(stderr, "\nlockmiss on %s\n", r->name);
+ continue;
+ }
+
out = r->out;
- nf = (in == out) ? df->config.buflen - 1 :
- ((in + r->buflen - out) % r->buflen) / r->framesize - 1;
- if (nf == 0 || (!last_pass && nf <= df->bufhead))
+ nf = (r->in == out) ? df->buflen - 1 :
+ ((r->in + r->buflen - out) % r->buflen) / r->framesize - 1;
+
+ if (nf == 0 || (!last_pass && nf <= df->bufhead)) {
+ pthread_mutex_unlock(&r->mx);
continue;
+ }
+
+ if (!(df_mode() & DF_MODE_ASYNC)) {
+ /* in synchronous mode, the reference field is special: we only write
+ * frames to it if all other fields have those frames committed */
+ if (i == df->ref_ind) {
+ /* adjust for shorter sticks */
+ for (j = 0; j < df->nraw; ++j)
+ if (j != df->ref_ind)
+ if (nf > df->raw[j].ahead)
+ nf = df->raw[j].ahead;
+ } else {
+ /* in synchronous mode, we never want to get more than one buflen
+ * ahead of the referenc efield */
+ if (r->ahead + nf >= df->buflen)
+ nf = df->buflen - r->ahead - 1;
+ }
- /* in synchronous mode, the reference field is special: we only write
- * frames to it if all other fields have those frames committed */
- if (i == df->ref_ind && !(df->config.mode_flags & DF_MODE_ASYNC)) {
- /* adjust for shorter sticks */
- for (j = 0; j < df->nraw; ++j)
- if (j != df->ref_ind)
- if (nf > df->raw[j].ahead)
- nf = df->raw[j].ahead;
-
/* check again */
- if (nf == 0 || (!last_pass && nf <= df->bufhead))
+ if (nf == 0 || (!last_pass && nf <= df->bufhead)) {
+ pthread_mutex_unlock(&r->mx);
continue;
+ }
}
/* point the local output pointer to the next datum to write */
@@ -876,6 +956,7 @@
/* bail on error */
if (gd_error(df->D)) {
DF_OUTSTATE_MX(ERROR);
+ pthread_mutex_unlock(&r->mx);
return NULL;
}
@@ -886,12 +967,13 @@
if (j != df->ref_ind) /* XXX */
df->raw[j].ahead -= nf;
} else
- r->ahead += nwrote / r->spf;
+ r->ahead += nf;
if (nwrote > 0) {
last_pass = 0;
- if (df->config.mode_flags & DF_MODE_DEBUG) {
- if (i != df->ref_ind && !(df->config.mode_flags & DF_MODE_ASYNC))
+ wait = 0;
+ if (debug) {
+ if (i != df->ref_ind && !(df_mode() & DF_MODE_ASYNC))
fprintf(stderr, "defile: output %lu %s of field %s "
"(lead is %lu %s)\n",
nf, (nf == 1) ? "frame" : "frames", r->name, r->ahead,
@@ -904,15 +986,19 @@
/* update write pointer */
r->out = (r->out + nwrote * GD_SIZE(r->type)) % r->buflen;
+ pthread_mutex_unlock(&r->mx);
/* update write counts */
+ pthread_mutex_lock(&rate_mx);
df->wpartial += nwrote * GD_SIZE(r->type);
while (df->wpartial >= df->wframesize) {
df->nwrote++;
df->wpartial -= df->wframesize;
}
+ pthread_mutex_unlock(&rate_mx);
}
- usleep(10000);
+ if (wait)
+ usleep(10000);
}
/* finished */
@@ -947,16 +1033,19 @@
static unsigned long long lw = 0;
static int init = 2;
- const char eol = (df->config.mode_flags & DF_MODE_DEBUG) ? '\n' : '\r';
+ const char eol = (df_mode() & DF_MODE_DEBUG) ? '\n' : '\r';
double delta, drate;
- unsigned long long nf;
- const unsigned long long nw = df->nwrote;
- const unsigned long long nr = df->nread;
+ unsigned long long nf, nw, nr;
char si;
struct timezone tz;
struct timeval now;
+ pthread_mutex_lock(&rate_mx);
+ nw = df->nwrote;
+ nr = df->nread;
+ pthread_mutex_unlock(&rate_mx);
+
if (df->quiet)
return rate;
@@ -1063,6 +1152,12 @@
struct sigaction action;
const struct df_optdef options[] = {
+ { DF_WOPT_APPEND, DF_OPT_NO_ARG, 'A', "append", "Append", 0, NULL,
+ "append data to an already existing output"
+ },
+ { DF_WOPT_NOAPPEND, DF_OPT_NO_ARG, 'A', "no-append", "NoAppend", 0, NULL,
+ "don't append data (default)"
+ },
{ DF_WOPT_ASYNC, DF_OPT_NO_ARG, 'a', "asyncrhonous", "Asynchronous", 0,
NULL, "operate in asynchronous mode"
},
@@ -1075,6 +1170,12 @@
{ DF_WOPT_NOCLOBBER, DF_OPT_NO_ARG, 0, "no-clobber", "NoClobber", 0, NULL,
"don't overwrite an existing dirfile (default)"
},
+ { DF_WOPT_CUTDIR, DF_OPT_ARG_RQ, '0', "cut-dirs", "CutDirs", 0, "NUM",
+ "remove all but the last NUM path elements from the output name"
+ },
+ { DF_WOPT_NOCUTDIR, DF_OPT_NO_ARG, '0', "no-cut-dirs", "NpCutDirs", 0, NULL,
+ "don't remove path elements from the output name (default)"
+ },
{ DF_WOPT_DIR, DF_OPT_ARG_RQ, 'D', "directory", "Directory", DF_OPT_EXPAND,
"DIR", "write the output dirfile as a subdirectory under DIR"
},
@@ -1084,6 +1185,14 @@
{ DF_WOPT_NODEBUG, DF_OPT_NO_ARG, 'd', "no-debug", "NoDebug", DF_OPT_PLUS,
NULL, "suppress printing debugging messages (default)"
},
+ { DF_WOPT_FOLLOW, DF_OPT_NO_ARG, 'f', "follow", "Follow", 0, NULL,
+ "keep monitoring the input and write data to the outpu as it becomes "
+ "available"
+ },
+ { DF_WOPT_NOFOLLOW, DF_OPT_NO_ARG, 'f', "no-follow", "NoFollow",
+ DF_OPT_PLUS, NULL,
+ "exit successfully upon reaching the end of the input (default)"
+ },
{ DF_WOPT_ADD, DF_OPT_ARG_RQ, 'F', "include", "IncludeField", 0, "FIELD",
"include FIELD in the output dirfile, if available"
},
@@ -1117,29 +1226,28 @@
{ DF_WOPT_OUTPUT, DF_OPT_ARG_RQ, 'o', "output", "Output", DF_OPT_EXPAND,
"PATH", "write data to a dirfile called PATH"
},
- { DF_WOPT_FOLLOW, DF_OPT_NO_ARG, 'f', "follow", "Follow", 0, NULL,
- "keep monitoring the input and write data to the outpu as it becomes "
- "available"
- },
- { DF_WOPT_NOFOLLOW, DF_OPT_NO_ARG, 'f', "no-follow", "NoFollow",
- DF_OPT_PLUS, NULL,
- "exit successfully upon reaching the end of the input (default)"
- },
{ DF_WOPT_QUIET, DF_OPT_NO_ARG, 'q', "quiet", "Quiet", 0, NULL,
"be less verbose"
},
{ DF_WOPT_VERBOSE, DF_OPT_NO_ARG, 'q', "verbose", "Verbose", DF_OPT_PLUS,
NULL, "be normally verbose"
},
+#if 0
+ { DF_WOPT_SKIP, DF_OPT_ARG_RQ, 's', "skip", "SkipFrames", 0, "NUM",
+ "skip the first NUM frames of the input (default: 0)"
+ },
+#endif
{ DF_WOPT_TYPE, DF_OPT_ARG_RQ, 't', "type", "InputType", 0, "TYPE",
"use TYPE as the input data type"
},
{ DF_WOPT_AUTOTYPE, DF_OPT_NO_ARG, 't', "autotype", "AutoType", DF_OPT_PLUS,
NULL, "attempt to automatically determine the data type of the input"
},
- // { DF_WOPT_GZIP, DF_OPT_NO_ARG, 'z', "gzip", "GZip", 0, NULL,
- // "gzip compress the output raw data"
- // },
+#if 0
+ { DF_WOPT_GZIP, DF_OPT_NO_ARG, 'z', "gzip", "GZip", 0, NULL,
+ "gzip compress the output raw data"
+ },
+#endif
DF_OPT_ENDOPT
};
@@ -1163,7 +1271,7 @@
df->iargv = malloc(sizeof(char*));
df->iargv[0] = argv[0];
df->iargc = 1;
- df->config.buflen = DF_BUFLEN;
+ df->buflen = DF_BUFLEN;
/* initialise LTDL and register its cleanup function */
lt_dlinit();
@@ -1172,10 +1280,10 @@
DF_Parse(argc, argv, &rcd, df);
/* sanitise */
- if (df->config.buflen < df->bufhead)
- df->config.buflen = df->bufhead;
- if (df->config.buflen < 2)
- df->config.buflen = 2;
+ if (df->buflen < df->bufhead)
+ df->buflen = df->bufhead;
+ if (df->buflen < 2)
+ df->buflen = 2;
if (!df->quiet)
puts(PACKAGE_STRING " " DEFILE_COPYRIGHT);
@@ -1251,7 +1359,7 @@
/* wait until the initialisation is done */
fflush(stdout);
- DF_WaitWhileInputStates(DF_ST_START, DF_ST_INIT, 1);
+ DF_WaitWhileInputState(DF_ST_START, 1);
DF_WaitWhileInputState(DF_ST_BUILD, 1);
/* record our "start" time */
Modified: trunk/defile/input/Makefile.am
===================================================================
--- trunk/defile/input/Makefile.am 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/input/Makefile.am 2012-03-22 23:01:51 UTC (rev 684)
@@ -20,11 +20,10 @@
#
AUTOMAKE_OPTIONS = foreign
+LDFLAGS = -module -shared -release $(PACKAGE_VERSION)
module_LTLIBRARIES = libdefile-dirfile.la libdefile-ascii.la
libdefile_dirfile_la_SOURCES = dirfile.c
-libdefile_dirfile_la_LDFLAGS = -module
libdefile_dirfile_la_LIBADD = ../lib/libdefile.la
libdefile_ascii_la_SOURCES = ascii.c
-libdefile_ascii_la_LDFLAGS = -module
libdefile_ascii_la_LIBADD = ../lib/libdefile.la
Modified: trunk/defile/input/ascii.c
===================================================================
--- trunk/defile/input/ascii.c 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/input/ascii.c 2012-03-22 23:01:51 UTC (rev 684)
@@ -216,7 +216,7 @@
}
/* in non-persistant mode, count lines */
- if (!(config->mode_flags & DF_MODE_FOLLOW)) {
+ if (!(df_mode() & DF_MODE_FOLLOW)) {
while ((n = fgetc(s)) != EOF)
if (ferror(s)) {
perror("defile-ascii: read error");
@@ -322,7 +322,7 @@
}
/* in persistant mode, wait a bit, and then go back and try again */
- if (config->mode_flags & DF_MODE_FOLLOW)
+ if (df_mode() & DF_MODE_FOLLOW)
usleep(100000);
else /* otherwise, we're done */
break;
Modified: trunk/defile/input/dirfile.c
===================================================================
--- trunk/defile/input/dirfile.c 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/input/dirfile.c 2012-03-22 23:01:51 UTC (rev 684)
@@ -156,7 +156,7 @@
unsigned int r = 0;
ssize_t nw;
const char **field;
- const int debug = config->mode_flags & DF_MODE_DEBUG;
+ const int debug = df_mode() & DF_MODE_DEBUG;
void *buffer = NULL;
/* set the abort handler */
@@ -170,7 +170,7 @@
/* In non-persist mode, it's useful to tell the writer how many frames we
* have */
- if (!(config->mode_flags & DF_MODE_FOLLOW))
+ if (!(df_mode() & DF_MODE_FOLLOW))
nframes = (uint64_t)gd_nframes(D);
/* Initialise -- dirfiles don't contain rate information so we leave that
@@ -333,7 +333,7 @@
} while (have_data);
/* in persistant mode, wait a bit, and then go back and try again */
- if (config->mode_flags & DF_MODE_FOLLOW)
+ if (df_mode() & DF_MODE_FOLLOW)
usleep(100000);
else /* otherwise, we're done */
break;
Modified: trunk/defile/lib/defile.h
===================================================================
--- trunk/defile/lib/defile.h 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/lib/defile.h 2012-03-22 23:01:51 UTC (rev 684)
@@ -77,14 +77,13 @@
};
/* mode flags */
-#define DF_MODE_ASYNC 0x1 /* asynchronous writing */
-#define DF_MODE_FOLLOW 0x2 /* follow mode */
+#define DF_MODE_APPEND 0x1 /* append mode */
+#define DF_MODE_ASYNC 0x2 /* asynchronous writing */
#define DF_MODE_DEBUG 0x4 /* debug mode */
+#define DF_MODE_FOLLOW 0x8 /* follow mode */
/* defile configuration */
struct df_config {
- unsigned int mode_flags;
- unsigned long buflen;
int n_config;
int *value;
const char **arg;
@@ -153,6 +152,8 @@
int df_check_abort(void);
int df_field_in_output(const char *name);
const char *df_input_name(int shell_expand);
+unsigned int df_mode(void);
+unsigned long long df_offset(void);
df_abort_func_t df_on_abort(df_abort_func_t func);
char *df_shell_expand(const char *s);
int df_strcmp(const void *needle, const void *candidate);
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/lib/internal.h 2012-03-22 23:01:51 UTC (rev 684)
@@ -26,6 +26,7 @@
#include <getdata.h>
#include <syslog.h>
+#include <pthread.h>
#ifdef HAVE_LTDL_H
#include <ltdl.h>
@@ -41,7 +42,7 @@
/* maximum allowed length of a input type name */
#define DF_TYPE_MAXLEN 256
-/* Plugin status values */
+/* plugin state values */
#define DF_ST_ERROR -1 /* thread reported an error */
#define DF_ST_START 0 /* thread start */
#define DF_ST_BUILD 1 /* building the frame */
@@ -49,8 +50,7 @@
#define DF_ST_DONE 3 /* finished */
#define DF_ST_ABORT 4 /* thread must abort processing and exit */
-/* non-public (ie. output-only) states */
-#define DF_ST_INIT 5 /* the output couldn't initialise */
+/* special states */
#define DF_ST_SAVESTART 6 /* save the current output and restart (OUT only) */
#define DF_ST_DROPSTART 7 /* drop the current output and restart (OUT only) */
@@ -59,7 +59,7 @@
#define DF_SETSTATE(w,n,s) \
do { \
df-> w ## _state = DF_ST_ ## s; \
- if (df->config.mode_flags & DF_MODE_DEBUG) \
+ if (df->mode_flags & DF_MODE_DEBUG) \
fputs("defile: " n " state: " #s "\n", stderr); \
} while (0)
#define DF_SETSTATE_MX(w,n,s) \
@@ -80,7 +80,6 @@
((i) == DF_ST_DONE) ? "DONE" : \
((i) == DF_ST_ABORT) ? "ABORT" : \
((i) == DF_ST_ERROR) ? "ERROR" : \
- ((i) == DF_ST_INIT) ? "INIT" : \
((i) == DF_ST_DROPSTART) ? "DROPSTART" : \
((i) == DF_ST_SAVESTART) ? "SAVESTART" : \
"UNKNOWN")
@@ -97,6 +96,7 @@
unsigned long ahead; /* difference between the number of frames written to
this raw and the reference -- only used in synchronous
mode */
+ pthread_mutex_t mx; /* buffer mutex */
};
/* internal framedef */
@@ -115,14 +115,19 @@
};
struct df_defile {
- unsigned long flags;
+ /* the input plugin */
+ lt_dlhandle inhandle;
+ const struct df_input_framework *input_framework;
+
+ /* pathname handling */
char *input;
- lt_dlhandle inhandle;
char *dir;
char *type;
char *output;
char *dirfile;
+ int cutdir;
+ /* threadkeeping */
pthread_t input_id;
int input_state;
pthread_t output_id;
@@ -143,10 +148,15 @@
unsigned long long nwrote, nread;
unsigned long long nframes;
+ /* operations */
struct df_config config;
+ unsigned long long offset;
+ unsigned long buflen;
+ unsigned int mode_flags;
unsigned long bufhead;
+ /* output buffering */
int nraw;
struct df_raw *raw;
@@ -157,18 +167,18 @@
int ref_ind;
/* parser stuff */
- const struct df_input_framework *input_framework;
int mode_switch;
int quiet;
int clobber;
int iargc;
char **iargv;
- unsigned long long offset;
DIRFILE *D;
};
extern struct df_defile *const df;
extern pthread_mutex_t state_mx;
+extern pthread_mutex_t config_mx;
+extern pthread_mutex_t rate_mx;
#endif
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2012-03-22 01:39:48 UTC (rev 683)
+++ trunk/defile/lib/libdefile.c 2012-03-22 23:01:51 UTC (rev 684)
@@ -147,28 +147,60 @@
/* skip RAW fields */
if (E->field_type == GD_RAW_ENTRY) {
- fputs("libdefile: Attempt to define RAW field via df_add_entry().\n",
+ fputs("libdefile: attempt to define RAW field via df_add_entry().\n",
stderr);
return DF_INPUT;
}
/* skip fields we've been asked to exclude */
- if (df_field_in_output(E->field))
- if (gd_add(df->D, E))
+ if (df_field_in_output(E->field)) {
+ if (df_mode() & DF_MODE_APPEND) {
+ /* check for existing field */
+ gd_entype_t type = gd_entry_type(df->D, E->field);
+ if (type == GD_NO_ENTRY)
+ return DF_INPUT;
+ else if (E->field_type != type) {
+ fprintf(stderr, "libdefile: incongruous specification of derived "
+ "field %s\n", E->field);
+ return DF_OUTPUT;
+ }
+ } else if (gd_add(df->D, E))
return DF_OUTPUT;
+ }
return DF_SUCCESS;
}
+#define DF_CHAR2ENTYPE(n) if (strcmp(t, #n) == 0) return GD_ ## n ## _ENTRY;
+const static df_type_from_token(const char *t)
+{
+ DF_CHAR2ENTYPE(BIT);
+ DF_CHAR2ENTYPE(CARRAY);
+ DF_CHAR2ENTYPE(CONST);
+ DF_CHAR2ENTYPE(DIVIDE);
+ DF_CHAR2ENTYPE(LINCOM);
+ DF_CHAR2ENTYPE(LINTERP);
+ DF_CHAR2ENTYPE(MPLEX);
+ DF_CHAR2ENTYPE(MULTIPLY);
+ DF_CHAR2ENTYPE(PHASE);
+ DF_CHAR2ENTYPE(POLYNOM);
+ DF_CHAR2ENTYPE(RECIP);
+ DF_CHAR2ENTYPE(SBIT);
+ DF_CHAR2ENTYPE(STRING);
+ DF_CHAR2ENTYPE(WINDOW);
+
+ return GD_NO_ENTRY;
+}
+
int df_add_spec(const char *spec, int fragment)
{
- char *ptr;
+ char *name, *type;
DF_CHECK_SEQUENCE(DF_ST_BUILD);
/* skip fields we've been asked to exclude, this requires isolating the
* field name; use GetData's parser to do this. */
- ptr = gd_tokenise(df->D, spec);
- if (ptr == NULL) {
+ name = gd_tokenise(df->D, spec);
+ if (name == NULL) {
if (gd_error(df->D) == GD_E_ALLOC)
return DF_OUTPUT; /* better to just fold this into gd_add_spec failures
than return DF_SYSTEM */
@@ -176,24 +208,42 @@
}
/* check OFL */
- if (!df_field_in_output(ptr)) {
- free(ptr);
+ if (!df_field_in_output(name)) {
+ free(name);
return DF_SUCCESS;
}
- free(ptr);
/* check the second token to see if it's a RAW field */
- ptr = gd_tokenise(df->D, NULL);
- if (ptr && strcmp(ptr, "RAW") == 0) {
+ type = gd_tokenise(df->D, NULL);
+ if (type && strcmp(type, "RAW") == 0) {
fputs("libdefile: Attempt to define RAW field via df_add_spec().\n",
stderr);
+ free(name);
+ free(type);
return DF_INPUT;
}
- free(ptr);
-
- /* add it */
- if (gd_add_spec(df->D, spec, fragment))
+
+ /* check for existing field in append mode */
+ if (df_mode() & DF_MODE_APPEND) {
+ gd_entype_t entype = gd_entry_type(df->D, name);
+ if (type == GD_NO_ENTRY) {
+ free(name);
+ free(type);
+ return DF_INPUT;
+ } else if (df_type_from_token(type) != entype) {
+ fprintf(stderr, "libdefile: incongruous specification of derived "
+ "field: %s\n", name);
+ free(name);
+ free(type);
+ return DF_OUTPUT;
+ }
+ } else if (gd_add_spec(df->D, spec, fragment)) {
+ free(name);
+ free(type);
return DF_OUTPUT;
+ }
+ free(name);
+ free(type);
return DF_SUCCESS;
}
@@ -252,6 +302,7 @@
static int df_set_raw(const struct df_fdef_field *f, int i, int count,
int fragment)
{
+ const int append = df_mode() & DF_MODE_APPEND;
int r, is_new = 0;
void *ptr;
@@ -270,10 +321,13 @@
perror("libdefile: strdup");
return DF_SYSTEM;
}
- /* try adding it to the dirfile */
- if (gd_add_raw(df->D, f->name, f->type, f->spf, fragment))
- return DF_OUTPUT;
+ if (!append) {
+ /* try adding it to the dirfile */
+ if (gd_add_raw(df->D, f->name, f->type, f->spf, fragment))
+ return DF_OUTPUT;
+ }
+
/* add a new record */
ptr = realloc(df->raw, sizeof(struct df_raw) * (df->nraw + 1));
if (ptr == NULL) {
@@ -297,16 +351,17 @@
df->raw[r].spf = f->spf * count;
df->raw[r].in = 0;
df->raw[r].b = NULL;
+ pthread_mutex_init(&df->raw[r].mx, NULL);
} else {
/* increase the samples per frame of this raw */
df->raw[r].spf += f->spf * count;
- if (gd_alter_raw(df->D, f->name, f->type, df->raw[r].spf, 0))
+ if (!append && gd_alter_raw(df->D, f->name, f->type, df->raw[r].spf, 0))
return DF_OUTPUT;
}
/* finalise buffer */
df->raw[r].framesize = GD_SIZE(f->type) * df->raw[r].spf;
- df->raw[r].buflen = df->...
[truncated message content] |
|
From: <ket...@us...> - 2012-07-20 23:34:39
|
Revision: 745
http://getdata.svn.sourceforge.net/getdata/?rev=745&view=rev
Author: ketiltrout
Date: 2012-07-20 23:34:30 +0000 (Fri, 20 Jul 2012)
Log Message:
-----------
Get the ASCII datasource working.
Modified Paths:
--------------
trunk/defile/bin/cli.c
trunk/defile/bin/rc.c
trunk/defile/configure.ac
trunk/defile/doc/Makefile.am
trunk/defile/doc/defile-input.7
trunk/defile/doc/defile.1.in
trunk/defile/doc/df_add_alias.3
trunk/defile/doc/df_add_entry.3
trunk/defile/doc/df_add_framedef.3
trunk/defile/doc/df_bsearch.3
trunk/defile/doc/df_check_abort.3
trunk/defile/doc/df_field_in_input.3
trunk/defile/doc/df_init.3
trunk/defile/doc/df_input_name.3
trunk/defile/doc/df_nframes_allowed.3
trunk/defile/doc/df_on_abort.3
trunk/defile/doc/df_push_frame.3
trunk/defile/doc/df_ready.3
trunk/defile/doc/df_set_const.3
trunk/defile/doc/df_shell_expand.3
trunk/defile/input/ascii.c
trunk/defile/lib/defile.h
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Added Paths:
-----------
trunk/defile/doc/df_mode.3
trunk/defile/doc/df_offset.3
Property Changed:
----------------
trunk/defile/doc/
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-07-12 22:38:02 UTC (rev 744)
+++ trunk/defile/bin/cli.c 2012-07-20 23:34:30 UTC (rev 745)
@@ -55,12 +55,19 @@
#define DEFILE_COPYRIGHT "Copyright (C) 2012 D. V. Wiebe"
#define DEFILE_CONTACT \
-"Please send reports of bugs and other communication to:\n" \
+"Please send reports of bugs and other communication regarding defile to:\n" \
" " PACKAGE_BUGREPORT "\n\n" \
"** NB: Contact and licensing information for input plugins accessible " \
"through\n this interface may be different than those of Defile " \
"itself. For such\n information, execute:\n\n" \
-" %s -t <input_type> -- --version"
+" %s -t <input_type> -- --version\n" \
+"\nThis program comes with NO WARRANTY, not even for MERCHANTABILITY or " \
+"FITNESS\nFOR A PARTICULAR PURPOSE. You may redistribute it under the " \
+"terms of the GNU\nGeneral Public License, either version 2 of the " \
+"License, or (at your option)\nany later version.\n\nYou should have " \
+"received a copy of the GNU General Public License along with\n" \
+"this program; if not, write to the Free Software Foundation, Inc.,\n" \
+"51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA\n"
#define DEFILE_USAGE_PREAMBLE "Usage:\n" \
" %s [DEFILE_OPTION]... [INPUT] [ -- [INPUT_OPTION]... ]\n\n" \
@@ -234,8 +241,8 @@
{
lt_dlhandle lib;
const struct df_input_framework *fwk = NULL;
- char *path, *base, *type;
- int i, have, result = 3;
+ char *path, *base, *type, *ptr;
+ int i, result = 3;
struct df_fwk_list *l = (struct df_fwk_list*)data;
@@ -247,11 +254,18 @@
}
base = basename(path);
- /* ensure it is of the form "[stuff]defile-<type>" */
+ /* ensure it is of the form "[stuff]defile-<type>[-stuff]" */
type = strstr(base, "defile-");
if (type) {
type += sizeof("defile-") - 1;
+ /* look for a trailing '-' */
+ for (ptr = type; *ptr; ++ptr)
+ if (*ptr == '-') {
+ *ptr = 0;
+ break;
+ }
+
/* check if it's the right type, if necessary */
if ((l->t == NULL) || strcmp(l->t, type) == 0) {
/* try to retrieve the framework */
@@ -260,6 +274,17 @@
}
free(path);
+ /* discard things we've already seen */
+ for (i = 0; i < l->n; ++i) {
+ if (strcmp(l->l[i]->name, type) == 0) {
+ if (df->mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: ignoring extra %s plugin: %s\n", type,
+ filename);
+ lt_dlclose(lib);
+ return 0;
+ }
+ }
+
/* if we have an input name, probe this plugin to see if it wants it */
if (result == 0 && l->i) {
if (fwk->probe && (*fwk->probe)(l->i))
@@ -276,37 +301,27 @@
/* success, add it to the list */
if (result == 0) {
- /* discard things we've already seen */
- have = 0;
- for (i = 0; i < l->n; ++i)
- if (strcmp(l->l[i]->name, type) == 0) {
- have = 1;
- }
+ l->r = 0;
- if (!have) {
- l->r = 0;
+ l->l = (const struct df_input_framework **)DF_Embiggen(l->l,
+ sizeof(const struct df_input_framework*) * (l->n + 1));
+ l->l[l->n] = fwk;
- l->l = (const struct df_input_framework **)DF_Embiggen(l->l,
- sizeof(const struct df_input_framework*) * (l->n + 1));
- l->l[l->n] = fwk;
+ l->p = (char **)DF_Embiggen(l->p, sizeof(char*) * (l->n + 1));
+ if ((l->p[l->n] = strdup(filename)) == NULL) {
+ perror("strdup");
+ exit(1);
+ }
- l->p = (char **)DF_Embiggen(l->p, sizeof(char*) * (l->n + 1));
- if ((l->p[l->n] = strdup(filename)) == NULL) {
- perror("strdup");
- exit(1);
- }
+ l->h = (lt_dlhandle*)DF_Embiggen(l->h, sizeof(lt_dlhandle) * (l->n + 1));
+ l->h[l->n] = lib;
- l->h = (lt_dlhandle*)DF_Embiggen(l->h, sizeof(lt_dlhandle) * (l->n + 1));
- l->h[l->n] = lib;
+ l->f = l->p[l->n];
+ l->n++;
- l->f = l->p[l->n];
- l->n++;
-
- /* stop searching if we're not compiling a list */
- if (l->t)
- return 1;
- } else
- lt_dlclose(lib);
+ /* stop searching if we're not compiling a list */
+ if (l->t)
+ return 1;
} else if (result < l->r) {
l->r = result;
free(l->f);
@@ -328,8 +343,10 @@
struct df_fwk_list *l)
{
int i;
- const char *dlp = getenv("DEFILE_LIBRARY_PATH");
+ const char *dlp;
+ dlp = getenv("DEFILE_LIBRARY_PATH");
+
memset(l, 0, sizeof(struct df_fwk_list));
l->i = input;
l->t = type;
@@ -433,7 +450,7 @@
return fwk;
}
-static int DF_InputOpt(struct df_config *rc, const struct df_rc_config *config,
+static int DF_InputOpt(struct df_config *rc, struct df_rc_config *config,
struct df_rc_data *rcd)
{
static int mode = 0;
@@ -462,7 +479,9 @@
sizeof(const char *) * (rc->n_config + 1));
rc->value[rc->n_config] = value;
+ /* steal the argument, if any */
rc->arg[rc->n_config++] = config->argument;
+ config->argument = NULL;
}
return -1;
@@ -659,26 +678,49 @@
static int DF_OutputFini(int delete)
{
- gd_close(df->D);
-
if (delete) {
- pid_t pid = fork();
- int status;
+ const char **list;
+ const char *name;
+ int i;
- if (pid > 0)
- waitpid(pid, NULL, 0);
- else if (pid == 0) {
- execl(PATH_RM, PATH_RM, "-rf", df->output, (char*)NULL);
- /* can't get here */
- fputs("defile: exec failure while deleting discarded output\n", stderr);
- exit(1);
+ /* since the bahaviour of encoding schemes can be murky, close all open data
+ * files so they're not recreated later during the discard. */
+ gd_raw_close(df->D, NULL);
+
+ /* loop over RAW fields, unlinking files */
+ for (list = gd_entry_list(df->D, NULL, GD_RAW_ENTRY,
+ GD_ENTRIES_HIDDEN | GD_ENTRIES_NOALIAS); *list; ++list)
+ {
+ name = gd_raw_filename(df->D, *list);
+ if (name && unlink(name))
+ perror("unlink");
}
- waitpid(pid, &status, 0);
- if (!WIFEXITED(status) || WEXITSTATUS(status)) {
- fputs("defile: error deleting discarded output\n", stderr);
- return 1;
+
+ /* loop over fragments, deleting metadata and directories */
+ for (i = gd_nfragments(df->D) - 1; i >= 0; --i) {
+ name = gd_fragmentname(df->D, i);
+ if (name) {
+ if (unlink(name))
+ perror("unlink");
+ else {
+ /* dirname(3) may modify its input */
+ char *buf = strdup(name);
+ if (buf) {
+ char *dir = dirname(buf);
+ if (dir && rmdir(dir))
+ perror("rmdir");
+ free(buf);
+ }
+ }
+ }
}
- }
+
+ /* discard the dirfile; since we've already unlinked everything, this
+ * should trigger deletion of files */
+ gd_discard(df->D);
+ } else
+ gd_close(df->D);
+
free(df->output);
return 0;
@@ -729,7 +771,7 @@
gd_verbose_prefix(df->D, "defile: ");
if (append) {
- unsigned long long nf = gd_nframes(df->D);
+ long long nf = gd_nframes(df->D);
pthread_mutex_lock(&config_mx);
if (df->offset < nf)
df->offset = nf;
@@ -914,7 +956,7 @@
pthread_mutex_unlock(&r->mx);
continue;
}
-
+
if (!(df_mode() & DF_MODE_ASYNC)) {
/* in synchronous mode, the reference field is special: we only write
* frames to it if all other fields have those frames committed */
@@ -1030,13 +1072,13 @@
#define RATE_TC (120.) /* rate IIR characteristic time (in seconds) */
static double DF_PrintRate(struct timeval *then, double rate)
{
- static unsigned long long lw = 0;
+ static long long lw = 0;
static int init = 2;
const char eol = (df_mode() & DF_MODE_DEBUG) ? '\n' : '\r';
double delta, drate;
- unsigned long long nf, nw, nr;
+ long long nf, nw, nr;
char si;
struct timezone tz;
struct timeval now;
@@ -1257,7 +1299,7 @@
.copyright = DEFILE_COPYRIGHT,
.default_config = "defilerc",
.flags = DF_RC_FLAG_NMSB,
- .licence = DF_LICENCE_GPL,
+ .licence = 0,
.name = NULL,
.options = options,
.parser = (df_parser_func_t)DF_OutputOpt,
@@ -1330,6 +1372,7 @@
action.sa_handler = DF_SignalHandler;
action.sa_mask = signals;
+ action.sa_flags = 0;
sigaction(SIGHUP, &action, NULL);
sigaction(SIGTERM, &action, NULL);
sigaction(SIGINT, &action, NULL);
@@ -1381,9 +1424,9 @@
/* one last check */
if (pthread_kill(df->input_id, 0) == 0)
- fputs("defile: terminating with input still active!\n", stderr);
+ fputs("defile: Warning: terminating with input still active!\n", stderr);
if (pthread_kill(df->output_id, 0) == 0)
- fputs("defile: terminating with output still active!\n", stderr);
+ fputs("defile: Warning: terminating with output still active!\n", stderr);
return 0;
}
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2012-07-12 22:38:02 UTC (rev 744)
+++ trunk/defile/bin/rc.c 2012-07-20 23:34:30 UTC (rev 745)
@@ -39,27 +39,27 @@
#define SHIFT_THIS 0x4
#define SAME_BOILERPLATE \
- "\nThis program comes with NO WARRANTY, not even for MERCHANTABILITY or " \
+ "\nThis plugin comes with NO WARRANTY, not even for MERCHANTABILITY or " \
"FITNESS\nFOR A PARTICULAR PURPOSE. You may redistribute it under the " \
"same terms as defile\nitself. For details, execute:\n\n" \
" %s --version\n"
#define GPL_BOILERPLATE \
- "\nThis program comes with NO WARRANTY, not even for MERCHANTABILITY or " \
+ "\nThis plugin comes with NO WARRANTY, not even for MERCHANTABILITY or " \
"FITNESS\nFOR A PARTICULAR PURPOSE. You may redistribute it under the " \
"terms of the GNU\nGeneral Public License, either version 2 of the " \
"License, or (at your option)\nany later version.\n\nYou should have " \
"received a copy of the GNU General Public License along with\n" \
-"this program; if not, write to the Free Software Foundation, Inc.,\n" \
+"this plugin; if not, write to the Free Software Foundation, Inc.,\n" \
"51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA"
#define LGPL_BOILERPLATE \
- "\nThis program comes with NO WARRANTY, not even for MERCHANTABILITY or " \
+ "\nThis plugin comes with NO WARRANTY, not even for MERCHANTABILITY or " \
"FITNESS\nFOR A PARTICULAR PURPOSE. You may redistribute it under the " \
"terms of the GNU\nLesser General Public License, either version 2.1 of the " \
"License, or (at your\noption) any later version.\n\nYou should have " \
"received a copy of the GNU General Public License along with\n" \
-"this program; if not, write to the Free Software Foundation, Inc.,\n" \
+"this plugin; if not, write to the Free Software Foundation, Inc.,\n" \
"51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA"
static const struct df_optdef df_no_opts[] = {
@@ -80,7 +80,7 @@
#define logprintf(...)
#endif
-#define DF_OPT_WIDTH 26
+#define DF_OPT_WIDTH 28
#define DF_TTY_WIDTH 80
static void DF_OptUsage(const struct df_optdef *opt, char shift_char)
@@ -267,8 +267,8 @@
free(opt);
- puts(" --help display this help and exit");
- puts(" --version display version information and exit\n");
+ puts(" --help display this help and exit");
+ puts(" --version display version information and exit\n");
if (rcd->usage_postamble)
puts(rcd->usage_postamble);
@@ -289,7 +289,9 @@
putchar('\n');
}
- if (rcd->contact)
+ if (!rcd->name)
+ printf(rcd->contact, argv0);
+ else if (rcd->contact)
puts(rcd->contact);
if (rcd->licence == DF_LICENCE_SAME)
@@ -769,7 +771,7 @@
else if (rcd->config[i].optind == DF_RC_RAW_ARG)
fprintf(stderr, "%i: ARGUMENT => %s\n", i, rcd->config[i].argument);
else
- fprintf(stderr, "%i (%s): %i (oE=%i) => %s\n", i,
+ fprintf(stderr, "%i: (%s): %i (oE=%i) => %s\n", i,
rcd->options[rcd->config[i].optind].lopt, rcd->config[i].optind,
rcd->config[i].error, rcd->config[i].argument);
#endif
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2012-07-12 22:38:02 UTC (rev 744)
+++ trunk/defile/configure.ac 2012-07-20 23:34:30 UTC (rev 745)
@@ -20,7 +20,7 @@
dnl
m4_define(defile_major, 0)
-m4_define(defile_minor, 1)
+m4_define(defile_minor, 0)
m4_define(defile_revision, 0)
m4_define(defile_extra, [])
m4_define(defile_version,
@@ -126,6 +126,7 @@
echo "*** Checking for external libraries"
echo
PKG_CHECK_MODULES([GETDATA], [getdata >= 0.7.0])
+AC_SEARCH_LIBS([exp],[m])
AC_SEARCH_LIBS([pthread_create],[pthread])
echo
@@ -153,7 +154,7 @@
echo
echo "*** Looking for additional library functions"
echo
-AC_CHECK_FUNCS([wordexp])
+AC_CHECK_FUNCS([pipe wordexp])
echo
echo "*** Looking for external programs"
Property changes on: trunk/defile/doc
___________________________________________________________________
Added: svn:ignore
+ defile.1
Makefile
Makefile.in
Modified: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am 2012-07-12 22:38:02 UTC (rev 744)
+++ trunk/defile/doc/Makefile.am 2012-07-20 23:34:30 UTC (rev 745)
@@ -24,5 +24,6 @@
man_MANS = defile.1 defile-input.7 df_add_alias.3 df_add_entry.3 \
df_add_framedef.3 df_bsearch.3 df_check_abort.3 df_field_in_input.3 \
- df_init.3 df_input_name.3 df_nframes_allowed.3 df_on_abort.3 \
- df_push_frame.3 df_ready.3 df_set_const.3 df_shell_expand.3
+ df_init.3 df_input_name.3 df_mode.3 df_nframes_allowed.3 \
+ df_offset.3 df_on_abort.3 df_push_frame.3 df_ready.3 \
+ df_set_const.3 df_shell_expand.3
Modified: trunk/defile/doc/defile-input.7
===================================================================
--- trunk/defile/doc/defile-input.7 2012-07-12 22:38:02 UTC (rev 744)
+++ trunk/defile/doc/defile-input.7 2012-07-20 23:34:30 UTC (rev 745)
@@ -13,7 +13,7 @@
.\" Texts. A copy of the license is included in the `COPYING.DOC' file
.\" as part of this distribution.
.\"
-.TH defile-input 7 "21 February 2012" "Version 0.1.0" DEFILE
+.TH defile-input 7 "23 March 2012" "Version 0.1.0" DEFILE
.SH NAME
defile-input \(em overview of the defile input plugin API
.SH DESCRIPTION
@@ -43,18 +43,19 @@
.B DEFILE OPERATION
below.
-An input plugin is a dynamically loaded shared library which reads data from a
-data source and passes it on to defile for output as a dirfile. Each input
-plugin is identified by it's type name. The type name identifies the type of
-data the input plugin is capable of reading. The type name should consist of
-one or more lower case letters, numbers, or the underscore (_) character. The
-type name must be unique among all defile input plugins: defile will only ever
-consider the first input plugin with a given type name that it finds.
+An input plugin is a dynamically loaded shared library which asynchronously
+reads data from a data source and passes it on to defile for output as a
+dirfile. Each input plugin is identified by it's type name. The type name
+identifies the type of data the input plugin is capable of reading. The type
+name should consist of one or more lower case letters, numbers, or the
+underscore (_) character. The type name must be unique among all defile input
+plugins: defile will only ever consider the first input plugin with a given type
+name that it finds.
The name of the shared library must be of the form:
.IP
.fam C
-[\fIanything\fR]\fBdefile-<\fItype\fR>[\fI.ext\fR]
+[\fIanything\fR]\fBdefile-<\fItype\fR>[-\fIanything\fR][\fI.ext\fR]
.fam
.PP
where
@@ -64,7 +65,9 @@
.B defile-
is literaly "defile" followed by a dash,
.I type
-is the input type name, and
+is the input type name,
+.I -anything
+contains more optional characters, separated from the type name by a dash, and
.I .ext
is a file extension appropriate for native dynamically loadable modules on the
host platform, if any.
@@ -108,17 +111,16 @@
.TP
.I name
is the type name of the plugin. Defile will consider the module malformed and
-ignore it if this is not identical to the type name used in both the exported
-df_input_framework symbol, and the filename of the module itself.
+ignore it if this is not identical to the type name used in both the name of
+the exported df_input_framework symbol, and the filename of the module itself.
.TP
.I version
a version string for the plugin, typically a verison triplet of the form
-"XX.YY.ZZ",
+"XX.YY.ZZ", although defile puts no restriction on it.
.TP
.I copyright
is a copyright string for the plugin, typically of the form
-"Copyright \(co <date> <author(s)>". (The non-ASCII \(co is typically
-approximated by "(C)".)
+"Copyright \(co <date> <author(s)>".
.TP
.IR contact ", " preamble ", and " postamble
are formatted strings used in the usage or version information output to the
@@ -164,10 +166,11 @@
.SH INPUT OPTIONS
-The input plugin may declare input options and directives which defile will
-parse for it using the
+The input plugin may declare input options and directives using the
.I opts
member of the framework. If the input plugin has none, this may simply be NULL.
+Defile uses this list to parse the input options and directives given by the
+user on the command line and in any associated run control files.
Defile treats input options and directives together. In the following, whenever
we refer input options, the remarks apply equally to input directives unless
@@ -196,11 +199,11 @@
where
.TP
.I value
-is a positive number used by defile to indentify the option. Bare arguments
-among the input options (those not associated with an option) are assigned a
-value of zero, so that should be avoided in the definition of input options.
-Unless two options should be treated as identical, each option should have a
-unique value. There is no requirement that consequitive values be used.
+is a positive number used by defile to indentify the option to the input plugin.
+Bare arguments among the input options (those not associated with an option) are
+assigned a value of zero, so that should be avoided in the definition of input
+options. Unless two options should be treated as identical, each option should
+have a unique value. There is no requirement that consequitive values be used.
After parsing the input options, defile will present the input plugin an array
of (value, argument) pairs. See the
.B Start State
@@ -210,7 +213,7 @@
by defile for bookkeeping purposes and should be avoided.
.TP
.I type
-should be one of the following symbols inidcating the type of option:
+should be one of the following symbols indicating the type of option:
.RS
.TP
.B DF_OPT_NO_ARG
@@ -238,7 +241,7 @@
this option record. If no directive is defined, this should be NULL.
.TP
.I flags
-either zero, or else one or more of the following flags providing further
+zero or more of the following flags, bitwise or'd together, providing further
information on how the option should be interpreted:
.RS
.TP
@@ -260,7 +263,7 @@
.B -x
on the command line. If
.I sopt
-is zero (inidcating no short option is associated with this record) this flag
+is zero (indicating no short option is associated with this record) this flag
is ignored.
.RE
.TP
@@ -273,7 +276,7 @@
value of "ARG" is used.
.TP
.I usage
-an unformated string explaining the argument. The strig will be used in the
+an unformated string explaining the argument. The string will be used in the
usage output (see the
.B HELP AND VERSION OUTPUT
section below). It should contain no newlines; they will be inserted where
@@ -330,18 +333,19 @@
.I preamble
string given in the framework definition, if any,
.IP \(bu
-an alphabetised and formatted list of the input options, including their usage
-strings, generated from the input option list given in the framework
-definiition,
+a sorted and formatted list of the input options, including their usage strings,
+generated from the input option list
+.I opts
+given in the framework definiition,
.IP \(bu
the
.I postamble
string given in the framework definition, if any.
.PP
Unless the plugin is simple enough to be summarised in the usage text, plugin
-developers are encouraged to provide an accompanying Unix manual page called
-defile-<\fItype\fR>(1). Other documentation, if any, should be mentioned in the
-usage preamble or postamble.
+developers are encouraged to provide accompanying documentation, if possible a
+Unix manual page called defile-<\fItype\fR>(1). Any such other documentation
+should be mentioned in the usage preamble or postamble.
.PP
When
.B \-\-version
@@ -421,17 +425,15 @@
the data provided in the input framework definition.
Once this has completed successfully, both the reader (input plugin) and
-output (dirfile writer) are started. The reader is started by calling the
+writer (dirfile output) are started. The reader is started by calling the
.I entry
procedure defined in the input plugin's framework definition. The input plugin
will be passed a struct df_config pointer which provides the input plugin with
-it's parsed configuration, and a few other important pieces of opterational
-information. The struct df_config is defined as:
+its parsed configuration. The struct df_config is defined as:
.IP
.nf
.fam C
struct df_config {
- unsigned int mode_flags;
int n_config;
int *value;
const char **arg;
@@ -441,32 +443,6 @@
.PP
where
.TP
-.I mode_flags
-a bitwise or'd collection of zero or more of the following flags, indicating
-defile's operational mode:
-.RS
-.TP
-.B DF_MODE_ASYNC
-defile is operating in asynchronous mode and the input plugin is not required to
-adhere to the restrictions of synchronous mode. (See the
-.B RUN State
-section below for a discussion of the implications of synchronous and
-asynchronous modes.) An input plugin is not required to implement asynchronous
-mode. (Implementing synchronous mode \fIis\fR required.)
-.TP
-.B DF_MODE_DEBUG
-defile is operating in debug mode and is printing extra debugging messages on
-standard error. This does not affect the operation of defile, but this flag may
-be used by the input plugin to decide when to emit its own debugging messages.
-Otherwise, it may be safely ignored by input plugins.
-.TP
-.B DF_MODE_FOLLOW
-defile is operating in follow mode and requests the input plugin to continue
-to monitor the input data source when it reaches the end, providing additional
-data to defile as it appears. An input plugin is not required to implement
-follow mode and, so, may exit upon running out of data regardless of this flag.
-.RE
-.TP
.I n_config
the number of elements in the configuration produced by parsing the input
options and input directives.
@@ -475,8 +451,8 @@
an array of integers of length
.I n_config
containing the values (see
-.B INPUT OPTIONS above)
-of the input options in the order they were encountered. Bare arguments
+.B INPUT OPTIONS
+above) of the input options in the order they were encountered. Bare arguments
encountered (arguments which are not associated with an option) will have a
value of zero. Other options will have the corresponding value that was
specified in the options array in the framework definition.
@@ -489,16 +465,20 @@
.PP
During
.B STARTUP
-the input plugin should perform initialisation, parse the input string
+the input plugin should perform initialisation, parse the input designator
(obtainable from
.BR df_input_name (3))
-and verify that the data type of the input is understood by the plugin. Once
-it is ready to proceed, it should call
+and verify that it does indeed specify a data source understood by the plugin.
+Once it is ready to proceed, it should call
.BR df_init (3),
which triggers the transition into the
.B BUILD
state.
+If the input plugin exits while in the
+.B START
+state, no output dirfile will be created, regardless of success or failure.
+
.SS BUILD State
The
.B BUILD
@@ -512,8 +492,15 @@
(See also the
.B DEFILE DATA MODEL
section below for an overview of how defile expects data to be marshalled.)
+This is also the earliest time at which the input plugin is able to determine
+the requested starting frame for the data (by calling
+.BR df_offset (3)).
-When it has finished creating the output dirfile's metadata, it should call
+If the input plugin exits while in the
+.B BUILD
+state, no output dirfile will be created, regardless of success or failure.
+Otherwise, when it has finished creating the output dirfile's metadata, it
+should call
.BR df_ready (3),
which triggers the transition into the
.B RUN
@@ -522,9 +509,9 @@
.SS RUN State
The
.B RUN
-state is when data frames are read by the input plugin, transferred to the
-dirfile writer, and written to disk. Typically defile will spend the bulk of
-it's time in this state.
+state is when data frames from the data source are transferred by the input
+plugin to the dirfile writer, and written to disk. Typically defile will spend
+the bulk of it's time in this state.
The input plugin provides data to the dirfile writer by calling
.BR df_push_frame (3),
@@ -552,7 +539,8 @@
The input plugin may trigger a return to the
.B BUILD
state by calling
-.BR gd_reinit (3).
+.BR gd_reinit (3),
+typically when the input plugin notices a change in the datasource metadata.
Otherwise, defile remains in the
.B RUN
state as long as the input plugin is running. The plugin returning success
@@ -573,10 +561,10 @@
Defile enters the
.B ABORT
-state in cases when an unrecoverable, catastrophic error occurs during
-operation, or else when it receives an asynchronous request to cancel (for
-example a keyboard interrupt). Most of the C API functions check whether
-defile has entered this state and return
+state in cases when an unrecoverable error occurs during operation, or else when
+it receives an asynchronous request to cancel (for example a keyboard
+interrupt). Most of the input plugin API functions check whether defile has
+entered this state and return
.B DF_ABORT
when detected. The input should monitor the return value of these functions to
detect an unexpected transition into the
@@ -616,8 +604,8 @@
For each field, the number of samples per frame is constant, but different
fields may have different number of samples per frame. As a result, not all
data need to be sampled at the same rate, but the ratio of sample rates of all
-fields is fixed and the frame period must be some multiple of the lowest common
-multiple of all sample periods.
+fields is fixed and the frame period must be some common multiple of all sample
+periods.
For data marshalling purposes, defile breaks the dirfile frame into one or more
subframes, each of fixed format (see
@@ -653,7 +641,7 @@
.SH OTHER CONSIDERATIONS WHILE RUNNING
A few other operational points to remember while writing the main loop for an
-inpuy plugin are discussed below.
+input plugin are discussed below.
.SS Signal Handling
@@ -673,7 +661,8 @@
The most common reason for this is the input plugin spending too long in the
.B ABORT
state, but it may happen in other exceptional circumstances. Input plugins which
-disable cancelability for long periods of time may be subject to more drastic measures without forewarning.
+disable cancelability for long periods of time may be subject to more drastic
+measures without forewarning.
.SH AUTOMATIC TYPE DETECTION
If desired, and possible, the plugin may participate in the automatic type
@@ -706,8 +695,9 @@
.SH API REFERENCE
-The following functions are provided by the defile C API, grouped here by
-sequencing restrictions. See the indicated manual pages for details.
+The following functions (some listed multiple times) are provided by the defile
+C API, grouped here by sequencing restrictions. See the indicated manual pages
+for details.
.TP
.RB "Functions which may be called at any time, regardless of current state:"
@@ -715,6 +705,7 @@
.BR df_check_abort (3),
.BR df_field_in_output (3),
.BR df_input_name (3),
+.BR df_mode (3),
.BR df_on_abort (3),
.BR df_shell_expand (3),
.BR df_strcmp (3)
@@ -725,8 +716,10 @@
Functions which may be called during the \fBBUILD\fR state:
.BR df_add_alias (3),
.BR df_add_entry (3),
-.BR df_add_spec (3),
+.BR df_add_fragment (3),
.BR df_add_framedef (3),
+.BR df_add_spec (3),
+.BR df_offset (3),
.BR df_ready (3),
.BR df_set_const (3),
.BR df_set_carray (3),
@@ -734,15 +727,12 @@
.TP
Functions which may be called during the \fBRUN\fR state:
.BR df_nframes_allowed (3),
+.BR df_offset (3),
.BR df_push_frame (3),
.BR df_reinit (3),
.BR df_set_const (3),
.BR df_set_carray (3),
.BR df_set_string (3)
-.IP
-NB: See
-.BR df_reinit (3)
-for caveats involving sequencing in the case when this function fails.
.SH SEE ALSO
.BR defile (1),
Modified: trunk/def...
[truncated message content] |
|
From: <ket...@us...> - 2012-08-13 23:22:53
|
Revision: 756
http://getdata.svn.sourceforge.net/getdata/?rev=756&view=rev
Author: ketiltrout
Date: 2012-08-13 23:22:42 +0000 (Mon, 13 Aug 2012)
Log Message:
-----------
Get dirfile plugin working.
Modified Paths:
--------------
trunk/defile/bin/cli.c
trunk/defile/configure.ac
trunk/defile/doc/Makefile.am
trunk/defile/doc/defile-input.7
trunk/defile/doc/df_add_alias.3
trunk/defile/doc/df_add_entry.3
trunk/defile/doc/df_add_framedef.3
trunk/defile/doc/df_init.3
trunk/defile/doc/df_nframes_allowed.3
trunk/defile/doc/df_offset.3
trunk/defile/doc/df_push_frame.3
trunk/defile/doc/df_set_const.3
trunk/defile/input/dirfile.c
trunk/defile/lib/defile.h
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
trunk/defile/test/make.c
trunk/defile/test/make.h
Added Paths:
-----------
trunk/defile/doc/df_add_fragment.3
trunk/defile/doc/df_add_lut.3
trunk/defile/doc/df_copy_file.3
trunk/defile/doc/df_hide.3
trunk/defile/doc/df_include.3
trunk/defile/doc/df_subset_contains.3
trunk/defile/doc/df_subset_init.3
trunk/defile/doc/df_subset_reset.3
trunk/defile/doc/df_subset_update.3
Removed Paths:
-------------
trunk/defile/doc/df_bsearch.3
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/bin/cli.c 2012-08-13 23:22:42 UTC (rev 756)
@@ -77,9 +77,9 @@
"The list of defile\noptions and the list of input options are separated " \
"by a double dash (--) to\ndistinguish them. If no input options are " \
"needed, this section, including the\ndouble dash, may be omitted.\n\n" \
-"The first bare argument (i.e. the first argument not associated with " \
+"The first bare argument (i.e. the first argument not associated with " \
"an option)\nbefore the double dash, if present, is taken to be INPUT. " \
-"If INPUT starts with\na dash (and would, therefore, be interpreted as " \
+"If INPUT starts with\n- or + (and would, therefore, be interpreted as " \
"an option), the --input= defile\noption can be used to specify it " \
"instead (see below).\n\n" \
"Defile options are listed below. For help with input options, which " \
@@ -487,44 +487,6 @@
return -1;
}
-static int DF_UpdateOFL(const char *name, int incl)
-{
- int is_new = 0, r;
-
- /* the type of the OFL may be changed when nothing is in it */
- if (df->nofl == 0)
- df->ofl_incl = incl;
-
- /* find the location */
- r = df_bsearch(name, df->ofl, 0, df->nofl - 1, sizeof(char*), &is_new,
- df_strcmp);
-
- if (df->ofl_incl == incl) {
- if (is_new) {
- /* insert */
- if (df->nofl + 1 > df->ofl_size)
- df->ofl = DF_Embiggen(df->ofl, sizeof(char*) * (++df->ofl_size));
- memmove(df->ofl + r + 1, df->ofl + r, sizeof(char*) * (df->nofl++ - r));
- df->ofl[r] = strdup(name);
- if (df->ofl[r] == NULL) {
- perror("malloc");
- exit(1);
- }
- }
- } else {
- if (!is_new) {
- /* remove */
- free(df->ofl[r]);
- memmove(df->ofl + r, df->ofl + r + 1, sizeof(char*) * (df->nofl-- - r));
- /* an empty OFL always means write everything */
- if (df->nofl == 0)
- df->ofl_incl = 0;
- }
- }
-
- return 0;
-}
-
enum {
DF_WOPT_ADD = 1, DF_WOPT_APPEND, DF_WOPT_ASYNC, DF_WOPT_AUTOTYPE,
DF_WOPT_BHEAD, DF_WOPT_BLEN, DF_WOPT_CLOBBER, DF_WOPT_CUTDIR, DF_WOPT_DEBUG,
@@ -538,7 +500,6 @@
static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
struct df_rc_data *rcd)
{
- int i;
struct df_fwk_list l;
if (df->mode_switch == 1) {
@@ -570,16 +531,17 @@
df->mode_switch = 1;
break;
case DF_WOPT_ADD: /* -F */
- DF_UpdateOFL(config->argument, 1);
+ if (df_subset_update(df->ofl, config->argument,
+ strlen(config->argument) + 1, DF_SUBSET_ADD))
+ exit(1);
break;
case DF_WOPT_DEL: /* +F */
- DF_UpdateOFL(config->argument, 0);
+ if (df_subset_update(df->ofl, config->argument,
+ strlen(config->argument) + 1, DF_SUBSET_DEL))
+ exit(0);
break;
case DF_WOPT_OFLCLR: /* --reset-field-list */
- for (i = 0; i < df->nofl; ++i)
- free(df->ofl[i]);
- df->nofl = 0;
- df->ofl_incl = 0;
+ df_subset_reset(df->ofl, DF_SUBSET_INIT);
break;
case DF_WOPT_INPUT: /* -i */
if (df->input == NULL) {
@@ -1159,9 +1121,7 @@
{
int i;
- for (i = 0; i < df->nofl; ++i)
- free(df->ofl[i]);
- free(df->ofl);
+ df_subset_delete(df->ofl);
for (i = 1; i < df->iargc; ++i)
free(df->iargv[i]);
@@ -1213,10 +1173,10 @@
{ DF_WOPT_NOCLOBBER, DF_OPT_NO_ARG, 0, "no-clobber", "NoClobber", 0, NULL,
"don't overwrite an existing dirfile (default)"
},
- { DF_WOPT_CUTDIR, DF_OPT_ARG_RQ, '0', "cut-dirs", "CutDirs", 0, "NUM",
+ { DF_WOPT_CUTDIR, DF_OPT_ARG_RQ, 0, "cut-dirs", "CutDirs", 0, "NUM",
"remove all but the last NUM path elements from the output name"
},
- { DF_WOPT_NOCUTDIR, DF_OPT_NO_ARG, '0', "no-cut-dirs", "NpCutDirs", 0, NULL,
+ { DF_WOPT_NOCUTDIR, DF_OPT_NO_ARG, 0, "no-cut-dirs", "NpCutDirs", 0, NULL,
"don't remove path elements from the output name (default)"
},
{ DF_WOPT_DIR, DF_OPT_ARG_RQ, 'D', "directory", "Directory", DF_OPT_EXPAND,
@@ -1320,6 +1280,10 @@
lt_dlinit();
atexit(DF_ShutdownLTDL);
+ /* initialise and parse command line */
+ df->ofl = df_subset_init(DF_SUBSET_FULL, df_subset_strcmp);
+ if (df->ofl == NULL)
+ exit(1);
DF_Parse(argc, argv, &rcd, df);
/* sanitise */
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/configure.ac 2012-08-13 23:22:42 UTC (rev 756)
@@ -151,10 +151,18 @@
AC_CHECK_TYPES([_Complex float, _Complex double])
+dnl Declare a few feature test macros
+AC_DEFINE([_GNU_SOURCE], [1], [ Expose GNU extensions ])
+AC_DEFINE([_BSD_SOURCE], [1], [ Expose BSD-derived definitions ])
+AC_DEFINE([_SVID_SOURCE], [1], [ Expose System V-derived definitions ])
+AC_DEFINE([_POSIX_SOURCE], [1], [ Expose POSIX.1-1990 conforming definitions ])
+AC_DEFINE([_POSIX_C_SOURCE], [200809L],
+ [ Expose POSIX:2008 conforming definitions ])
+
echo
echo "*** Looking for additional library functions"
echo
-AC_CHECK_FUNCS([pipe wordexp])
+AC_CHECK_FUNCS([canonicalize_file_name realpath wordexp])
echo
echo "*** Looking for external programs"
Modified: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/doc/Makefile.am 2012-08-13 23:22:42 UTC (rev 756)
@@ -20,10 +20,36 @@
#
AUTOMAKE_OPTIONS = foreign
-BUILT_SOURCES=defile.1
+nodist_man_MANS = defile.1
man_MANS = defile.1 defile-input.7 df_add_alias.3 df_add_entry.3 \
- df_add_framedef.3 df_bsearch.3 df_check_abort.3 df_field_in_input.3 \
+ df_add_fragment.3 df_add_framedef.3 df_add_lut.3 df_check_abort.3 \
+ df_copy_file.3 df_field_in_input.3 df_hide.3 df_include.3 \
df_init.3 df_input_name.3 df_mode.3 df_nframes_allowed.3 \
df_offset.3 df_on_abort.3 df_push_frame.3 df_ready.3 \
- df_set_const.3 df_shell_expand.3
+ df_set_const.3 df_shell_expand.3 df_subset_contains.3 \
+ df_subset_init.3 df_subset_reset.3 df_subset_update.3
+
+# these are real_file:alias
+MAN3ALIASES=df_add_entry.3:df_add_spec \
+ df_init.3:df_reinit.3 \
+ df_set_const:df_set_carray df_set_const:df_set_string \
+ df_subset_init:df_subset_delete df_subset_init:df_subset_strcmp
+
+
+install-data-hook:
+ cd $(DESTDIR)$(mandir)/man3 && \
+ for m in $(MAN3ALIASES); do \
+ rm -f `echo $$m | $(SED) -e 's/.*://'`; \
+ $(LN_S) `echo $$m | $(SED) -e 's/:/ /'`; \
+ done
+
+uninstall-hook:
+ cd $(DESTDIR)$(mandir)/man3 && \
+ for m in $(MAN3ALIASES); do \
+ rm -f `echo $$m | $(SED) -e 's/.*://'`; \
+ done
+
+clean-local:
+ rm -f *~
+ rm -f $(nodist_man_MANS)
Modified: trunk/defile/doc/defile-input.7
===================================================================
--- trunk/defile/doc/defile-input.7 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/doc/defile-input.7 2012-08-13 23:22:42 UTC (rev 756)
@@ -394,7 +394,7 @@
occur in separate threads within defile. To achieve synchronisation between
input and output, defile operates in one of several states while converting
data. During any particular state of operation, the input plugin is restricted
-to a certain subset of the defile input API. A consice list of functions
+to a certain subset of the defile input API. A concise list of functions
available in each state is provided below in the
.B API REFERNCE
section. We now discuss basic operation of defile.
@@ -406,8 +406,8 @@
.BR ABORT .
The input plugin's role in each state is discussed below.
-The input plugin may return at any time, if it's job is complete (or if it ran
-into an unrecoverable error). Returning a value of zero indicates to defile
+The input plugin may return at any time if it's job is complete, or if it ran
+into an unrecoverable error. Returning a value of zero indicates to defile
that the input plugin completed normally (success). Returning any other value
indicates that the input plugin encountered an unrecoverable error (failure).
@@ -566,7 +566,9 @@
interrupt). Most of the input plugin API functions check whether defile has
entered this state and return
.B DF_ABORT
-when detected. The input should monitor the return value of these functions to
+when detected (see
+.BR "Standard API Return Semaphores" ).
+The input should monitor the return value of these functions to
detect an unexpected transition into the
.B ABORT
state.
@@ -664,6 +666,50 @@
disable cancelability for long periods of time may be subject to more drastic
measures without forewarning.
+.SS Standard API Return Semaphores
+
+Many API functions return a standard
+.I int
+semaphore indicating success or failure of the operation. The standard return
+semaphore values are:
+.TP
+.B DF_SUCCESS \fR(=0)
+The operation completed successfully. (Some functions return a non-negative
+number on success instead.)
+.TP
+.B DF_ABORT
+A catastrophic error occurred. See
+.BR "ABORT state" .
+If an abort handler is registered by the plugin (using
+.BR df_on_abort (3))
+this value will never be returned, and the function simply does not return.
+.TP
+.B DF_SYSTEM
+A system error (usually a memory allocation error) occurred. The input plugin
+may try the call again, although the same error may result.
+.TP
+.B DF_SEQUENCE
+A
+.I sequencing error
+occurred: a call was made to a function which is not available in the current
+defile state (see
+.B API REFERENCE
+for a list of functions broken down by state).
+The call should not be repeated until the correct state is achieved.
+.TP
+.B DF_INPUT
+A bad call was made to the API, typically due to passing bad parameter data.
+The call should not be repeated without correction.
+.TP
+.B DF_OUTPUT
+An error occurred manipulating the output dirfile. The input plugin may try the
+call again, although the same error may result.
+.P
+Except for
+.BR DF_SUCCESS ,
+which evaluates to zero, these all have negative values. See the specific
+function manual pages for more details on function return values.
+
.SH AUTOMATIC TYPE DETECTION
If desired, and possible, the plugin may participate in the automatic type
detection system used by defile when an explicit input type is not provided.
Modified: trunk/defile/doc/df_add_alias.3
===================================================================
--- trunk/defile/doc/df_add_alias.3 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/doc/df_add_alias.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -67,7 +67,7 @@
.B DF_SEQUENCE
The function was called while defile was not in the
.B BUILD
-state. (See defile-input (7)).
+state. (See defile-input(7)).
.PP
All errors are accompanied by a message on standard error.
.SH SEE ALSO
Modified: trunk/defile/doc/df_add_entry.3
===================================================================
--- trunk/defile/doc/df_add_entry.3 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/doc/df_add_entry.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -101,7 +101,7 @@
.B DF_SEQUENCE
The function was called while defile was not in the
.B BUILD
-state. (See defile-input (7)).
+state. (See defile-input(7)).
.PP
All errors are accompanied by a message on standard error.
.SH SEE ALSO
Added: trunk/defile/doc/df_add_fragment.3
===================================================================
--- trunk/defile/doc/df_add_fragment.3 (rev 0)
+++ trunk/defile/doc/df_add_fragment.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -0,0 +1,117 @@
+.\" df_add_fragment.3. The df_add_fragment man page.
+.\"
+.\" Copyright (C) 2012 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" This file is part of the Defile package.
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH df_add_fragment 3 "13 August 2012" "Version 0.0.0" DEFILE
+.SH NAME
+df_add_fragment \(em create a new fragment in the defile output dirfile
+.SH SYNOPSIS
+.nh
+.ad l
+.fam C
+.B #include <defile.h>
+.HP
+.BI "int df_add_fragment(const char *" name ", int " parent ,
+.BI "unsigned int " encoding ", unsigned long " byte_sex ,
+.BI "const char *" prefix ", const char *" suffix );
+.fam
+.ad
+.hy
+.SH DESCRIPTION
+This function is part of the defile input plugin C API. See defile-input(7) for
+an overview.
+.PP
+The
+.BR df_add_fragment ()
+creates a new format file fragment (see dirfile-format(5)) called
+.I name
+in the defile output dirfile. The
+.I name
+may contain path elements relative to the base output dirfile directory.
+Subdirectories specified in
+.I name
+will be created if they do not already exist. The new fragment will be included
+in the fragment indexed by
+.IR parent ,
+which should be zero (for the primary format file) or else a fragment index
+returned by a previous call to
+.BR df_add_fragment ().
+
+The encoding of data files used in the new fragment is specified with
+.IR encoding ,
+which should be one of the symbols listed in
+.BR gd_alter_encoding (3),
+although not all encodings provide write capability (see dirfile-encoding(5) for
+details). If this is set to
+.BR GD_AUTO_ENCODED ,
+the encoding scheme will be chosen by defile. The byte sex of data files in
+the new fragment is specified with
+.IR byte_sex ,
+which should be one of the values listed in
+.BR gd_alter_endianness (3).
+To use default values for these parameters, set them to zero.
+
+The prefix and suffix of the new fragment may specified by
+.I prefix
+and
+.IR suffix ,
+which may be NULL if the corresponding affix is not needed.
+
+In append mode (see
+.BR df_mode (3)),
+this function only tests for the existence of
+.IR name ,
+and returns an error if it is not found. In that case, all other parameters are
+ignored.
+
+.SH RETURN VALUE
+Upon success,
+.BR df_add_fragment ()
+returns a positive integer fragment index corresponding to the newly created
+fragment. The value should be cached by the caller and used in subsequent calls
+requiring a fragment index. On error, one of the following error values (all
+less than zero) will be returned:
+.TP
+.B DF_ABORT
+Another defile subprocess has requested the input plugin immediately abort.
+Upon receiving this signal the plugin should shutdown and return. If it takes
+too long, defile will cancel it. If an abort handler has been registered (see
+.BR df_on_abort (3)),
+this value is never returned and, instead, this function does not return.
+.TP
+.B DF_OUTPUT
+An error occured attempting to add the fragment to the output dirfile. See
+.BR gd_include_affix (3)
+and
+.BR mkdir (2)
+for possible reasons. Or, defile was in append mode, and the specified
+.I name
+did not exist.
+.TP
+.B DF_SEQUENCE
+The function was called while defile was not in the
+.B BUILD
+state. (See defile-input(7)).
+.TP
+.B DF_SYSTEM
+A memory allocation error occurred.
+.PP
+All errors are accompanied by a message on standard error.
+.SH SEE ALSO
+.PP
+defile-input(7), defile(1),
+.BR df_include (3),
+.BR gd_include_affix (3),
+.BR gd_alter_encoding (3),
+.BR gd_alter_endianness (3)
Modified: trunk/defile/doc/df_add_framedef.3
===================================================================
--- trunk/defile/doc/df_add_framedef.3 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/doc/df_add_framedef.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -1,4 +1,4 @@
-.\" df_add_framdef.3. The df_add_framdef man page.
+.\" df_add_framedef.3. The df_add_framedef man page.
.\"
.\" Copyright (C) 2012 D. V. Wiebe
.\"
@@ -13,9 +13,9 @@
.\" Texts. A copy of the license is included in the `COPYING.DOC' file
.\" as part of this distribution.
.\"
-.TH df_add_framdef 3 "27 February 2012" "Version 0.0.0" DEFILE
+.TH df_add_framedef 3 "27 February 2012" "Version 0.0.0" DEFILE
.SH NAME
-df_add_framdef \(em define a defile input subframe
+df_add_framedef \(em define a defile input subframe
.SH SYNOPSIS
.nh
.ad l
@@ -32,7 +32,7 @@
an overview.
.PP
The
-.BR df_add_framdef ()
+.BR df_add_framedef ()
function defines a defile subframe definition,
.IR fdef ,
and adds
@@ -149,7 +149,7 @@
(see defile(1)) are accepted in subframe definitions and simply ignored.
.SH RETURN VALUE
Upon success,
-.BR df_add_framdef ()
+.BR df_add_framedef ()
returns the assigned index number of this subframe definition, which is always
greater than or equal to zero. This number should be cached by the input plugin
and used in subsequent calls to
@@ -177,7 +177,7 @@
.B DF_SEQUENCE
The function was called while defile was not in the
.B BUILD
-state. (See defile-input (7)).
+state. (See defile-input(7)).
.TP
.B DF_SYSTEM
A memory allocation error occurred.
Added: trunk/defile/doc/df_add_lut.3
===================================================================
--- trunk/defile/doc/df_add_lut.3 (rev 0)
+++ trunk/defile/doc/df_add_lut.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -0,0 +1,116 @@
+.\" df_add_lut.3. The df_add_lut man page.
+.\"
+.\" Copyright (C) 2012 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" This file is part of the Defile package.
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH df_add_lut 3 "13 August 2012" "Version 0.0.0" DEFILE
+.SH NAME
+df_add_lut \(em create a look-up table in the defile output dirfile
+.SH SYNOPSIS
+.nh
+.ad l
+.fam C
+.B #include <defile.h>
+.HP
+.BI "int df_add_lut(const char *" name ", size_t " n ", gd_type_t " x_type ,
+.BI "const void *" x ", gd_type_t " y_type ", const void *" y );
+.fam
+.ad
+.hy
+.SH DESCRIPTION
+This function is part of the defile input plugin C API. See defile-input(7) for
+an overview.
+.PP
+The
+.BR df_add_lut ()
+creates a dirfile-compatible LINTERP look-up table (LUT) called
+.I name
+in the defile output dirfile from the supplied data. The
+.I name
+may contain path elements relative to the base output dirfile directory.
+Subdirectories specified in
+.I name
+will be created if they do not already exist.
+
+The data for the LUT are given in the arrays pointed to by
+.I x
+and
+.I y .
+The data types of the two arrays are given by
+.I x_type
+and
+.IR y_type ,
+which should be one of the GetData data type symbols (see
+.BR gd_add_raw (3)),
+with the further restriction that the
+.I x
+data type may not be complex valued.
+
+In append mode (see
+.BR df_mode (3)),
+this function only tests for the existence of
+.IR name ,
+and returns an error if it is not found. In that case, all other parameters are
+ignored.
+
+This function does not add any references to the created file to the output
+dirfile.
+
+.SH RETURN VALUE
+Upon success,
+.BR df_add_lut ()
+returns
+.B DF_SUCCESS
+(=0). On error, one of the following error values (all less than zero) will be
+returned:
+.TP
+.B DF_ABORT
+Another defile subprocess has requested the input plugin immediately abort.
+Upon receiving this signal the plugin should shutdown and return. If it takes
+too long, defile will cancel it. If an abort handler has been registered (see
+.BR df_on_abort (3)),
+this value is never returned and, instead, this function does not return.
+.TP
+.B DF_INPUT
+The value of
+.I x_type
+or
+.I y_type
+was invalid.
+.B DF_OUTPUT
+An error occured attempting to create the LUT. See
+.BR mkdir (2),
+.BR open (2),
+.BR fcntl (2),
+.BR fclose (3),
+.BR fprintf (3),
+and
+.BR fputc (3)
+for possible reasons. Or, defile was in append mode, and the specified
+.I name
+did not exist.
+.TP
+.TP
+.B DF_SEQUENCE
+The function was called while defile was not in the
+.B BUILD
+state. (See defile-input(7)).
+.TP
+.B DF_SYSTEM
+A memory allocation error occurred.
+.PP
+All errors are accompanied by a message on standard error.
+.SH SEE ALSO
+.PP
+defile-input(7), defile(1), dirfile-format(5),
+.BR df_copy_file (3)
Deleted: trunk/defile/doc/df_bsearch.3
===================================================================
--- trunk/defile/doc/df_bsearch.3 2012-08-04 01:42:13 UTC (rev 755)
+++ trunk/defile/doc/df_bsearch.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -1,164 +0,0 @@
-.\" df_bsearch.3. The df_bsearch man page.
-.\"
-.\" Copyright (C) 2012 D. V. Wiebe
-.\"
-.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
-.\"
-.\" This file is part of the Defile package.
-.\"
-.\" Permission is granted to copy, distribute and/or modify this document
-.\" under the terms of the GNU Free Documentation License, Version 1.2 or
-.\" any later version published by the Free Software Foundation; with no
-.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
-.\" Texts. A copy of the license is included in the `COPYING.DOC' file
-.\" as part of this distribution.
-.\"
-.TH df_bsearch 3 "27 February 2012" "Version 0.0.0" DEFILE
-.SH NAME
-df_bsearch, df_strcmp \(em perform a binary search within defile
-.SH SYNOPSIS
-.nh
-.ad l
-.fam C
-.B #include <defile.h>
-.HP
-.BI "int df_bsearch(const void *" needle ", const void *" haystack ,
-.BI "int " start ", int " end ", size_t " size ", int *" is_new ,
-.BI "int (*" compar ")(const void*, const void*));
-.HP
-.BI "int df_strcmp(const void *" needle ", const void *" candidate );
-.fam
-.ad
-.hy
-.SH DESCRIPTION
-These functions are part of the defile input plugin C API. See defile-input(7)
-for an overview.
-.PP
-The
-.BR df_bsearch ()
-function performs a binary search on a subset of the array
-.IR haystack ,
-which has elements of size
-.I size
-bytes, looking for the element
-.IR needle ,
-using the comparison function
-.IR compar .
-The first element considered is indexed by
-.I start
-and the last element considered is immediately before the element indexed by
-.IR end .
-(In other words, to search the entirety of a
-.IR n -element
-.IR haystack ,
-set
-.I start
-to zero and
-.I end
-to
-.IR n .)
-.P
-If
-.I needle
-is not found,
-.BI * is_new
-is set to one and the index where it would be, were it in the array, is
-returned.
-If
-.I needle
-is found in
-.IR haystack ,
-its index is returned, and
-.BI * is_new
-is left untouched (so the caller should zero it beforehand, if necessary). This
-function is similar to
-.BR bsearch (3),
-but behaves differently in the case where
-.I needle
-is not in
-.IR haystack .
-It is designed for use during an insertion sort.
-.PP
-The function pointed to by
-.I compar
-will be called to compare elements of
-.I haystack
-to
-.IR needle .
-The first argument passed to
-.I compar
-is always
-.IR needle ;
-the second argument will be a pointer to the candidate array element.
-.PP
-The
-.BR df_strcmp ()
-function is provided as a potential
-.I compar
-function, useable when
-.I haystack
-is an array of character strings. The call
-.RS
-.HP
-.nh
-.ad l
-.fam C
-.BI df_strcmp( a ", " b )
-.fam
-.ad
-.hy
-.RE
-.PP
-is equivalent to the
-.BR strcmp (3)
-call:
-.RS
-.HP
-.nh
-.ad l
-.fam C
-.BI "strcmp((const char*)" a ", *(const char**)" b ")\fR.
-.fam
-.ad
-.hy
-.RE
-.PP
-(ie. it derefences its second parameter).
-.PP
-These functions are used internally by defile(1) and are simply exported to the
-defile C API as a convenience to input plugin authors. Error checking is
-minimal: out of range
-.I start
-or
-.I end
-values may cause segmentation faults. No restriction is placed on when this
-function may be called in an input plugin.
-.SH RETURN VALUE
-The
-.BR df_bsearch ()
-function returns the location of
-.I needle
-in
-.IR haystack ,
-if found, or the element in which to insert
-.I needle
-if not found. Note: while the returned value will be strictly less than
-.I end
-if found, if not found,
-.I end
-may be returned (indicating that
-.I needle
-should be appended to
-.IR haystack .
-.PP
-The
-.BR df_strcmp ()
-function returns a number with the same meaning as the return value of
-.BR strcmp (3),
-as indicated in the
-.B DISCUSSION
-section above.
-.SH SEE ALSO
-defile-input(7),
-.BR bsearch (3),
-.BR strcmp (3).
Added: trunk/defile/doc/df_copy_file.3
===================================================================
--- trunk/defile/doc/df_copy_file.3 (rev 0)
+++ trunk/defile/doc/df_copy_file.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -0,0 +1,100 @@
+.\" df_add_lut.3. The df_add_lut man page.
+.\"
+.\" Copyright (C) 2012 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" This file is part of the Defile package.
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH df_copy_file 3 "13 August 2012" "Version 0.0.0" DEFILE
+.SH NAME
+df_add_lut \(em create a file in the defile output dirfile
+.SH SYNOPSIS
+.nh
+.ad l
+.fam C
+.B #include <defile.h>
+.HP
+.BI "int df_copy_file(const char *" name ", int " fd );
+.fam
+.ad
+.hy
+.SH DESCRIPTION
+This function is part of the defile input plugin C API. See defile-input(7) for
+an overview.
+.PP
+The
+.BR df_copy_file ()
+creates a new file called
+.I name
+in the defile output dirfile and copyies the contents of the open file
+referenced by the file descriptor
+.I fd
+to it. The
+.I name
+may contain path elements relative to the base output dirfile directory.
+Subdirectories specified in
+.I name
+will be created if they do not already exist.
+
+In append mode (see
+.BR df_mode (3)),
+this function only tests for the existence of
+.IR name ,
+and returns an error if it is not found. In that case, all other parameters are
+ignored.
+
+This function does not add any references to the created file to the output
+dirfile.
+
+.SH RETURN VALUE
+Upon success,
+.BR df_copy_file ()
+returns
+.B DF_SUCCESS
+(=0). On error, one of the following error values (all less than zero) will be
+returned:
+.TP
+.B DF_ABORT
+Another defile subprocess has requested the input plugin immediately abort.
+Upon receiving this signal the plugin should shutdown and return. If it takes
+too long, defile will cancel it. If an abort handler has been registered (see
+.BR df_on_abort (3)),
+this value is never returned and, instead, this function does not return.
+.TP
+.B DF_OUTPUT
+An error occured attempting to create the file. See
+.BR mkdir (2),
+.BR open (2),
+.BR read (2),
+.BR write (2),
+and
+.BR close (2)
+for possible reasons. Or, defile was in append mode, and the specified
+.I name
+did not exist.
+.TP
+.B DF_SEQUENCE
+The function was called while defile was not in the
+.B BUILD
+state. (See defile-input(7)).
+.TP
+.B DF_SYSTEM
+A memory allocation error occurred.
+.PP
+All errors are accompanied by a message on standard error.
+.SH NOTES
+Don't use this function to create raw data files. (The desire to do so is
+often an indication that your input plugin doesn't really need to be using
+defile at all.)
+.SH SEE ALSO
+.PP
+defile-input(7), defile(1), dirfile-format(5),
+.BR df_add_lut (3)
Added: trunk/defile/doc/df_hide.3
===================================================================
--- trunk/defile/doc/df_hide.3 (rev 0)
+++ trunk/defile/doc/df_hide.3 2012-08-13 23:22:42 UTC (rev 756)
@@ -0,0 +1,82 @@
+.\" df_hide.3. The df_hide man page.
+.\"
+.\" Copyright (C) 2012 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" This file is part of the Defile package.
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH df_hide 3 "21 July 2012" "Version 0.0.0" DEFILE
+.SH NAME
+df_hide \(em hide a field in the defile output dirfile
+.SH SYNOPSIS
+.nh
+.ad l
+.fam C
+.B #include <defile.h>
+.HP
+.BI "int df_hide(cons...
[truncated message content] |
|
From: <ket...@us...> - 2012-08-14 22:27:30
|
Revision: 758
http://getdata.svn.sourceforge.net/getdata/?rev=758&view=rev
Author: ketiltrout
Date: 2012-08-14 22:27:24 +0000 (Tue, 14 Aug 2012)
Log Message:
-----------
type conversions.
Modified Paths:
--------------
trunk/defile/input/ascii.c
trunk/defile/lib/defile.h
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/input/ascii.c
===================================================================
--- trunk/defile/input/ascii.c 2012-08-14 20:26:01 UTC (rev 757)
+++ trunk/defile/input/ascii.c 2012-08-14 22:27:24 UTC (rev 758)
@@ -417,7 +417,7 @@
*ptr = ptr2 + (*ptr - *tok);
*tok = ptr2;
}
- *((*ptr)++) = c;
+ *((*ptr)++) = (char)c;
return 0;
}
@@ -1342,10 +1342,10 @@
int16_t i16 = (int16_t)l;
memcpy(d, &i16, 2);
} else if (type == GD_INT32) {
- int16_t i32 = (int32_t)l;
+ int32_t i32 = (int32_t)l;
memcpy(d, &i32, 4);
} else if (type == GD_INT64) {
- int16_t i64 = (int64_t)l;
+ int64_t i64 = (int64_t)l;
memcpy(d, &i64, 8);
}
} else {
@@ -1357,10 +1357,10 @@
uint16_t u16 = (uint16_t)u;
memcpy(d, &u16, 2);
} else if (type == GD_INT32) {
- uint16_t u32 = (uint32_t)u;
+ uint32_t u32 = (uint32_t)u;
memcpy(d, &u32, 4);
} else if (type == GD_INT64) {
- uint16_t u64 = (uint64_t)u;
+ uint64_t u64 = (uint64_t)u;
memcpy(d, &u64, 8);
}
}
Modified: trunk/defile/lib/defile.h
===================================================================
--- trunk/defile/lib/defile.h 2012-08-14 20:26:01 UTC (rev 757)
+++ trunk/defile/lib/defile.h 2012-08-14 22:27:24 UTC (rev 758)
@@ -22,6 +22,9 @@
#ifndef DEFILE_H
#define DEFILE_H
+#ifndef GD_64BIT_API
+#define GD_64BIT_API
+#endif
#include <getdata.h>
/* command line options */
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2012-08-14 20:26:01 UTC (rev 757)
+++ trunk/defile/lib/internal.h 2012-08-14 22:27:24 UTC (rev 758)
@@ -111,14 +111,14 @@
/* internal framedef */
struct df_infdef_field {
int index;
- off_t offset;
+ long long offset;
unsigned int spf;
size_t cadence;
size_t framesize;
};
struct df_infdef {
- off_t framesize;
+ long long framesize;
int n_fields;
struct df_infdef_field *field;
};
@@ -150,7 +150,7 @@
/* rate tracking */
double rate, tc;
- off_t wpartial, rpartial;
+ long long wpartial, rpartial;
long long nwrote, nread;
long long nframes;
@@ -168,7 +168,7 @@
int nfd;
struct df_infdef *fd;
- off_t rframesize, wframesize;
+ long long rframesize, wframesize;
int ref_ind;
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2012-08-14 20:26:01 UTC (rev 757)
+++ trunk/defile/lib/libdefile.c 2012-08-14 22:27:24 UTC (rev 758)
@@ -555,7 +555,7 @@
if (fd->field[i].offset + (fd->field[i].spf - 1) * c + z > fd->framesize) {
fprintf(stderr, "libdefile: input error: frame definition error: "
- "data extends beyond end of frame (%zu > %zu)\n",
+ "data extends beyond end of frame (%llu > %llu)\n",
fd->field[i].offset + (fd->field[i].spf - 1) * c + z, fd->framesize);
return DF_INPUT;
}
@@ -770,7 +770,7 @@
pthread_mutex_unlock(&config_mx);
if (offset > 0)
for (i = 0; i < df->nraw; ++i)
- if (gd_seek(df->D, df->raw[i].name, offset, 0,
+ if (gd_seek64(df->D, df->raw[i].name, offset, 0,
GD_SEEK_SET | GD_SEEK_WRITE) < 0)
{
return DF_OUTPUT;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2012-08-16 00:19:04
|
Revision: 763
http://getdata.svn.sourceforge.net/getdata/?rev=763&view=rev
Author: ketiltrout
Date: 2012-08-16 00:18:55 +0000 (Thu, 16 Aug 2012)
Log Message:
-----------
Fix LGPL boilerplate
Modified Paths:
--------------
trunk/defile/bin/rc.c
trunk/defile/configure.ac
trunk/defile/doc/defile-input.7
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2012-08-15 18:28:24 UTC (rev 762)
+++ trunk/defile/bin/rc.c 2012-08-16 00:18:55 UTC (rev 763)
@@ -58,7 +58,7 @@
"FITNESS\nFOR A PARTICULAR PURPOSE. You may redistribute it under the " \
"terms of the GNU\nLesser General Public License, either version 2.1 of the " \
"License, or (at your\noption) any later version.\n\nYou should have " \
-"received a copy of the GNU General Public License along with\n" \
+"received a copy of the GNU Lesser General Public License along with\n" \
"this plugin; if not, write to the Free Software Foundation, Inc.,\n" \
"51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA"
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2012-08-15 18:28:24 UTC (rev 762)
+++ trunk/defile/configure.ac 2012-08-16 00:18:55 UTC (rev 763)
@@ -125,7 +125,7 @@
echo
echo "*** Checking for external libraries"
echo
-PKG_CHECK_MODULES([GETDATA], [getdata >= 0.8.0])
+PKG_CHECK_MODULES([GETDATA], [getdata >= 0.8.1])
AC_SEARCH_LIBS([exp],[m])
AC_SEARCH_LIBS([pthread_create],[pthread])
Modified: trunk/defile/doc/defile-input.7
===================================================================
--- trunk/defile/doc/defile-input.7 2012-08-15 18:28:24 UTC (rev 762)
+++ trunk/defile/doc/defile-input.7 2012-08-16 00:18:55 UTC (rev 763)
@@ -161,8 +161,10 @@
section below.
.PP
All of the above members except for
+.IR name
+and
.IR entry
-are optional. If unneeded, they should be set to zero.
+are optional. If unneeded, they should be set to zero/NULL.
.SH INPUT OPTIONS
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2012-08-17 21:26:56
|
Revision: 764
http://getdata.svn.sourceforge.net/getdata/?rev=764&view=rev
Author: ketiltrout
Date: 2012-08-17 21:26:49 +0000 (Fri, 17 Aug 2012)
Log Message:
-----------
Fix input banner after autotyping. More debugging.
Modified Paths:
--------------
trunk/defile/bin/cli.c
trunk/defile/doc/Makefile.am
Modified: trunk/defile/bin/cli.c
===================================================================
--- trunk/defile/bin/cli.c 2012-08-16 00:18:55 UTC (rev 763)
+++ trunk/defile/bin/cli.c 2012-08-17 21:26:49 UTC (rev 764)
@@ -194,13 +194,20 @@
char sym[DF_TYPE_MAXLEN + 11];
/* make sure the type is sane */
- if (!DF_ValidType(type, len))
+ if (!DF_ValidType(type, len)) {
+ if (df->mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: invalid type name: %s\n", type);
return -1;
+ }
*lib = lt_dlopenext(libpath);
- if (*lib == NULL)
+ if (*lib == NULL) {
+ if (df->mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: ltdl error opening %s: %s\n", libpath,
+ lt_dlerror());
return 3;
+ }
/* find the framework */
strcpy(sym, type);
@@ -208,6 +215,8 @@
*fwk = (const struct df_input_framework *)lt_dlsym(*lib, sym);
if (*fwk == NULL) {
+ if (df->mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: ltdl error: %s\n", lt_dlerror());
lt_dlclose(*lib);
return 2;
}
@@ -216,6 +225,8 @@
if ((*fwk)->entry == NULL || (*fwk)->name == NULL ||
strcmp((*fwk)->name, type))
{
+ if (df->mode_flags & DF_MODE_DEBUG)
+ fprintf(stderr, "defile: malformed input plugin: %s\n", libpath);
lt_dlclose(*lib);
return 1;
}
@@ -1308,7 +1319,7 @@
rcd.default_config = NULL;
rcd.flags = 0,
rcd.licence = df->input_framework->licence;
- rcd.name = df->type;
+ rcd.name = df->input_framework->name;
rcd.options = df->input_framework->opts;
rcd.parser = (df_parser_func_t)DF_InputOpt;
rcd.usage_preamble = df->input_framework->preamble;
Modified: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am 2012-08-16 00:18:55 UTC (rev 763)
+++ trunk/defile/doc/Makefile.am 2012-08-17 21:26:49 UTC (rev 764)
@@ -22,7 +22,7 @@
nodist_man_MANS = defile.1
-man_MANS = defile.1 defile-input.7 df_add_alias.3 df_add_entry.3 \
+man_MANS = defile-input.7 df_add_alias.3 df_add_entry.3 \
df_add_fragment.3 df_add_framedef.3 df_add_lut.3 df_check_abort.3 \
df_copy_file.3 df_field_in_input.3 df_hide.3 df_include.3 \
df_init.3 df_input_name.3 df_mode.3 df_nframes_allowed.3 \
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-02-19 02:09:01
|
Revision: 797
http://sourceforge.net/p/getdata/code/797
Author: ketiltrout
Date: 2013-02-19 02:08:55 +0000 (Tue, 19 Feb 2013)
Log Message:
-----------
Fix cut-dir. Error reporting in RC.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/bin/rc.c
trunk/defile/doc/defile.1.in
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-02-19 00:14:17 UTC (rev 796)
+++ trunk/defile/bin/defile.c 2013-02-19 02:08:55 UTC (rev 797)
@@ -1116,13 +1116,23 @@
/* cut directories */
if (cutleft == 0)
ptr2 = ptr_base;
- else {
- cutleft = df->cutdir;
+ else if (cutleft < 0) {
+ cutleft = -cutleft;
+ /* cut from end */
for (ptr2 = ptr_base + strlen(ptr_base); ptr2 > ptr_base; --ptr2)
if (*(ptr2 - 1) == '/')
if (--cutleft == 0)
break;
+ } else {
+ /* cut from start */
+ for (ptr2 = ptr_base; *ptr2; ++ptr2)
+ if (*ptr2 == '/')
+ if (--cutleft == 0) {
+ ptr2++;
+ break;
+ }
}
+
/* remove forbidden characters */
for (ptr3 = ptr2; *ptr3; ++ptr3)
if (*ptr3 == '/')
@@ -2301,7 +2311,7 @@
"don't overwrite an existing dirfile (default)"
},
{ DF_WOPT_CUTDIR, DF_OPT_ARG_RQ, 0, "cut-dirs", "CutDirs", 0, "NUM",
- "remove all but the last NUM path elements from the output name"
+ "remove path elements from the output name"
},
{ DF_WOPT_DIR, DF_OPT_ARG_RQ, 'D', "directory", "Directory", DF_OPT_EXPAND,
"DIR", "write the output dirfile as a subdirectory under DIR"
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2013-02-19 00:14:17 UTC (rev 796)
+++ trunk/defile/bin/rc.c 2013-02-19 02:08:55 UTC (rev 797)
@@ -76,8 +76,8 @@
int required;
};
-static void DF_ReadConfigFile(const char* name, int skip_ok, const char* label,
- struct df_rc_data *rcd);
+static void DF_ReadConfigFile(const char* name, const char* label, const char *origin,
+ int origline, struct df_rc_data *rcd);
#ifdef DF_DEBUG
#define logprintf(...) fprintf(stderr, __VA_ARGS__)
@@ -422,8 +422,8 @@
}
}
-static void DF_DoRunControl (const char *val, const char *cwd,
- struct df_rc_data *rcd)
+static void DF_DoRunControl (const char *val, const char *cwd, const char *origin,
+ int origline, struct df_rc_data *rcd)
{
if (val == NULL)
logprintf(" Skipping unspecified configuration file.\n");
@@ -433,7 +433,7 @@
snprintf(file, 4096, "%s/%s", cwd, val);
else
strcpy(file, val);
- DF_ReadConfigFile(file, 0, val, rcd);
+ DF_ReadConfigFile(file, val, origin, origline, rcd);
}
}
@@ -450,7 +450,7 @@
struct df_rc_data *rcd)
{
const struct df_optdef *opt = rcd->options + i;
- const int fac = (rcd->name == NULL) ? DF_PRN_INPUT : DF_PRN_DF;
+ const int fac = (rcd->name == NULL) ? DF_PRN_DF : DF_PRN_INPUT;
if (i < 0)
opt = NULL;
@@ -459,7 +459,7 @@
rcd->options[i].value, rcd->confc, line);
if (opt) {
if (opt->type == DF_OPT_CONFIG) {
- DF_DoRunControl(val, cwd, rcd);
+ DF_DoRunControl(val, cwd, label, line, rcd);
return;
}
@@ -501,8 +501,8 @@
rcd->config[rcd->confc++].optind = i;
}
-static void DF_ReadConfigFile(const char* name, int skip_ok, const char* label,
- struct df_rc_data *rcd)
+static void DF_ReadConfigFile(const char* name, const char* label, const char *origin,
+ int orig_line, struct df_rc_data *rcd)
{
FILE* stream;
char buffer[1024];
@@ -511,16 +511,17 @@
int i, found, ln = 0;
char *file, *section = NULL;
- const int fac = (rcd->name == NULL) ? DF_PRN_INPUT : DF_PRN_DF;
+ const int fac = (rcd->name == NULL) ? DF_PRN_DF : DF_PRN_INPUT;
/* expand */
file = df_rc_shell_expand(fac, name);
if (file == NULL) {
- if (skip_ok)
+ struct df_rc_config cf;
+ /* missing is okay */
+ if (orig_line == -1)
return;
- df_iprintf(DF_PRN_DF, DF_PRN_ERR, "could not read %s: %s\n", file,
- strerror(errno));
+ df_config_error(0, &cf, rcd, "could not read %s", name);
exit(1);
}
@@ -530,16 +531,23 @@
/* attempt to open the file */
if ((stream = fopen(file, "rt")) == NULL) {
+ struct df_rc_config cf;
logprintf(" cannot open %s ", file);
- if (skip_ok)
+ if (orig_line == -1)
logprintf("(skipping)");
logprintf("\n error returned was: %s\n", strerror(errno));
- if (skip_ok) {
+
+ /* missing is okay */
+ if (orig_line == -1) {
free(file);
return;
}
- df_iprintf(DF_PRN_DF, DF_PRN_ERR, "could not read %s: %s\n", file,
+ memset(&cf, 0, sizeof(cf));
+ cf.label = origin;
+ cf.line = orig_line;
+
+ df_config_error(0, &cf, rcd, "could not read %s: %s", file,
strerror(errno));
exit(1);
}
@@ -610,7 +618,8 @@
if (strcasecmp(opt, "Include") == 0 && (section == NULL || (rcd->name &&
strcmp(section, rcd->name) == 0)))
{
- DF_DoRunControl(val, cwd, rcd);
+ DF_DoRunControl(val, cwd, file, ln, rcd);
+ logprintf(" Continuing with %s...\n", file);
continue;
}
@@ -729,7 +738,7 @@
struct shortarg *shortarg;
char* value;
- const int fac = (rcd->name == NULL) ? DF_PRN_INPUT : DF_PRN_DF;
+ const int fac = (rcd->name == NULL) ? DF_PRN_DF : DF_PRN_INPUT;
/* Find the shift character */
for (i = 0; rcd->options[i].type != DF_OPT_END_OO; ++i)
@@ -775,7 +784,7 @@
/* conf */
if (strcmp(&argv[i][2], "conf") == 0) {
- DF_DoRunControl(value, ".", rcd);
+ DF_DoRunControl(value, ".", NULL, 0, rcd);
continue;
}
@@ -865,16 +874,16 @@
rcd->global_config_fatal = 0;
/* Read the initial configuration files */
- if (rcd->local_config != NULL) {
- logprintf(" local configuration file: %s\n", rcd->global_config);
- DF_ReadConfigFile(rcd->local_config, 1, rcd->global_config, rcd);
- }
-
if (rcd->global_config != NULL) {
logprintf(" global configuration file: %s\n", rcd->global_config);
- DF_ReadConfigFile(rcd->global_config, 1, rcd->global_config, rcd);
+ DF_ReadConfigFile(rcd->global_config, rcd->global_config, NULL, -1, rcd);
}
+ if (rcd->local_config != NULL) {
+ logprintf(" local configuration file: %s\n", rcd->local_config);
+ DF_ReadConfigFile(rcd->local_config, rcd->local_config, NULL, -1, rcd);
+ }
+
/* Parse the command line */
DF_ParseCommandLine(argc, argv, rcd);
Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in 2013-02-19 00:14:17 UTC (rev 796)
+++ trunk/defile/doc/defile.1.in 2013-02-19 02:08:55 UTC (rev 797)
@@ -203,8 +203,7 @@
will be provided by the input plugin, or else be based on
.IR INPUT .
If
-.B \-\-output
-does specify an absolute path, this option is ignored. (Directive:
+.B \-\-output does specify an absolute path, this option is ignored. (Directive:
.BR Directory )
.TP
.BR \-o ", " \-\-output=\fIPATH
@@ -240,15 +239,20 @@
.I INPUT
designator (after possible modification by the input plugin). If
.I N
-is greater than zero, all but the last
+is greater than zero, the first
.I N
-path elements of
-.I INPUT
-are stripped from it before using it to generate the output dirfile name.
-Remaining path separators will be replaced by the underscore character. The
-default is zero, indicating no directories are cut. Ignored if
+path elements of the designator are stripped from it before using it to generate the
+output dirfile name. Remaining path separators will be replaced by the underscore
+character. If
+.I N
+is less than zero, all but the last
+.RI \- N
+path elements are stripped.
+
+The default is zero, indicating no directories are cut. Ignored if
.B \-\-output
-is used to explicitly name the output dirfile.
+is used to explicitly name the output dirfile. (Directive:
+.BR CutDirs )
.SS Running Mode Options
The following options affect the modes in which defile runs.
@@ -341,7 +345,7 @@
.BR \-q ", " \-\-quiet
.I Quiet Mode on\fR:
write nothing to the controlling terminal except error messages. Debugging
-messages output as a result of
+messages produced by
.B \-\-debug
ignore this. (Directive:
.BR Quiet )
@@ -396,7 +400,9 @@
Read data from the local system. This is the default behaviour, and this option
is only needed to cancel a
.B \-\-remote
-option given earlier in the configuration. (Directive:
+option given earlier in the configuration. This is not the same as specifying
+.BR \-\-remote=localhost .
+(Directive:
.BR Local )
.TP
.BI \-\-remote\-command= COMMAND
@@ -405,7 +411,7 @@
as the defile program on the remote host. The default is simply "defile" and
any other program specified must support the defile command-line. The
.I COMMAND
-is interpreted by the (local) shell on the remote host, which (assuming
+is interpreted by the shell on the remote host, which (assuming
the shell on the remote machine is sh(1)-compatible) permits doing something
like:
.RS
@@ -417,7 +423,7 @@
.ad n
.RE
.IP
-to set the value of DEFILE_PLUGIN_PATH on the fly. See
+to set the value of DEFILE_PLUGIN_PATH on the remote machine on the fly. See
.BR ENVIRONMENT
below for details on DEFILE_PLUGIN_PATH. (Directive:
.BR RemoteCommand )
@@ -434,8 +440,8 @@
.BI \-\-rsh= COMMAND
Use
.I COMMAND
-as the remote shell used to log into the remote host. By default this is "ssh",
-and
+as the remote shell used to log into the remote host. By default this is "ssh" (see
+ssh(1)), and
.I COMMAND
must be "ssh compatible" in the sense that the second last argument passed to
it will be the
@@ -494,7 +500,7 @@
.I Note:
this option does not change the state of
.IR Follow ", " Append ", or " "Asynchronous Mode" .
-It is always best to explicitly set these mode to the desired state after using
+It is always best to explicitly set these modes to the desired state after using
this option. (Directive:
.BR NoListen )
.TP
@@ -552,10 +558,12 @@
set the length of the output buffer to
.I N
frames. The default value is 3000 frames. Decreasing this number will reduce
-memory usage and may reduce output latency at the expense of read and write
+memory usage, and may reduce output latency, at the expense of read and write
speed. If this number is smaller than the buffer head (see
.BR \-\-buf\-head ),
-the output buffer will be exapnded to the size of the buffer head.
+the output buffer will be expanded to the size of the buffer head. The buffer can
+never be made smaller than two frames. (Directive:
+.BR BufferLength )
.SS Output Field List Options
A restricted set of fields to write to the output dirfile may be specified with
@@ -677,7 +685,7 @@
.SS Miscellaneous options
Unless otherwise specified, including these options on the command line will
-result defile performing the indicated behaviour and exiting successfully,
+result in defile performing the indicated action and exiting successfully,
without doing any actual data conversion.
In addition to appearing as defile options (before the double dash), the
@@ -707,7 +715,7 @@
.B \-\-list\-types
print a list of all input plugins available to defile. For each input plugin
found, defile will print the plugin name, author, a brief description of the
-data format handled (if provided by the plugin), and the location of the plugin
+data format handled (if the plugin provides these data), and the location of the plugin
on disk.
.TP
.B \-\-help
@@ -726,29 +734,41 @@
.IP
\fBDirective\fR [\fIvalue\fR]
.PP
-A hash mark (#) is used as a comment delimeter; any line starting with a hash
-mark is ignored. At the top of every run control file are the defile directives
-(which correspond with defile options). After the defile directives may come
-sections of directives for various input plugins. Each of these sections is
+A hash mark (#) is used as a comment delimeter; everything from the hash mark to
+the end of the line is ignored. At the top of every run control file are the defile
+directives (which correspond with defile options). After the defile directives may
+come sections of directives for various input plugins. Each of these sections is
headed by a section heading indicating the input plugin type between square
brackets. For example:
.IP
[ascii]
.PP
+If a file contains more than one section for the same input plugin, they are
+concatenated together into a single section.
+.PP
Run Control files may themselves include other run control files to arbitrary
depth using the
.B Include
-directive. Unlike all other directives, this directive ignores sections: the
-included file is read wherever it appears in the file and the contents of all
-it's sections are effectively merged into the appropriate sections of the
-parent file.
+directive.
+.B Include
+directives which appear in the top (unnamed) section reserved for Defile options are
+always processed, regardless of whether Defile is processing Defile options or input
+options (of any type).
+.B Include
+directives placed inside an input plugin section are only processed when reading
+configuration for the specified input plugin. Despite this, these included files
+must still contain an appropriate section header to be properly processed.
-The default run control file, @absolute_sysconfdir@/defilerc, is read (if
+The default global run control file, @absolute_sysconfdir@/defilerc, is read (if
present) and all the defile directives found within (plus those in any other
-configuration files it includes) are inserted before the first command line
-option. The contents of the appropriate input plugin section is also inserted
-before the first input option.
+configuration files it includes) first. After this, the per-user run control file,
+${HOME}/.defilerc, is read (if present) and processed similarly. The local per-user
+configuration file is read second to permit it to override the global configuration.
+Configuration found in these two default run control files are inserted before the
+first command line option. The contents of the appropriate input plugin sections from
+these files are also inserted before the first input option.
+
Parsing of other run control files specified by the
.B \-\-conf
option on the command line occurs a little differently: in this case, only one
@@ -756,7 +776,9 @@
defile directives, or else the input plugin section for the input type in use,
depending on whether the
.B \-\-conf
-option appears before or after the double dash.
+option appears before or after the double dash (with the exception of
+.I Include
+directives in the top section, which are always followed).
.SH NETWORKING
@@ -766,15 +788,20 @@
In the
.IR "server-client model" ,
the local defile connects to a defile running in server mode on the remote host.
-It is designed for tranmitting data from an in-progress data acquisition to a
-many client hosts.
+It is designed for tranmitting data from an in-progress data acquisition to many client
+hosts.
.IP \(bu
In the
.IR "remote shell model" ,
-the local defile logs into the remote host (using a remote shell such as ssh)
+the local defile logs into the remote host (using a remote shell such as ssh(1))
and starts a one-time-use defile process on the remote host. It is designed for
the transmission of a single data source from the remote to the local host.
.PP
+Both methods require defile to be installed on the remote machine, and the
+local and remote defiles must speak compatible versions of the defile network
+protocol.
+
+.SS The Server-Client Model
A defile server is started by specifying a numeric port to listen on with the
.B \-\-listen
option. Servers are almost always run in
@@ -792,13 +819,16 @@
by the input plugin which, in turn, is typically rate limited by whatever
process is producing the data. If a network connection or client is unable to
keep up with the data rate, the server will simply discard unsent frames rather
-than get backlogged. However, a bandwidth limit is also imposed (this may be
+than get backlogged. However, a bandwidth throttle is also imposed (this may be
adjusted using the
.B \-\-server\-bwlimit
-option). A server will rate limit itself to the specified limit, even if that
-results in the server not keeping up with the input data rate. Note: at
-high input data rates (eg. when reading previously acquired data, where the data
-rate is usually limited only by the speed at which the input plugin can extract
+option). A server will rate-limit itself to the specified limit, even if that
+results in the server not keeping up with the input data rate. This is a global rate,
+not a per-client rate: more clients connecting to a throttled server will
+effectively increase the amount of per-client throttling.
+
+Note: at high input data rates (eg. when reading previously acquired data, where the
+data rate is usually limited only by the speed at which the input plugin can extract
it from the input data source) the server may want to write data faster than the
underlying network link allows. In this case data will always be dropped. As
a result, it's always a good idea to match the server bandwidth limit to the
@@ -807,7 +837,7 @@
To connect to a server, specify the remote host and port using the
.B \-\-remote
option. When the server detects a connection from a new client, it first sends
-the output dirfile metadata, and then sends data frames as they are ready. The
+the output dirfile metadata, and then sends data frames as they become ready. The
data that the client receives won't start with the first frame of the input
data source, but will be whatever frame the server happened to transmit next.
As a result, dirfile created using the server-client method almost always have
@@ -817,12 +847,13 @@
designator should be given to the server: such arguments to the client defile
will be simply ignored.
+.SS The Remote Shell Model
The remote shell method is instantiated by specifying the remote host, without a
port number, using the
.B \-\-remote
option. A username may also be given if the remote username is different than
the local one. In this method, the local defile will log into the remote
-host using ssh (or another transparent remote shell specified with
+host using ssh(1) (or another transparent remote shell specified with
.BR \-\-rsh )
and then run an instance of defile on the remote host. In this case, the data
rate will be limited both by the input plugin as well as the network connection
@@ -833,20 +864,21 @@
interpretation. The input type need not even be supported by the local,
receiving defile.
-Both methods require defile to be installed on the remote machine, and the
-local and remote defiles must speak compatible versions of the defile network
-protocol.
+Because this method requires each client to start its own defile process on the
+remote machine, the remote host can get quickly bogged down by multiple clients
+simultaneously requesting data from it.
.SH FILES
.TP
-.I ${HOME}/.defilerc
-the per-user configuration file, which will be read first, before both the
-system-wide configuration file, and also options on the command line. See the
+.I @absolute_sysconfdir@/defilerc
+the defile system-wide configuration file, which will be read first, before both
+the per-user configuration file, and also options on the command line. See
+the
.B RUN CONTROL FILES
section above for details on syntax.
.TP
-.I @absolute_sysconfdir@/defilerc
-the defile system-wide configuration file, which will be read after the per-user
+.I ${HOME}/.defilerc
+the defile per-user configuration file, which will be read after the system-wide
configuration file (if any), but before the options on the command line.
See the
.B RUN CONTROL FILES
@@ -879,7 +911,7 @@
Defile was written by D. V. Wiebe.
.SH REPORTING BUGS
-Please send reports of bugs to get...@li...
+Please send reports of bugs to @PACKAGE_BUGREPORT@.
The Defile home page is: <http://getdata.sourceforge.net/defile/>
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-05 04:34:26
|
Revision: 799
http://sourceforge.net/p/getdata/code/799
Author: ketiltrout
Date: 2013-03-05 04:34:19 +0000 (Tue, 05 Mar 2013)
Log Message:
-----------
Add df_update_frames. Also some aclocal macros.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/lib/Makefile.am
trunk/defile/lib/defile.h
trunk/defile/lib/libdefile.c
Added Paths:
-----------
trunk/defile/lib/defile.m4
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-03-05 01:14:46 UTC (rev 798)
+++ trunk/defile/bin/defile.c 2013-03-05 04:34:19 UTC (rev 799)
@@ -2516,7 +2516,11 @@
}
/* remember and release our CWD */
- getcwd(df->orig_cwd, 4096);
+ if (getcwd(df->orig_cwd, 4096) == NULL) {
+ df_dperror(DF_PRN_ERR, "getcwd");
+ exit(1);
+ }
+
if (chdir("/") < 0) {
df_dperror(DF_PRN_ERR, "chdir");
exit(1);
@@ -2527,9 +2531,18 @@
df->syslog = 1;
/* drop the TTY */
- freopen("/dev/null", "r", stdin);
- freopen("/dev/null", "w", stdout);
- freopen("/dev/null", "w", stderr);
+ if (freopen("/dev/null", "r", stdin) == NULL) {
+ df_dperror(DF_PRN_ERR, "redirecting stdin");
+ exit(1);
+ }
+ if (freopen("/dev/null", "w", stdout) == NULL) {
+ df_dperror(DF_PRN_ERR, "redirecting stdout");
+ exit(1);
+ }
+ if (freopen("/dev/null", "w", stderr) == NULL) {
+ df_dperror(DF_PRN_ERR, "redirecting stderr");
+ exit(1);
+ }
}
}
}
Modified: trunk/defile/lib/Makefile.am
===================================================================
--- trunk/defile/lib/Makefile.am 2013-03-05 01:14:46 UTC (rev 798)
+++ trunk/defile/lib/Makefile.am 2013-03-05 04:34:19 UTC (rev 799)
@@ -20,6 +20,9 @@
#
AUTOMAKE_OPTIONS = foreign
+aclocaldir = $(datadir)/aclocal
+aclocal_DATA = defile.m4
+
lib_LTLIBRARIES = libdefile.la
include_HEADERS = defile.h
AM_CPPFLAGS = ${GETDATA_CFLAGS}
Modified: trunk/defile/lib/defile.h
===================================================================
--- trunk/defile/lib/defile.h 2013-03-05 01:14:46 UTC (rev 798)
+++ trunk/defile/lib/defile.h 2013-03-05 04:34:19 UTC (rev 799)
@@ -166,6 +166,7 @@
int df_set_string(const char *name, const char *string);
int df_set_carray(const char *name, unsigned int start, size_t len,
gd_type_t type, const void *data_in);
+int df_update_nframes(long long nframes, int add);
/* the remaining functions declared here can be used at any time */
int df_check_abort(void);
Added: trunk/defile/lib/defile.m4
===================================================================
--- trunk/defile/lib/defile.m4 (rev 0)
+++ trunk/defile/lib/defile.m4 2013-03-05 04:34:19 UTC (rev 799)
@@ -0,0 +1,87 @@
+dnl Copyright (C) 2013 D. V. Wiebe
+dnl
+dnl llllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllllll
+dnl
+dnl This file is free software; the copyright holder gives unlimited
+dnl permission to copy and/or distribute it, with or without modifications,
+dnl as long as this notice is preserved.
+dnl
+dnl This program is distributed in the hope that it will be useful,
+dnl but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+dnl even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+dnl PARTICULAR PURPOSE.
+
+dnl DF_CHECK_DEFILE_H
+dnl -----------------------------------------------------
+dnl Look for defile.h. AC_DEFINEs HAVE_DEFILE_H if found.
+AC_DEFUN([DF_CHECK_DEFILE_H],
+[AC_CHECK_HEADERS([defile.h],, [AC_MSG_FAILURE([defile.h not found])])])
+
+dnl DF_CHECK_LIBDEFILE
+dnl -----------------------------------------------------------
+dnl Try to find libdefile -- a normal AC_SEARCH_LIBS doesn't work, unfortunately
+dnl due to unresolved symbols in the library
+AC_DEFUN([DF_CHECK_LIBDEFILE],
+ [AC_CACHE_CHECK([for libdefile], [ac_cv_libdefile],
+ [df_check_lib_save_LIBS=$LIBS
+ LIBS="-ldefile $LIBS"
+ AC_LINK_IFELSE([AC_LANG_CALL([
+void *df;
+void *df_rpc_push_string;
+void *df_rpc_push_double;
+void *df_rpc_push_uint32;
+void *df_discard_metadata;
+void *df_copy_swap;
+void *df_rpc_push_uint8;
+void *df_rpc_push_data;
+void *df_ivprintf;
+void *df_rpc_push_mem;
+void *df_rpc_push_uint64;
+void *df_rpc_fini;
+void *df_rpc_push_uint16;
+void *df_rc_shell_expand;
+void *df_rpc_del;
+void *df_rpc_init;
+void *df_iperror;
+void *df_iputc;
+void *df_iputs;
+void *df_iprintf;
+void *df_rpc_push_complex_double;
+void *df_state_mx;
+void *df_config_mx;
+void *df_rate_mx;
+void *df_print_mx;
+], [df_init])],
+ [ac_cv_libdefile="yes"], [ac_cv_libdefile="no"])])
+ LIBS="$df_check_lib_save_LIBS"
+ if test "x$ac_cv_libdefile" = "xyes"; then
+ LIBS="-ldefile $LIBS"
+ else
+ AC_MSG_FAILURE([libdefile not found])
+ fi dnl
+])
+
+dnl DF_CHECK_DEFILE
+dnl ----------------------------------------------------
+dnl Look for the defile program. Declares DEFILE as precious (see AC_ARG_VAR)
+AC_DEFUN([DF_CHECK_DEFILE],
+ [AC_ARG_VAR([DEFILE], [the path to the defile program])
+ AC_PATH_PROGS([DEFILE], [defile], [not found])
+ if test "x$DEFILE" = "xnot found"; then
+ AC_MSG_ERROR([defile not found])
+ fi
+ ])
+
+dnl DF_PLUGIN_PATH
+dnl -------------------------------------------
+dnl Determine the plugin path by running defile. Declares "defiledir" to be
+dnl precious.
+AC_DEFUN([DF_PLUGIN_PATH],
+ [AC_REQUIRE([DF_CHECK_DEFILE])
+ AC_ARG_VAR([defiledir], [the defile plugin directory])
+ AC_MSG_CHECKING([the defile plugin path])
+ if test "x$defiledir" = "x"; then
+ defiledir=`$DEFILE --lib-dir`
+ fi
+ AC_MSG_RESULT([$defiledir])dnl
+])
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-03-05 01:14:46 UTC (rev 798)
+++ trunk/defile/lib/libdefile.c 2013-03-05 04:34:19 UTC (rev 799)
@@ -665,6 +665,9 @@
free(type);
}
+ if (ret == DF_SUCCESS)
+ df_lprintf(DF_PRN_DEBUG_I, "added spec: \"%s\"\n", spec);
+
free(name);
return ret;
@@ -725,7 +728,7 @@
if (!(append || df->mode_flags & DF_MODE_SERVER)) {
/* try adding it to the dirfile */
- if (gd_add_raw(df->D, f->name, f->type, f->spf, fragment))
+ if (gd_add_raw(df->D, f->name, f->type, f->spf * count, fragment))
return DF_OUTPUT;
}
@@ -1042,6 +1045,23 @@
return DF_SUCCESS;
}
+int df_update_nframes(long long nframes, int add)
+{
+ DF_CHECK_SEQUENCE2(DF_ST_BUILD, DF_ST_RUN);
+
+ /* don't bother doing this in follow mode */
+ if (!(df->mode_flags & DF_MODE_FOLLOW)) {
+ pthread_mutex_lock(&df_config_mx);
+ if (add)
+ df->nframes += nframes;
+ else
+ df->nframes = nframes;
+ pthread_mutex_unlock(&df_config_mx);
+ }
+
+ return DF_SUCCESS;
+}
+
int df_init(long long nframes, double rate, const char *name)
{
DF_CHECK_SEQUENCE(DF_ST_START);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-07 04:31:39
|
Revision: 800
http://sourceforge.net/p/getdata/code/800
Author: ketiltrout
Date: 2013-03-07 04:31:35 +0000 (Thu, 07 Mar 2013)
Log Message:
-----------
doc updates; fix defile-ascii and defile-dirfile; conditionalise compiling ./test;
Modified Paths:
--------------
trunk/defile/configure.ac
trunk/defile/doc/Makefile.am
trunk/defile/doc/defile-input.7
trunk/defile/doc/defile.1.in
trunk/defile/doc/df_add_alias.3
trunk/defile/doc/df_add_entry.3
trunk/defile/doc/df_add_fragment.3
trunk/defile/doc/df_add_framedef.3
trunk/defile/doc/df_add_lut.3
trunk/defile/doc/df_check_abort.3
trunk/defile/doc/df_exit.3
trunk/defile/doc/df_field_in_input.3
trunk/defile/doc/df_get_offset.3
trunk/defile/doc/df_hide.3
trunk/defile/doc/df_include.3
trunk/defile/doc/df_init.3
trunk/defile/doc/df_input_name.3
trunk/defile/doc/df_mode.3
trunk/defile/doc/df_nframes_allowed.3
trunk/defile/doc/df_on_abort.3
trunk/defile/doc/df_push_frame.3
trunk/defile/doc/df_puts.3
trunk/defile/doc/df_ready.3
trunk/defile/doc/df_set_const.3
trunk/defile/doc/df_shell_expand.3
trunk/defile/doc/df_subset_contains.3
trunk/defile/doc/df_subset_init.3
trunk/defile/doc/df_subset_reset.3
trunk/defile/doc/df_subset_type.3
trunk/defile/doc/df_subset_update.3
trunk/defile/doc/df_write_file.3
trunk/defile/input/Makefile.am
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/internal.h
trunk/defile/test/Makefile.am
Added Paths:
-----------
trunk/defile/doc/defile-ascii.1
trunk/defile/doc/defile-dirfile.1
trunk/defile/doc/df_update_length.3
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2013-03-05 04:34:19 UTC (rev 799)
+++ trunk/defile/configure.ac 2013-03-07 04:31:35 UTC (rev 800)
@@ -45,6 +45,17 @@
echo
AC_CANONICAL_HOST
+AC_ARG_ENABLE(test, AS_HELP_STRING([--enable-test],
+ [build the defile test programs]),
+ [
+ case "${enable_val}" in
+ yes) enable_test="yes" ;;
+ *) enable_test="no" ;;
+ esac
+ ], [ enable_test="no" ])
+AC_MSG_CHECKING([whether to build the test programs])
+AC_MSG_RESULT([$enable_test])
+
AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],
[enable debugging messages]),
[
@@ -163,7 +174,7 @@
echo
echo "*** Looking for additional library functions"
echo
-AC_CHECK_FUNCS([canonicalize_file_name realpath wordexp])
+AC_CHECK_FUNCS([canonicalize_file_name realpath strtoll strtoull wordexp])
echo
echo "*** Looking for external programs"
@@ -191,6 +202,9 @@
AC_SUBST([absolute_moduledir])
AC_SUBST([absolute_sysconfdir])
+dnl Automake conditionals
+AM_CONDITIONAL(TEST_DEFILE, [test "x$enable_test" == "xyes"])
+
echo
echo "*** Writing configure output"
echo
Modified: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am 2013-03-05 04:34:19 UTC (rev 799)
+++ trunk/defile/doc/Makefile.am 2013-03-07 04:31:35 UTC (rev 800)
@@ -20,16 +20,17 @@
#
AUTOMAKE_OPTIONS = foreign
-nodist_man_MANS = defile.1
+nodist_man1_MANS = defile.1
-man_MANS = defile-input.7 df_add_alias.3 df_add_entry.3 \
- df_add_fragment.3 df_add_framedef.3 df_add_lut.3 df_check_abort.3 \
- df_exit.3 df_field_in_input.3 df_hide.3 df_get_offset.3 \
- df_include.3 df_init.3 df_input_name.3 df_mode.3 \
- df_nframes_allowed.3 df_on_abort.3 df_push_frame.3 df_puts.3 \
- df_ready.3 df_set_const.3 df_shell_expand.3 df_subset_contains.3 \
- df_subset_init.3 df_subset_reset.3 df_subset_update.3 \
- df_write_file.3
+man1_MANS = defile-ascii.1 defile-dirfile.1
+man7_MANS = defile-input.7
+man3_MANS = df_add_alias.3 df_add_entry.3 df_add_fragment.3 df_add_framedef.3 \
+ df_add_lut.3 df_check_abort.3 df_exit.3 df_field_in_input.3 \
+ df_hide.3 df_get_offset.3 df_include.3 df_init.3 df_input_name.3 \
+ df_mode.3 df_nframes_allowed.3 df_on_abort.3 df_push_frame.3 \
+ df_puts.3 df_ready.3 df_set_const.3 df_shell_expand.3 \
+ df_subset_contains.3 df_subset_init.3 df_subset_reset.3 \
+ df_subset_update.3 df_update_length.3 df_write_file.3
# these are real_file:alias
MAN3ALIASES=df_add_entry.3:df_add_spec.3 \
@@ -57,4 +58,4 @@
clean-local:
rm -f *~
- rm -f $(nodist_man_MANS)
+ rm -f $(nodist_man1_MANS)
Added: trunk/defile/doc/defile-ascii.1
===================================================================
--- trunk/defile/doc/defile-ascii.1 (rev 0)
+++ trunk/defile/doc/defile-ascii.1 2013-03-07 04:31:35 UTC (rev 800)
@@ -0,0 +1,489 @@
+.\" defile-ascii.1 The ASCII input plugin man page for defile.
+.\"
+.\" Copyright (C) 2013 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH defile-ascii 1 "5 March 2013" "Version 0.0.1" DEFILE
+.SH NAME
+defile-ascii \(em an ASCII input plugin for Defile
+.SH SYNOPSIS
+.nh
+.ad l
+\fBdefile\fR [ \fIDEFILE\-OPTION\fR ]... [ -t \fIascii\fR ]
+[ [ -i ] \fIINPUT\fR ] [ \fB\-\-\fR [ \fIINPUT\-OPTION\fR ]... ]
+.hy
+.ad n
+.SH DESCRIPTION
+The defile-ascii plugin provides support in defile(1) for converting ASCII
+tabular data into dirfiles.
+
+For defile-ascii, the input designator,
+.IR INPUT ,
+passed to defile should be the pathname to the input file.
+
+The defile-ascii plugin expects each frame to appear on a single line, with
+succuessive frames separated by a newline. Fields are separated into columns.
+This input plugin is somewhat configurable for different input formats: either
+fixed-width columns, or else separator-delimited columns can be read, with
+optional comments and delimiters.
+
+In separator-delimited mode, a line, representing one frame, is expected to be
+of the form:
+.IP
+\fB[\fISD\fB]\fRfield1\fB[\fIED\fB]\fISEP\fB[\fISD\fB]\fRfield2\fB[\fIED\fB]\fISEP\fB
+\fB...\~[\fISD\fB]\fRfield\fIn\fB[\fIED\fB][\fICD\fR \fB...]\\n\fR
+.PP
+where:
+.TP
+.I SD
+is the start delimiter, if any (see
+.BR \-\-start\-delim );
+.TP
+.I ED
+is the end delimiter, if any (see
+.BR \-\-end\-delim );
+.TP
+.I SEP
+is the separator (see
+.BR \-\-separator );
+.TP
+.I CD \fB...
+is, optionally, a comment delimiter (see
+.BR \-\-comment )
+followed by a free-form comment; and
+.TP
+.B \\\\n
+is the newline character.
+.PP
+The field delimiters,
+.I SD
+and
+.I ED
+may be optional or required. The delimiters
+.IR SD ", " ED ", and " CD
+may be multiple characters long, in which case the whole string must be matched.
+The separator,
+.IR SEP ,
+may either be a string or a collection of possible separator characters (see
+.BR \-\-sep\-set ).
+
+In both modes, lines which begin with the comment delimiter (i.e. lines
+containing only comments) and blank lines are simply skipped.
+
+If the configurability of the input plugin is insufficient for the intended
+input data source, arbitrary external filters (see
+.BR \-\-filter )
+can be employed to convert any arbitrary input data file into something
+defile-ascii can cope with. See also the
+.B NOTES
+section below.
+
+Numerical values in the input data source are parsed using one of
+.BR strtod (3),
+.BR strtoul (3),
+or
+.BR strtol (3),
+as appropriate for the data type of the field. (If available, "long long"
+versions of the integer conversion functions will be used; C99 and POSIX define
+.BR strtoull (3)
+and
+.BR strtoll (3).)
+The default data type is double-precision floating point; use
+.BR \-\-type
+to change data types. In addition to decimal numbers, octal and hexadecimal
+integers are supported, and also special strings representing floating-point
+infinities and NANs. If
+.BR strtod (3)
+is C99-conformant, hexadecimal floating point numbers are also accepted. In
+some implementations, these functions are locale-aware. See the listed manual
+pages for exact syntax specification for numbers.
+
+As an extension to the above syntax, defile-ascii can also read complex-valued
+floating-point data, which are composed of two floating-point numbers (parsed by
+.BR strtod (3))
+separated by a semicolon
+.RI ( ; ),
+with no intervening whitespace. This is the GetData convension; see
+dirfile-format(5). The first floating-point number is the value of the real
+part of the complex number and the second is the value of the imaginary part.
+
+The defile-ascii plugin cannot handle data fields with more than one sample per
+frame.
+
+.SS Metadata
+Field names, and limited other field metdatada, can be read from the file in
+the form of an optional multi-line header block. Field names, if provided,
+must be given in the first line of the header. Subsequent header lines may
+contain per-field scalar or string metadata.
+
+The presence of field names at the top of the header is indicated using the
+.B \-\-names
+option. The meaning of successive rows of the header are defined by
+.BR \-\-const\-row
+and
+.BR \-\-string\-row ,
+which are treated in the order they are presented to defile-ascii. For each
+header line, including the field names, the line is parsed in the same manner as
+the rest of the data: either as fixed-width columns, or else with the same
+delimiters specified. By default, these header data may appear as comment lines
+Use
+.B \-\-no\-meta\-comments
+to disable this.
+
+If no field names are provided, defile-ascii will make generic numbered column
+names of the form
+.BI col #
+will be used, with the zero-padded numerical part an appropriate width for the
+number of columns in the file. Column numbering starts at zero. So, for
+example, a file containing four fields will start with
+.RI """" col0 """"
+and end with
+.RI """" col3 ""","
+while file containing twenty-four fields will start with
+.RI """" col00 """"
+and end with
+.RI """" col23 """."
+
+.SH INPUT OPTIONS
+
+All these options must be placed after the double-dash in the defile command
+line. When used as directives in defile run control files, they must be placed
+in the
+.IP
+[ascii]
+.PP
+section. Mandatory arguments to long options are mandatory for short options
+too.
+
+.TP
+.BR \-c ", " \-\-comment=\fIDELIM\fR
+use
+.I DELIM
+as the comment delimiter: everything from the comment delimiter to the end of
+the line is ignored. To disable comments, set this to nothing:
+.RS
+.IP
+.ad l
+.nh
+\-\-comment=
+.hy
+.ad n
+.RE
+.IP
+The default is
+.RI ' # '.
+(Directive:
+.BR CommentDelim )
+.TP
+.BR \-\-clear\-header
+forget any previously defined header rows; that is, ignore any previous
+.BR \-\-names ,
+.BR \-\-const\-row ,
+or
+.B \-\-string\-row
+options seen. (Directive:
+.BR ClearHeader )
+.TP
+.BR \-\-const\-row= \fINAME\fR[,\fITYPE\fR]
+for each field in the output, create a metafield called
+.IR field / NAME
+for each
+.I field
+in the input of type
+.B CONST
+(and data type
+.IR TYPE ,
+if given), from data in the first available row in the header. See the
+.B Metadata
+section above for detals on the header. This option may be given multiple
+times (with different
+.IR NAME s)
+to define multiple
+.B CONST
+metadata fields. See
+.B \-\-type
+for details on permitted values for
+.IR TYPE .
+The default type is
+.BR FLOAT64 .
+(Directive:
+.BR ConstRow )
+.TP
+.BR \-d ", " \-\-delim\-req
+the start-of-field and end-of-field delimiters are required. (Directive:
+.BR DelimitersRequired )
+.TP
+.BR +d ", " \-\-delim\-opt
+the start-of-field and end-of-field delimiters are option. This is the default.
+(Directive:
+.BR DelimitersOptional )
+.TP
+.BR \-e ", " \-\-escape=\fICHAR
+use
+.I CHAR
+as the escape character: the escape character can be used to include a character
+in a item which would otherwise be interpreted as a delimiter. To disable
+character escapes, set this to nothing:
+.RS
+.IP
+.ad l
+.nh
+\-\-escape\-char=
+.hy
+.ad n
+.RE
+.IP
+The default is
+.RI ' \e '.
+(Directive:
+.BR EscapeChar )
+.TP
+.BI \-\-end\-delim= DELIM
+use
+.I DELIM
+as the end-of-field delimiter. The default is nothing. (Directive:
+.BR EndDelimiter )
+.TP
+.BR \-f ", " \-\-filter=\fICOMMAND
+pipe the input file through the filter
+.I COMMAND
+and parse the filtered output, instead of the file itself. The
+.I COMMAND
+string specified is interpreted by the shell. This option may be given multiple
+times to build up a filter chain. Each filter should read its standard input
+and write to its standard output. See the
+.B NOTES
+section below for a discussion of this feature. A filter which exits with a
+non-zero status will cause defile-ascii to abort conversion. (Directive:
+.BR Filter )
+.TP
+.BR +f ", " \-\-clear\-filters
+ignore all previous
+.B \-\-filter
+definition. (Directive:
+.BR ClearFilters )
+.TP
+.BR \-l ", " \-\-lazy
+allow trailing, non-numerical characters after a numerical value. (Directive:
+.BR Lazy )
+.TP
+.BR +l ", " \-\-strict
+prohibit trailing, non-numerical characters after a numberical value. This
+is the default. (Directive:
+.BR Strict )
+.TP
+.BR \-m ", " \-\-no\-meta\-comments
+the metadata header (see
+.B Metadata
+above) appear without a preceding comment delimiter. In this case, lines
+containing only comments within the metadata header are simply skipped.
+(Directive:
+.BR MetadataSkipComments )
+.TP
+.BR +m ", " \-\-meta\-comments
+the metadata header (see
+.B Metadata
+above)
+.I may
+appear inside comments. This is the default. (Directive:
+.BR MetadataInComments )
+.TP
+.BR \-n ", " \-\-names
+read field names from the first row of the header. See the
+.B Metdata
+section above for details of the header. (Directive:
+.BR Names )
+.TP
+.BR \-q ", " \-\-quoted
+items are enclosed in quotation marks; equivalent to
+.BI \-\-start\-delim= \e"
+.BI \-\-end\-delim= \e"
+(Directive:
+.BR Quoted )
+.TP
+.BR \-S ", " \-\-sep\-set
+consider the separator provided to
+.B \-\-sep
+to be a set of characters, and separate items with a run of one or more of these
+characters, in any combination. (Directive:
+.BR SepSet )
+.TP
+.BR +S ", " \-\-sep\-literal
+consider the separator provided to
+.B \-\-sep
+to be a literal string which is found verbatim between items. This is the
+default behaviour. (Directive:
+.BR SepLiteral )
+.TP
+.BR \-s ", " \-\-sep=\fISEPAR
+use
+.I SEPAR
+as the field separator. How this is interpreted is affected by
+.BR \-\-sep\-set
+and
+.BR \-\-sep\-literal
+(q.v.). This option also turns off fixed-width mode if it had been previously
+enabled. The default separator is the comma
+.RI ( , ).
+(Directive:
+.BR Separator )
+.TP
+.BI \-\-skip= NUM
+skip the first
+.I NUM
+lines. The metadata header (see
+.B Metadata
+above), if present, starts with the next row after the lines skipped. The
+default is zero. (Directive:
+.BR Skip )
+.TP
+.BI \-\-start\-delim= DELIM
+use
+.I DELIM
+as the start-of-field delimiter. The default is nothing. (Directive:
+.BR StartDelimiter )
+.TP
+.BI \-\-string\-row= NAME
+for each field in the output, create a metafield called
+.IR field / NAME
+for each
+.I field
+in the input of type
+.B STRING
+from data in the first available row in the header. See the
+.B Metadata
+section above for detals on the header. This option may be given multiple
+times (with different
+.IR NAME s)
+to define multiple
+.B STRING
+metadata fields. (Directive:
+.BR StringRow )
+.TP
+.BR \-t ", " \-\-type= [\fICOL\fR,]\fITYPE
+If a numeric
+.I COL
+is given, make column
+.I COL
+of type
+.IR TYPE .
+If no
+.I COL
+is given, use
+.I TYPE
+as the default type for all other columns. This affects both how defile-ascii
+parses the input value (see the
+.B DISCUSSION
+above), and also determines the type of the data in the output dirfile. The
+.I TYPE
+should be one of the following GetData type codes:
+.RS
+.IP
+.BR NULL ,
+.BR UINT8 ,
+.BR INT8 ,
+.BR UINT16 ,
+.BR INT16 ,
+.BR UINT32 ,
+.BR INT32 ,
+.BR UINT64 ,
+.BR INT64 ,
+.BR FLOAT32 ,
+.BR FLOAT64 ,
+.BR COMPLEX64 ,
+.BR COMPLEX128
+.RE
+.IP
+The type
+.B NULL
+indicates a column should be ignored. The remainder are hopefully fairly
+self-explanatory; see dirfile-format(5) for their meanings. Columns which
+aren't assigned a type via this option default to
+.BR FLOAT64 .
+(Directive:
+.BR Type )
+.TP
+.BR \-W ", " \-\-whitespace
+use tab and space as a separator set. Turns off fixed-width mode if it had been
+previously enabled. (Directive:
+.BR WhiteSpace )
+.TP
+.BR \-w ", " \-\-fixed\-width=\fIWIDTHS
+turn on fixed-width mode with column widths given by
+.IR WIDTHS ,
+which should be a comma-separated list of column widths. If there are more
+columns in the file than widths specified, the last width specified is repeated
+for all other columns. If all columns have the same width,
+.I WIDTHS
+may be simply that number. A column width of zero indicates a variable-width
+column extending to the end of the row. Any other column widths specified
+after a zero are ignored. Fixed-width mode is turned off if a later option
+specifies a field separator
+.RB ( \-\-sep
+or similar).
+
+.SH NOTES
+
+The external filter feature (see
+.B \-\-filter
+above) can be quite powerful. With a suitably crafted external filter,
+defile-ascii can be coerced into providing data to defile from
+.I any
+data source, whatever its type, as long as its fields adhere to the one
+sample-per-frame limitation of defile-ascii. The input designator passed to
+defile,
+.IR INPUT ,
+must be a readable file, since it is opened, read, and piped into the filter's
+standard input, but the filter is allowed to simply discard this.
+
+For example, the defile-ascii plugin can be used to read a dirfile (but see
+defile-dirfile(1) for a better alternative), using the dirfile2ascii(1) utility
+provided with GetData, by doing something like:
+.IP
+.nh
+.ad l
+defile \-\-output=/output/dirfile \-\-type=ascii /dev/null \-\-
+\-\-filter="dirfile2ascii /input/dirfile field1 field2 field3 \-d,"
+.ad n
+.hy
+.PP
+Here, dirfile2ascii(1) ignores its standard input, producing data based on the
+arguments passed to it. A file is still needed by the defile-ascii plugin to
+read, so /dev/null has been used for that purpose. Any other readable file
+would do. In this example, the dirfile created by defile will have the default
+.BI col #
+names. It is trivial to replace the direct dirfile2ascii(1) call with a script
+with first outputs a name row to defile-ascii to name the fields.
+
+.SH AUTHOR
+Defile was written by D. V. Wiebe.
+
+.SH REPORTING BUGS
+Please send reports of bugs to get...@li....
+
+The Defile home page is: <http://getdata.sourceforge.net/defile/>
+
+.SH COPYRIGHT
+Copyright \(co 2012, 2013 D. V. Wiebe
+.PP
+Defile is free software; you may change and/or redistribute it under the terms
+of the GNU General Public License, either version 2 of the license, or (at your
+option) any later version. There is no warranty, to the extent permitted by
+law.
+
+.SH SEE ALSO
+.BR defile (1)
+.BR dirfile2ascii (1)
+.BR strtod (3),
+.BR strtol (3),
+.BR strtoul (3),
+defile-format(5)
+
Added: trunk/defile/doc/defile-dirfile.1
===================================================================
--- trunk/defile/doc/defile-dirfile.1 (rev 0)
+++ trunk/defile/doc/defile-dirfile.1 2013-03-07 04:31:35 UTC (rev 800)
@@ -0,0 +1,207 @@
+.\" defile-dirfile.1 The dirfile input plugin man page for defile.
+.\"
+.\" Copyright (C) 2013 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH defile-dirfile 1 "5 March 2013" "Version 0.0.1" DEFILE
+.SH NAME
+defile-dirfile \(em a dirfile input plugin for Defile
+.SH SYNOPSIS
+.nh
+.ad l
+\fBdefile\fR [ \fIDEFILE\-OPTION\fR ]... [ -t \fIdirfile\fR ]
+[ [ -i ] \fIINPUT\fR ] [ \fB\-\-\fR [ \fIINPUT\-OPTION\fR ]... ]
+.hy
+.ad n
+.SH DESCRIPTION
+The defile-dirfile plugin provides support in defile(1) for
+.I reading
+dirfiles, allowing dirfile to dirfile conversion.
+
+For defile-dirfile, the input designator,
+.IR INPUT ,
+passed to defile should be the pathname to the input dirfile.
+
+In addition to being simply an expensive way of recursively copying a dirfile
+directory tree from one place to another, this plugin may be useful in other
+situations. In the default configuration, runing defile to copy a dirfile
+will also copy into the output dirfile directory all fragments and
+.B LINTERP
+tables outside the directory tree, creating a fully self-contained dirfile (see
+.B \-\-copy\-extern\-frags
+and
+.B \-\-copy\-extern\-luts
+below).
+
+Defile can be used to extract a dirfile subset using the Defile output field
+list (see defile(1)):
+.IP
+.nh
+.ad l
+defile \-\-include=FOO \-\-include=BAR /path/to/dirfile
+.hy
+.ad n
+
+.SH INPUT OPTIONS
+
+All these options must be placed after the double-dash in the defile command
+line. When used as directives in defile run control files, they must be placed
+in the
+.IP
+[dirfile]
+.PP
+section. Mandatory arguments to long options are mandatory for short options
+too.
+
+.TP
+.BR \-E ", " \-\-expand\-all
+expand all derived fields (ie. turn them into
+.B RAW
+fields in the output dirfile). This causes any field list built up with
+.B \-\-expand
+and
+.B \-\-no\-expand
+to be forgotten. (Directive:
+.BR ExpandAll )
+.TP
+.BR +E ", " \-\-expand\-none
+don't expand any derived fields: keep them as derived fields in the output
+dirfile. This causes any field list built up with
+.B \-\-expand
+and
+.B \-\-no\-expand
+to be forgotten. (Directive:
+.BR ExpandNone )
+.TP
+.BR \-e ", " \-\-expand=\fIFIELD\fR
+compute the derived field called
+.IR FIELD ,
+making it a
+.B RAW
+field in the output dirfile. This option can be specified multiple times to
+build up a list of fields to expand. If
+.I FIELD
+is not in the input dirfile, or is not a derived field, this option is
+ignored. (Directive:
+.BR Expand )
+.TP
+.BR +e ", " \-\-no\-expand=\fIFIELD\fR
+don't convert the derived field called
+.I FIELD
+into a
+.B RAW
+field in the output dirfile. This is the default behaviour, and this option
+does nothing if
+.I FIELD
+hadn't earlier been flagged for expansion (either implicitly via
+.BR \-\-expand\-all ,
+or explicitly with
+.BI \-\-expand= FIELD\fR).
+This directive can be used multiple times, usually after
+.BR \-\-expand\-all ,
+to build up a list of fields to skip expansion. If
+.I FIELD
+is not in the input dirfile, or is not a derived field, this option is
+ignored. (Directive:
+.BR NoExpand )
+.TP
+.BI \-\-ext\-dir= DIR
+Put external fragments copied into the output dirfile (see
+.BR \-\-copy\-extern\-frags )
+in the subdirectory
+.I DIR
+under the output dirfile base directory, instead of the default "<ext>". The
+directory specified must be relative. (Directive:
+.BR ExtDir )
+.TP
+.BR \-f ", " \-\-flatten
+merge all metadata fragments in the input dirfile into a single fragment in the
+output dirfile. External fragments, ie. fragments outside the dirfile tree,
+aren't included in this flattening if
+.BR \-\-link\-extern\-frags
+is specified. (Directive:
+.BR Flatten )
+.TP
+.BR +f ", " \-\-no\-flatten
+keep the same fragment architecture in the output dirfile as the input dirfile.
+This is the default behaviour. (Directive:
+.BR NoFlatten )
+.TP
+.BR \-L ", " \-\-link\-extern\-luts
+when
+.B LINTERP
+entries in the input dirfile refer to look-up tables (LUTs) outside the input
+dirfile's directory tree, the corresponding
+.B LINTERP
+specification in the output dirfile use the same out-of-tree file path.
+(Directive:
+.BR LinkExternLUTs )
+.TP
+.BR +L ", " \-\-copy\-extern\-luts
+when
+.B LINTERP
+entries in the input dirfile refer to look-up tables (LUTs) outside the input
+dirfile's directory tree, they will be copied to the output dirfile directory
+tree, usually in a special subdirectory called "./<lut>" (but see
+.BR \-\-lut\-dir ).
+This ensures that the output dirfile is self-contained and is the default
+behaviour. (Directive:
+.BR CopyExternLUTs )
+.TP
+.BI \-\-lut\-dir= DIR
+Put external
+.B LINTERP
+look-up tables (LUTs) copied into the output dirfile (see
+.BR \-\-copy\-extern\-luts )
+in the subdirectory
+.I DIR
+under the output dirfile base directory, instead of the default "<lut>". The
+directory specified must be relative. (Directive:
+.BR LUTDir )
+.TP
+.BR \-X ", " \-\-link\-extern\-frags
+fragments outside the input dirfile's directory tree which are included in the
+input dirfile will be linked in the output dirfile, instead of copying the
+contents of these external fragments. Notably,
+.B RAW
+fields in these external fragments aren't copied by defile to the output dirfile
+since the external inclusion will draw them into the output dirfile from their
+external location. (Directive:
+.BR LinkExternFrags )
+.TP
+.BR +X ", " \-\-copy\-extern\-frags
+fragments outside the input dirfile's directory tree which are included in the
+input dirfile are copied into the output dirfile tree from their external
+location. By default, they will be placed under the subdirectory "./<ext>"
+in the output dirfile tree (but see
+.BR \-\-ext\-dir ).
+This ensures that the output dirfile is self-contained and is the default
+behaviour. (Directive:
+.BR CopyExternFrags )
+
+.SH AUTHOR
+Defile was written by D. V. Wiebe.
+
+.SH REPORTING BUGS
+Please send reports of bugs to get...@li....
+
+The Defile home page is: <http://getdata.sourceforge.net/defile/>
+
+.SH COPYRIGHT
+Copyright \(co 2012, 2013 D. V. Wiebe
+.PP
+Defile is free software; you may change and/or redistribute it under the terms
+of the GNU General Public License, either version 2 of the license, or (at your
+option) any later version. There is no warranty, to the extent permitted by
+law.
+
+.SH SEE ALSO
+.BR defile (1)
Modified: trunk/defile/doc/defile-input.7
===================================================================
--- trunk/defile/doc/defile-input.7 2013-03-05 04:34:19 UTC (rev 799)
+++ trunk/defile/doc/defile-input.7 2013-03-07 04:31:35 UTC (rev 800)
@@ -13,12 +13,14 @@
.\" Texts. A copy of the license is included in the `COPYING.DOC' file
.\" as part of this distribution.
.\"
-.TH defile-input 7 "24 January 2013" "Version 0.0.0" DEFILE
+.TH defile-input 7 "24 January 2013" "Version 0.0.1" DEFILE
.SH NAME
defile-input \(em overview of the defile input plugin API
.SH DESCRIPTION
-This manual page describes how to write input plugins for defile(1), a program
-which can convert data sources in a variety of formats into dirfile(5)
+This manual page describes how to write input plugins for
+.BR defile (1),
+a program which can convert data sources in a variety of formats into
+.BR dirfile (5)
databases.
Defile consists of three parts: a front\-end, which handles I/O with the
@@ -28,9 +30,12 @@
input data source and provides data to the back\-end.
Input plugins are loaded at runtime, based on options given to defile at the
-time of invocation. See defile(1) for information on invoking defile. Because
-the input plugin is loaded dynamically, input types available to defile(1) can
-be extended easily, without having to modify defile itself.
+time of invocation. See
+.BR defile (1)
+for information on invoking defile. Because the input plugin is loaded
+dynamically, input types available to
+.BR defile (1)
+can be extended easily, without having to modify defile itself.
Defile provides a C API for input plugins. Functions provided by the API are
described in individual manual pages in section 3 of the manual. A list of
@@ -74,7 +79,9 @@
.PP
The type name is also the value of the
.B \-\-type
-option given to defile(1) to indicate which input plugin to use.
+option given to
+.BR defile (1)
+to indicate which input plugin to use.
.SH EXPORTED SYMBOLS
@@ -130,8 +137,9 @@
.TP
.I description
is a short string describing the data type read by the plugin (not describing
-the plugin itself). This is used when lists of input plugins are printed
-by defile(1) (ie., if the
+the plugin itself). This is used when lists of input plugins are printed by
+.BR defile (1)
+(ie., if the
.B \-\-list\-types
option was given, or when automatic input type detection returns more than one
candidate input type).
@@ -305,7 +313,8 @@
.BR \-\-conf ,
and the directive
.B Input
-which are handled specially, as they are in the defile options. See defile(1)
+which are handled specially, as they are in the defile options. See
+.BR defile (1)
for their use. Even if an input plugin provides no input options (by setting
.I opts
to zero, these are always accepted as input options by defile. Briefly,
@@ -319,7 +328,7 @@
and
.B Include
specify additional run control files to read for more configuration (see
-defile(1)).
+.BR defile (1)).
.SH HELP AND VERSION OUTPUT
@@ -346,7 +355,7 @@
.PP
Unless the plugin is simple enough to be summarised in the usage text, plugin
developers are encouraged to provide accompanying documentation, if possible a
-Unix manual page called defile-<\fItype\fR>(1). Any such other documentation
+Unix manual page called \fBdefile-<\fItype\fR>(1). Any such other documentation
should be mentioned in the usage preamble or postamble.
.PP
When
@@ -607,8 +616,10 @@
.SH DEFILE DATA MODEL
The defile data model is fundamentally rooted in limitations of the Dirfile
-Standards (see dirfile(5)). The primarly limitation thi...
[truncated message content] |
|
From: <ket...@us...> - 2013-03-09 03:52:22
|
Revision: 808
http://sourceforge.net/p/getdata/code/808
Author: ketiltrout
Date: 2013-03-09 03:52:18 +0000 (Sat, 09 Mar 2013)
Log Message:
-----------
strto[u]q
Modified Paths:
--------------
trunk/defile/configure.ac
trunk/defile/lib/internal.h
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2013-03-09 01:55:11 UTC (rev 807)
+++ trunk/defile/configure.ac 2013-03-09 03:52:18 UTC (rev 808)
@@ -26,7 +26,8 @@
m4_define(defile_version,
defile_major.defile_minor.defile_revision[]defile_extra)
-AC_INIT([Defile],[defile_version],[get...@li...])
+AC_INIT([Defile],[defile_version],[get...@li...],
+ [http://getdata.sourceforge.net/dirfile/])
AC_PREREQ([2.63])
AC_CONFIG_MACRO_DIR([m4])
@@ -174,7 +175,8 @@
echo
echo "*** Looking for additional library functions"
echo
-AC_CHECK_FUNCS([canonicalize_file_name realpath strtoll strtoull wordexp])
+AC_CHECK_FUNCS([canonicalize_file_name realpath strtoll strtoq strtoull \
+ strtouq wordexp])
echo
echo "*** Looking for external programs"
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2013-03-09 01:55:11 UTC (rev 807)
+++ trunk/defile/lib/internal.h 2013-03-09 03:52:18 UTC (rev 808)
@@ -133,12 +133,20 @@
/* long long integer conversion */
#ifndef HAVE_STRTOULL
+#ifdef HAVE_STRTOUQ
+#define strtoull strtouq
+#else
#define strtoull strtoul
#endif
+#endif
#ifndef HAVE_STRTOLL
+#ifdef HAVE_STRTOQ
+#define strtoll strtoq
+#else
#define strtoll strtol
#endif
+#endif
/* state macros */
#define DF_SETSTATE(w,n,s,t) \
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-09 04:57:23
|
Revision: 809
http://sourceforge.net/p/getdata/code/809
Author: ketiltrout
Date: 2013-03-09 04:57:19 +0000 (Sat, 09 Mar 2013)
Log Message:
-----------
Distinguish serve frame size from outpu frame size -- in theory we could use the same variable for both, but this might be a little clearer.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-03-09 03:52:18 UTC (rev 808)
+++ trunk/defile/bin/defile.c 2013-03-09 04:57:19 UTC (rev 809)
@@ -1528,12 +1528,12 @@
} else {
const double delta = (now.tv_sec - then.tv_sec) +
(now.tv_usec - then.tv_usec) / 1e6;
- if (nf * df->wframesize > delta * df->srv_rate * 1e6 / 8.)
- nf = delta * df->srv_rate * 1e6 / 8. / df->wframesize;
+ if (nf * df->sframesize > delta * df->srv_rate * 1e6 / 8.)
+ nf = delta * df->srv_rate * 1e6 / 8. / df->sframesize;
}
- if (nf * df->wframesize > DF_MAX_PACKET)
- nf = (df->wframesize > DF_MAX_PACKET) ? 1 : DF_MAX_PACKET / df->wframesize;
+ if (nf * df->sframesize > DF_MAX_PACKET)
+ nf = (df->sframesize > DF_MAX_PACKET) ? 1 : DF_MAX_PACKET / df->sframesize;
if (nf > 0)
memcpy(&then, &now, sizeof(then));
@@ -1737,12 +1737,12 @@
if (nf_out + df->srv_base > df->buflen)
n = df->buflen - df->srv_base;
- df_rpc_push_mem(&data, df->wframesize * n,
- df->srv_buffer + df->srv_base * df->wframesize);
+ df_rpc_push_mem(&data, df->sframesize * n,
+ df->srv_buffer + df->srv_base * df->sframesize);
/* handle wrap-around */
if (n < nf_out)
- df_rpc_push_mem(&data, df->wframesize * (nf_out - n),
+ df_rpc_push_mem(&data, df->sframesize * (nf_out - n),
df->srv_buffer);
df_rpc_fini(&data, debug);
@@ -1757,12 +1757,12 @@
} else if (wrote < data.len) {
/* this is going to result in the client dropping some data */
cli_data[i].partial = -(data.len - wrote);
- df_dprintf(DF_PRN_DEBUG_O,
+ df_dprintf(DF_PRN_WARN,
"sent partial data packet (%lu of %lu) to %s\n", wrote,
data.len, cli_data[i].name);
} else if (debug)
df_dprintf(DF_PRN_DEBUG_O, "sent data packet of length %lu "
- "(%llu frames) to %s\n", data.len, data.len / df->wframesize,
+ "(%llu frames) to %s\n", data.len, data.len / df->sframesize,
cli_data[i].name);
}
}
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2013-03-09 03:52:18 UTC (rev 808)
+++ trunk/defile/lib/internal.h 2013-03-09 04:57:19 UTC (rev 809)
@@ -300,7 +300,7 @@
int nfd;
struct df_infdef *fd;
- long long rframesize, wframesize;
+ long long rframesize, wframesize, sframesize;
int ref_ind;
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-03-09 03:52:18 UTC (rev 808)
+++ trunk/defile/lib/libdefile.c 2013-03-09 04:57:19 UTC (rev 809)
@@ -820,6 +820,7 @@
}
df->fd = (struct df_infdef*)ptr;
df->fd[df->nfd].framesize = fd->framesize;
+ df->sframesize += fd->framesize;
df->fd[df->nfd].foff = 0;
df->fd[df->nfd].index = 0;
df->fd[df->nfd].boff = 0;
@@ -1136,8 +1137,10 @@
if (df_rpc_fini(df->srv_slug, df_mode() & DF_MODE_DEBUG_I))
return DF_OUTPUT;
/* make the frame buffer */
+ df_lprintf(DF_PRN_DEBUG_O, "server frame size: %lli\n", df->sframesize);
+
df->srv_base = 0;
- df->srv_buffer = (char*)malloc(df->buflen * df->wframesize);
+ df->srv_buffer = (char*)malloc(df->buflen * df->sframesize);
if (df->srv_buffer == NULL)
return DF_SYSTEM;
pthread_mutex_init(&df->srv_mx, NULL);
@@ -1329,7 +1332,7 @@
if (n > nf)
n = nf;
- memcpy(df->srv_buffer + df->wframesize * r + fd->index * fd->framesize +
+ memcpy(df->srv_buffer + df->sframesize * r + fd->index * fd->framesize +
fd->foff, b, fd->framesize * n);
/* update the indices */
@@ -1351,7 +1354,7 @@
while (nf >= fd->count) {
while (r >= df->buflen)
r -= df->buflen;
- memcpy(df->srv_buffer + df->wframesize * r + fd->foff, b,
+ memcpy(df->srv_buffer + df->sframesize * r + fd->foff, b,
fd->framesize * fd->count);
nf -= fd->count;
fd->boff++;
@@ -1361,7 +1364,7 @@
/* now a trailing partial */
if (nf > 0) {
- memcpy(df->srv_buffer + df->wframesize * r + fd->foff, b,
+ memcpy(df->srv_buffer + df->sframesize * r + fd->foff, b,
fd->framesize * nf);
fd->index = nf;
}
@@ -1478,7 +1481,7 @@
df->wpartial = 0;
df->nread = 0;
df->nframes = (uint64_t)length;
- df->wframesize = df->rframesize = 0;
+ df->wframesize = df->rframesize = df->sframesize = 0;
pthread_mutex_unlock(&df_rate_mx);
/* good to go -- do the server initialisation, if necessary */
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-09 05:09:03
|
Revision: 811
http://sourceforge.net/p/getdata/code/811
Author: ketiltrout
Date: 2013-03-09 05:08:59 +0000 (Sat, 09 Mar 2013)
Log Message:
-----------
Defer gd_add_raw if there aren't any samples yet; also don't require an INPUT in client mode.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/doc/defile-ascii.1
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-03-09 05:07:16 UTC (rev 810)
+++ trunk/defile/bin/defile.c 2013-03-09 05:08:59 UTC (rev 811)
@@ -2621,7 +2621,9 @@
df_parse(df->iargc, df->iargv, &rcd, &df->config);
}
- if (df->input == NULL) {
+ /* Don't need an INPUT if we're connecting to a server */
+ if (df->input == NULL && (df->host == NULL || strchr(df->host, ':') == NULL))
+ {
df_dputs(DF_PRN_ERR, "no input specified.\n");
exit(1);
}
Modified: trunk/defile/doc/defile-ascii.1
===================================================================
--- trunk/defile/doc/defile-ascii.1 2013-03-09 05:07:16 UTC (rev 810)
+++ trunk/defile/doc/defile-ascii.1 2013-03-09 05:08:59 UTC (rev 811)
@@ -22,8 +22,9 @@
.hy
.ad n
.SH DESCRIPTION
-The defile-ascii plugin provides support in defile(1) for converting ASCII
-tabular data into dirfiles.
+The defile-ascii plugin provides support in
+.BR defile (1)
+for converting ASCII tabular data into dirfiles.
For defile-ascii, the input designator,
.IR INPUT ,
@@ -445,8 +446,10 @@
standard input, but the filter is allowed to simply discard this.
For example, the defile-ascii plugin can be used to read a dirfile (but see
-defile-dirfile(1) for a better alternative), using the dirfile2ascii(1) utility
-provided with GetData, by doing something like:
+.BR defile-dirfile (1),
+provided with defile, for a better alternative), using the
+.BR dirfile2ascii (1)
+utility provided with GetData, by doing something like:
.IP
.nh
.ad l
@@ -455,13 +458,17 @@
.ad n
.hy
.PP
-Here, dirfile2ascii(1) ignores its standard input, producing data based on the
-arguments passed to it. A file is still needed by the defile-ascii plugin to
-read, so /dev/null has been used for that purpose. Any other readable file
-would do. In this example, the dirfile created by defile will have the default
+Here,
+.BR dirfile2ascii (1)
+ignores its standard input, producing data based on the arguments passed to it.
+A file is still needed by the defile-ascii plugin to read, so /dev/null has been
+used for that purpose. Any other readable file would do. In this example, the
+dirfile created by defile will have the default
.BI col #
-names. It is trivial to replace the direct dirfile2ascii(1) call with a script
-with first outputs a name row to defile-ascii to name the fields.
+names. It is trivial to replace the direct
+.BR dirfile2ascii (1)
+call with a script with first outputs a name row to defile-ascii to name the
+fields.
.SH AUTHOR
Defile was written by D. V. Wiebe.
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2013-03-09 05:07:16 UTC (rev 810)
+++ trunk/defile/lib/internal.h 2013-03-09 05:08:59 UTC (rev 811)
@@ -201,6 +201,7 @@
this buffer and the reference -- only used in
synchronous mode */
pthread_mutex_t mx; /* buffer mutex */
+ int added; /* have we added it to the output dirfile yet? */
};
struct df_rpc {
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-03-09 05:07:16 UTC (rev 810)
+++ trunk/defile/lib/libdefile.c 2013-03-09 05:08:59 UTC (rev 811)
@@ -642,9 +642,11 @@
* field name; use GetData's parser to do this. */
name = gd_strtok(df->D, spec);
if (name == NULL) {
- if (gd_error(df->D) == GD_E_ALLOC)
+ if (gd_error(df->D) == GD_E_ALLOC) {
+ df_lprintf(DF_PRN_ERR, "Failed to parse spec: %s\n", spec);
return DF_OUTPUT; /* better to just fold this into gd_add_spec failures
than return DF_SYSTEM */
+ }
return DF_INPUT; /* unparseable input */
}
@@ -652,7 +654,9 @@
if (df_field_in_output(name)) {
/* check the second token to see if it's a RAW field */
type = gd_strtok(df->D, NULL);
- if (type && strcmp(type, "RAW") == 0) {
+ if (gd_error(df->D))
+ df_lprintf(DF_PRN_ERR, "Failed to parse spec: %s\n", spec);
+ else if (type && strcmp(type, "RAW") == 0) {
df_lputs(DF_PRN_INERR,
"attempt to define RAW field via df_add_spec().\n");
ret = DF_INPUT;
@@ -707,7 +711,7 @@
int fragment, int set_buf)
{
const int append = df_mode() & DF_MODE_APPEND;
- int r, is_new = 0;
+ int r, is_new = 0, added = 0;
void *ptr;
/* check the OFL */
@@ -726,12 +730,6 @@
return DF_SYSTEM;
}
- if (!(append || df->mode_flags & DF_MODE_SERVER)) {
- /* try adding it to the dirfile */
- if (gd_add_raw(df->D, f->name, f->type, f->spf * count, fragment))
- return DF_OUTPUT;
- }
-
/* add a new record */
ptr = realloc(df->raw, sizeof(struct df_raw) * (df->nraw + 1));
if (ptr == NULL) {
@@ -750,6 +748,7 @@
/* set up the new record */
df->raw[r].ahead = 0;
+ df->raw[r].added = 0;
df->raw[r].type = f->type;
df->raw[r].name = name;
df->raw[r].spf = f->spf * count;
@@ -759,9 +758,23 @@
} else {
/* increase the samples per frame of this raw */
df->raw[r].spf += f->spf * count;
- if (!(append || df->mode_flags & DF_MODE_SERVER)) {
- if (gd_alter_raw(df->D, f->name, f->type, df->raw[r].spf, 0))
+ }
+
+ /* add/modify the field in the output dirfile, if necessary */
+ if (!(append || df->mode_flags & DF_MODE_SERVER)) {
+ if (df->raw[r].spf > 0) {
+ if (!df->raw[r].added) {
+ if (gd_add_raw(df->D, df->raw[r].name, df->raw[r].type, df->raw[r].spf,
+ fragment))
+ {
+ return DF_OUTPUT;
+ } else
+ df->raw[r].added = 1;
+ } else if (gd_alter_raw(df->D, df->raw[r].name, df->raw[r].type,
+ df->raw[r].spf, 0))
+ {
return DF_OUTPUT;
+ }
}
}
@@ -1040,9 +1053,11 @@
if (hello && df_rpc_push(df->srv_slug, DF_RPC_HELLO))
return DF_OUTPUT;
- if (df_rpc_push(df->srv_slug, DF_RPC_INIT, length, rate, name ? name
- : df->input))
+ if (df_rpc_push(df->srv_slug, DF_RPC_INIT, length, rate, name ? name :
+ df->input))
+ {
return DF_OUTPUT;
+ }
}
return DF_SUCCESS;
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-10 19:40:26
|
Revision: 812
http://sourceforge.net/p/getdata/code/812
Author: ketiltrout
Date: 2013-03-10 19:40:23 +0000 (Sun, 10 Mar 2013)
Log Message:
-----------
Curfiles/symlinks. Also some changes to the df_optdef.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/bin/rc.c
trunk/defile/doc/defile-input.7
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/defile.h
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-03-09 05:08:59 UTC (rev 811)
+++ trunk/defile/bin/defile.c 2013-03-10 19:40:23 UTC (rev 812)
@@ -32,6 +32,7 @@
#include <syslog.h>
#include <stdarg.h>
#include <arpa/inet.h>
+#include <libgen.h>
#include <poll.h>
#include <netinet/in.h>
#include <netinet/tcp.h>
@@ -940,7 +941,8 @@
DF_WOPT_OUTPUT, DF_WOPT_QUIET, DF_WOPT_SKIP, DF_WOPT_SYNC, DF_WOPT_TYPE,
DF_WOPT_VERBOSE, DF_WOPT_REMOTE, DF_WOPT_LOCAL, DF_WOPT_RSH, DF_WOPT_LISTEN,
DF_WOPT_NOLISTEN, DF_WOPT_RCMD, DF_WOPT_RDEBUG, DF_WOPT_NORDEBUG,
- DF_WOPT_SRVRATE, DF_WOPT_DAEMON
+ DF_WOPT_SRVRATE, DF_WOPT_DAEMON, DF_WOPT_CURFILE, DF_WOPT_SYMLINK,
+ DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE
};
static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
@@ -1092,6 +1094,12 @@
SETFLOAT(SRVRATE, srv_rate);
SETSTRING(RCMD, remcmd);
+
+ SETSTRING(CURFILE, curfile);
+ CLRSTRING(NOCURFILE, curfile);
+
+ SETSTRING(SYMLINK, symlink);
+ CLRSTRING(NOSYMLINK, symlink);
}
return -1;
@@ -1402,6 +1410,170 @@
usleep(10000);
}
+/* update curfile and/or symlink */
+static int df_curfile(void)
+{
+ int fd;
+ char *ptr1, *ptr2, *ptr3;
+ char *dirfile;
+ size_t len;
+
+ /* nothing to do */
+ if (df->curfile == NULL && df->symlink == NULL)
+ return 0;
+
+ /* create an absolute pathname to the output dirfile */
+#ifdef HAVE_CANONICALIZE_FILE_NAME
+ dirfile = canonicalize_file_name(df->dirfile);
+ if (dirfile == NULL) {
+ df_dperror(DF_PRN_ERR, "canonicalize_file_name");
+ return 1;
+ }
+#elif defined HAVE_REALPATH && defined PATH_MAX
+ dirfile = malloc(PATH_MAX);
+ if (dirfile == NULL) {
+ DF_OOM(DF_PRN_DF);
+ return 1;
+ }
+ if (realpath(df->dirfile, dirfile) == NULL) {
+ df_dperror(DF_PRN_ERR, "realpath");
+ free(dirfile);
+ return 1;
+ }
+#else
+ if (df->dirfile[0] == '/')
+ dirfile = strdup(df->dirfile);
+ else {
+ ptr1 = malloc(4096);
+ if (ptr1 == NULL) {
+ DF_OOM(DF_PRN_DF);
+ return 1;
+ }
+ if (getcwd(ptr1, 4096) == NULL) {
+ df_dperror(DF_PRN_ERR, "getcwd");
+ free(ptr1);
+ return 1;
+ }
+ dirfile = malloc(strlen(ptr1) + strlen(df->dirfile) + 2);
+ if (dirfile == NULL) {
+ df_dperror(DF_PRN_ERR, "getcwd");
+ free(ptr1);
+ return 1;
+ }
+ sprintf(dirfile, "%s/%s", ptr1, df->dirfile);
+ free(ptr1);
+ }
+#endif
+ len = strlen(dirfile);
+
+ /* update the curfile (out-of-place) */
+ if (df->curfile) {
+ /* temporary file */
+ ptr1 = strdup(df->curfile);
+ if (ptr1 == NULL) {
+ DF_OOM(DF_PRN_DF);
+ free(dirfile);
+ return 1;
+ }
+ ptr2 = dirname(ptr1);
+ ptr3 = malloc(strlen(ptr2) + 6);
+ if (ptr3 == NULL) {
+ DF_OOM(DF_PRN_DF);
+ free(dirfile);
+ free(ptr1);
+ return 1;
+ }
+ strcat(strcpy(ptr3, ptr2), "XXXXXX");
+ free(ptr1);
+
+ /* create temporary file */
+ fd = mkstemp(ptr3);
+ if (fd < 0) {
+ df_dperror(DF_PRN_ERR, "mkstemp");
+ free(dirfile);
+ free(ptr3);
+ return 1;
+ }
+
+ if (write(fd, dirfile, len) < 0) {
+ df_dperror(DF_PRN_ERR, "write");
+ free(dirfile);
+ free(ptr3);
+ return 1;
+ }
+ if (close(fd)) {
+ df_dperror(DF_PRN_ERR, "close");
+ free(dirfile);
+ free(ptr3);
+ return 1;
+ }
+
+ /* move into place */
+ if (rename(ptr3, df->curfile)) {
+ df_dperror(DF_PRN_ERR, "rename");
+ free(dirfile);
+ /* unlnk on error */
+ if (unlink(ptr3))
+ df_dperror(DF_PRN_ERR, "unlink");
+ free(ptr3);
+ return 1;
+ }
+ free(ptr3);
+ }
+
+ /* make a symlink (out of place) */
+ if (df->symlink) {
+ /* temporary symlink */
+ ptr1 = strdup(df->curfile);
+ if (ptr1 == NULL) {
+ DF_OOM(DF_PRN_DF);
+ free(dirfile);
+ return 1;
+ }
+ ptr2 = dirname(ptr1);
+ ptr3 = malloc(strlen(ptr2) + 6);
+ if (ptr3 == NULL) {
+ DF_OOM(DF_PRN_DF);
+ free(dirfile);
+ free(ptr1);
+ return 1;
+ }
+ strcat(strcpy(ptr3, ptr2), "XXXXXX");
+ free(ptr1);
+
+ /* create temporary symlink */
+ mktemp(ptr3);
+ if (ptr3[0] == 0) {
+ df_dperror(DF_PRN_ERR, "mktemp");
+ free(dirfile);
+ free(ptr3);
+ return 1;
+ }
+
+ if (symlink(dirfile, ptr3)) {
+ df_dperror(DF_PRN_ERR, "symlink");
+ free(dirfile);
+ free(ptr3);
+ return 1;
+ }
+
+ /* move into place */
+ if (rename(ptr3, df->symlink)) {
+ df_dperror(DF_PRN_ERR, "rename");
+ free(dirfile);
+ /* unlnk on error */
+ if (unlink(ptr3))
+ df_dperror(DF_PRN_ERR, "unlink");
+ free(ptr3);
+ return 1;
+ }
+ free(ptr3);
+ }
+
+ free(dirfile);
+ return 0;
+}
+
/* create the output dirfile and then wait while the Input defines the
* metadata */
static int DF_MakeOutput(void)
@@ -1436,6 +1608,10 @@
if (gd_metaflush(df->D))
return 1;
+ /* update curfile/symlink */
+ if (df_curfile())
+ return 1;
+
/* signal we're ready for data */
DF_OUTSTATE_MX(RUN);
@@ -1514,7 +1690,7 @@
/* no data */
if (nf == 0 || have_fd == 0)
return df->buflen + 1;
-
+
/* stdout mode has no rate limiting */
if (!listen)
return nf;
@@ -1856,7 +2032,7 @@
client_done[i] = 2; /* error */
} else {
df_dprintf(DF_PRN_DEBUG_O, "sent REINIT(%i) to %s on %i\n",
- reinit[5], cli_data[i].name, pfd[i].fd);
+ reinit[5], cli_data[i].name, pfd[i].fd);
client_done[i] = 1; /* success */
}
pfd[i].events = 0;
@@ -2292,137 +2468,150 @@
struct sigaction action;
const struct df_optdef options[] = {
- { DF_WOPT_APPEND, DF_OPT_NO_ARG, 'A', "append", "Append", 0, NULL,
+ { DF_WOPT_APPEND, DF_OPT_NO_ARG, 'A', "append", "Append", NULL,
"append data to an already existing output"
},
- { DF_WOPT_NOAPPEND, DF_OPT_NO_ARG, 'A', "no-append", "NoAppend", 0, NULL,
+ { DF_WOPT_NOAPPEND, DF_OPT_NO_ARG, 'A', "no-append", "NoAppend", NULL,
"don't append data (default)"
},
- { DF_WOPT_ASYNC, DF_OPT_NO_ARG, 'a', "async", "Asynchronous", 0,
+ { DF_WOPT_ASYNC, DF_OPT_NO_ARG, 'a', "async", "Asynchronous",
NULL, "turn on asynchronous mode"
},
- { DF_WOPT_SYNC, DF_OPT_NO_ARG, 'a', "sync", "Synchronous", DF_OPT_PLUS,
+ { DF_WOPT_SYNC, DF_OPT_PLUS | DF_OPT_NO_ARG, 'a', "sync", "Synchronous",
NULL, "turn off synchronous mode"
},
- { DF_WOPT_CLOBBER, DF_OPT_NO_ARG, 0, "clobber", "Clobber", 0, NULL,
+ { DF_WOPT_CLOBBER, DF_OPT_NO_ARG, 0, "clobber", "Clobber", NULL,
"if the output dirfile already exists, overwrite it; use with caution"
},
- { DF_WOPT_NOCLOBBER, DF_OPT_NO_ARG, 0, "no-clobber", "NoClobber", 0, NULL,
+ { DF_WOPT_NOCLOBBER, DF_OPT_NO_ARG, 0, "no-clobber", "NoClobber", NULL,
"don't overwrite an existing dirfile (default)"
},
- { DF_WOPT_CUTDIR, DF_OPT_ARG_RQ, 0, "cut-dirs", "CutDirs", 0, "NUM",
+ { DF_WOPT_CUTDIR, DF_OPT_ARG_RQ, 0, "cut-dirs", "CutDirs", "NUM",
"remove path elements from the output name"
},
- { DF_WOPT_DIR, DF_OPT_ARG_RQ, 'D', "directory", "Directory", DF_OPT_EXPAND,
+ { DF_WOPT_DIR, DF_OPT_EXPAND | DF_OPT_ARG_RQ, 'D', "directory", "Directory",
"DIR", "write the output dirfile as a subdirectory under DIR"
},
- { DF_WOPT_DEBUG, DF_OPT_ARG_OP, 'd', "debug", "Debug", 0, "WHAT",
+ { DF_WOPT_DEBUG, DF_OPT_ARG_OP, 'd', "debug", "Debug", "WHAT",
"print debugging messages, restricted according to WHAT (default: "
"everything)"
},
- { DF_WOPT_NODEBUG, DF_OPT_NO_ARG, 'd', "no-debug", "NoDebug", DF_OPT_PLUS,
+ { DF_WOPT_NODEBUG, DF_OPT_PLUS | DF_OPT_NO_ARG, 'd', "no-debug", "NoDebug",
NULL, "suppress printing debugging messages (default)"
},
- { DF_WOPT_FOLLOW, DF_OPT_NO_ARG, 'f', "follow", "Follow", 0, NULL,
+ { DF_WOPT_FOLLOW, DF_OPT_NO_ARG, 'f', "follow", "Follow", NULL,
"keep monitoring the input and write data to the outpu as it becomes "
"available"
},
- { DF_WOPT_NOFOLLOW, DF_OPT_NO_ARG, 'f', "no-follow", "NoFollow",
- DF_OPT_PLUS, NULL,
+ { DF_WOPT_NOFOLLOW, DF_OPT_PLUS | DF_OPT_NO_ARG, 'f', "no-follow",
+ "NoFollow", NULL,
"exit successfully upon reaching the end of the input (default)"
},
- { DF_WOPT_ADD, DF_OPT_ARG_RQ, 'F', "include", "IncludeField", 0, "FIELD",
+ { DF_WOPT_ADD, DF_OPT_ARG_RQ, 'F', "include", "IncludeField", "FIELD",
"include FIELD in the output dirfile, if available"
},
- { DF_WOPT_DEL, DF_OPT_ARG_RQ, 'F', "exclude", "ExcludeField", DF_OPT_PLUS,
+ { DF_WOPT_DEL, DF_OPT_PLUS | DF_OPT_ARG_RQ, 'F', "exclude", "ExcludeField",
"FIELD", "exclude FIELD from the output dirfile"
},
- { DF_WOPT_BHEAD, DF_OPT_ARG_RQ, 'h', "buf-head", "BufferHead", 0, "N",
+ { DF_WOPT_BHEAD, DF_OPT_ARG_RQ, 'h', "buf-head", "BufferHead", "N",
"set the length of the output buffer head to N frames; the default "
"value is zero"
},
- { DF_WOPT_INPUT, DF_OPT_ARG_RQ, 'i', "input", "Input", 0, "NAME",
+ { DF_WOPT_INPUT, DF_OPT_ARG_RQ, 'i', "input", "Input", "NAME",
"read input data from NAME; typically the NAME can be specified as "
"a bare argument, without the need for -i; this option is provided "
"only for use in situations where NAME begins with - or + (and would, "
"therefore, be mistaken for an option)"
},
- { DF_WOPT_BLEN, DF_OPT_ARG_RQ, 'l', "buf-len", "BufferLen", 0, "N",
+ { DF_WOPT_BLEN, DF_OPT_ARG_RQ, 'l', "buf-len", "BufferLen", "N",
"set the length of the output buffer to N frames; the default value "
"is " DF_STRINGIFY(DF_BUFLEN) ""
},
- { DF_WOPT_LIBDIR, DF_OPT_NO_ARG, 0, "lib-dir", NULL, 0, NULL,
+ { DF_WOPT_LIBDIR, DF_OPT_NO_ARG, 0, "lib-dir", NULL, NULL,
"print the default input plugin path and exit"
},
- { DF_WOPT_LIST, DF_OPT_NO_ARG, 0, "list-types", NULL, 0, NULL,
+ { DF_WOPT_LIST, DF_OPT_NO_ARG, 0, "list-types", NULL, NULL,
"print a list of available input plugins and exit"
},
- { DF_WOPT_OFLCLR, DF_OPT_NO_ARG, 0, "clear-field-list", "ClearFieldList", 0,
+ { DF_WOPT_OFLCLR, DF_OPT_NO_ARG, 0, "clear-field-list", "ClearFieldList",
NULL,
"ignore all previous options which modifiy the ouput field list (-F, +F)"
},
- { DF_WOPT_OUTPUT, DF_OPT_ARG_RQ, 'o', "output", "Output", DF_OPT_EXPAND,
+ { DF_WOPT_OUTPUT, DF_OPT_EXPAND | DF_OPT_ARG_RQ, 'o', "output", "Output",
"PATH", "write data to a dirfile called PATH"
},
- { DF_WOPT_QUIET, DF_OPT_NO_ARG, 'q', "quiet", "Quiet", 0, NULL,
+ { DF_WOPT_QUIET, DF_OPT_NO_ARG, 'q', "quiet", "Quiet", NULL,
"be less verbose"
},
- { DF_WOPT_VERBOSE, DF_OPT_NO_ARG, 'q', "verbose", "Verbose", DF_OPT_PLUS,
+ { DF_WOPT_VERBOSE, DF_OPT_PLUS | DF_OPT_NO_ARG, 'q', "verbose", "Verbose",
NULL, "be normally verbose"
},
- { DF_WOPT_SKIP, DF_OPT_ARG_RQ, 's', "skip", "SkipFrames", 0, "NUM",
+ { DF_WOPT_SKIP, DF_OPT_ARG_RQ, 's', "skip", "SkipFrames", "NUM",
"skip the first NUM frames of the input (default: 0)"
},
- { DF_WOPT_TYPE, DF_OPT_ARG_RQ, 't', "type", "InputType", 0, "TYPE",
+ { DF_WOPT_TYPE, DF_OPT_ARG_RQ, 't', "type", "InputType", "TYPE",
"use TYPE as the input data type"
},
- { DF_WOPT_AUTOTYPE, DF_OPT_NO_ARG, 't', "autotype", "AutoType", DF_OPT_PLUS,
- NULL, "attempt to automatically determine the data type of the input"
+ { DF_WOPT_AUTOTYPE, DF_OPT_PLUS | DF_OPT_NO_ARG, 't', "autotype",
+ "AutoType", NULL,
+ "attempt to automatically determine the data type of the input"
},
#if 0
- { DF_WOPT_GZIP, DF_OPT_NO_ARG, 'z', "gzip", "GZip", 0, NULL,
+ { DF_WOPT_GZIP, DF_OPT_NO_ARG, 'z', "gzip", "GZip", NULL,
"gzip compress the output raw data"
},
#endif
- { DF_WOPT_REMOTE, DF_OPT_ARG_RQ, 'R', "remote", "RemoteHost", 0,
+ { DF_WOPT_REMOTE, DF_OPT_ARG_RQ, 'R', "remote", "RemoteHost",
"{ HOST:PORT | [USER@]HOST }", "read from remote host HOST. In the first"
" form, connect to the server listening on port PORT, otherwise start "
"a remote defile (as user USER, if given)"
},
- { DF_WOPT_LOCAL, DF_OPT_NO_ARG, 'R', "local", "Local", DF_OPT_PLUS,
+ { DF_WOPT_LOCAL, DF_OPT_PLUS | DF_OPT_NO_ARG, 'R', "local", "Local",
NULL, "do not read data over the network (default)"
},
- { DF_WOPT_RSH, DF_OPT_ARG_RQ, 0, "rsh", "RemoteShell", 0, "COMMAND",
+ { DF_WOPT_RSH, DF_OPT_ARG_RQ, 0, "rsh", "RemoteShell", "COMMAND",
"use COMMAND to log in to a remote host; (default: ssh)"
},
- { DF_WOPT_RCMD, DF_OPT_ARG_RQ, 0, "remote-command", "RemoteCommand", 0,
+ { DF_WOPT_RCMD, DF_OPT_ARG_RQ, 0, "remote-command", "RemoteCommand",
"PROGRAM", "use PATH as the defile to run on the remote host; (default: "
"defile)"
},
- { DF_WOPT_LISTEN, DF_OPT_ARG_RQ, 'L', "listen", "Listen", 0, "PORT",
+ { DF_WOPT_LISTEN, DF_OPT_ARG_RQ, 'L', "listen", "Listen", "PORT",
"run a defile server, listening on port PORT; in this case, no output "
"dirfile is created on the local system"
},
- { DF_WOPT_NOLISTEN, DF_OPT_NO_ARG, 'L', "no-listen", "NoListen",
- DF_OPT_PLUS, NULL, "don't run a server (default)"
+ { DF_WOPT_NOLISTEN, DF_OPT_PLUS | DF_OPT_NO_ARG, 'L', "no-listen",
+ "NoListen", NULL, "don't run a server (default)"
},
- { DF_WOPT_RDEBUG, DF_OPT_ARG_OP, 0, "remote-debug", "RemoteDebug", 0,
+ { DF_WOPT_RDEBUG, DF_OPT_ARG_OP, 0, "remote-debug", "RemoteDebug",
"WHAT", "print debugging messages from the remote defile restricted to "
"WHAT (or everything)"
},
- { DF_WOPT_NORDEBUG, DF_OPT_NO_ARG, 0, "no-remote-debug", "NoRemoteDebug", 0,
+ { DF_WOPT_NORDEBUG, DF_OPT_NO_ARG, 0, "no-remote-debug", "NoRemoteDebug",
NULL, "don't print debugging messages from the remote defile (default)"
},
{ DF_WOPT_SRVRATE, DF_OPT_ARG_RQ, 0, "server-bwlimit", "ServerBandwidth",
- 0, "RATE", "limit data rate of a server to RATE megabits per second "
+ "RATE", "limit data rate of a server to RATE megabits per second "
"(default: 10)"
},
- { DF_WOPT_DAEMON, DF_OPT_NO_ARG, 0, "daemon", "Daemon", 0, NULL,
+ { DF_WOPT_DAEMON, DF_OPT_NO_ARG, 0, "daemon", "Daemon", NULL,
"when running a server, daemonise on start up (default)"
},
- { DF_WOPT_NODAEMON, DF_OPT_NO_ARG, 0, "no-daemon", "NoDaemon", 0, NULL,
+ { DF_WOPT_NODAEMON, DF_OPT_NO_ARG, 0, "no-daemon", "NoDaemon", NULL,
"when running a server, don't daemonise on start up"
},
+ { DF_WOPT_SYMLINK, DF_OPT_ARG_RQ, 'S', "symlink", "Symlink", "PATH",
+ "maintain a symlink called PATH to the current output dirfile"
+ },
+ { DF_WOPT_NOSYMLINK, DF_OPT_PLUS | DF_OPT_NO_ARG, 'S', "no-symlink",
+ "NoSymlink", NULL, "don't maintain a symlink (default)"
+ },
+ { DF_WOPT_CURFILE, DF_OPT_ARG_RQ, 'C', "curfile", "CurFile", "PATH",
+ "write a curfile called PATH"
+ },
+ { DF_WOPT_NOCURFILE, DF_OPT_PLUS | DF_OPT_NO_ARG, 'C', "no-curfile",
+ "NoCurFile", NULL, "don't write a curfile (default)"
+ },
DF_OPT_ENDOPT
};
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2013-03-09 05:08:59 UTC (rev 811)
+++ trunk/defile/bin/rc.c 2013-03-10 19:40:23 UTC (rev 812)
@@ -66,6 +66,14 @@
"this plugin; if not, write to the Free Software Foundation, Inc.,\n" \
"51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA\n"
+/* option flags we're not yet prepared to put in the public API */
+#define DF_OPT_SREDIR (DF_OPT_SPECIAL | 0x1)
+#define DF_OPT_SH_ONE (DF_OPT_SPECIAL | 0x2)
+#define DF_OPT_SH_TGL (DF_OPT_SPECIAL | 0x3)
+#define DF_OPT_ANT_OP (DF_OPT_SPECIAL | 0x4)
+#define DF_OPT_SHIFT 0x080 /* short option is shifted */
+#define DF_OPT_NOSHIFT 0x100 /* short option ignores shift */
+
static const struct df_optdef df_no_opts[] = {
DF_OPT_ENDOPT
};
@@ -145,10 +153,14 @@
int pos = 7;
char sopt[5] = " ";
+ /* skip specials */
+ if (opt->bits & DF_OPT_SPECIAL)
+ return;
+
/* short option */
if (opt->sopt) {
- sopt[0] = (opt->flags & DF_OPT_PLUS) ? '+' : '-';
- if (opt->flags & DF_OPT_SHIFT) {
+ sopt[0] = (opt->bits & DF_OPT_PLUS) ? '+' : '-';
+ if (opt->bits & DF_OPT_SHIFT) {
sopt[1] = shift_char;
sopt[2] = opt->sopt;
sopt[3] = ',';
@@ -163,10 +175,10 @@
/* long option */
if (opt->lopt) {
pos += df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "--%s", opt->lopt);
- if (opt->type == DF_OPT_ARG_RQ)
+ if ((opt->bits & DF_OPT_MASK) == DF_OPT_ARG_RQ)
pos += df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "=%s",
opt->argname ? opt->argname : "ARG");
- else if (opt->type == DF_OPT_ARG_OP)
+ else if ((opt->bits & DF_OPT_MASK) == DF_OPT_ARG_OP)
pos += df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "[=%s]",
opt->argname ? opt->argname : "ARG");
}
@@ -247,9 +259,9 @@
val = (asopt <= b->lopt[0]) ? -1 : 1;
} else if (asopt == bsopt) {
if (a->sopt == b->sopt) {
- if ((a->flags & DF_OPT_PLUS) && !(b->flags & DF_OPT_PLUS))
+ if ((a->bits & DF_OPT_PLUS) && !(b->bits & DF_OPT_PLUS))
val = 1;
- else if (!(a->flags & DF_OPT_PLUS) && (b->flags & DF_OPT_PLUS))
+ else if (!(a->bits & DF_OPT_PLUS) && (b->bits & DF_OPT_PLUS))
val = -1;
else
val = 0; /* the same short option! that'll prove problematic later */
@@ -307,8 +319,11 @@
"mandatory for short options too.\n\n");
/* Count the options */
- for (nopt = 0; rcd->options[nopt].type != DF_OPT_END_OO; ++nopt)
+ for (nopt = 0; (rcd->options[nopt].bits & DF_OPT_MASK) != DF_OPT_END_OO;
+ ++nopt)
+ {
;
+ }
/* Sort the options */
opt = malloc(sizeof(struct df_optdef*) * nopt);
@@ -385,9 +400,9 @@
char *ptr = message + strlen(message);
int len = 1023 - strlen(message);
if (config->sopt > 0) {
- if (opt->flags & DF_OPT_PLUS)
+ if (opt->bits & DF_OPT_PLUS)
sign = '+';
- if (opt->flags & DF_OPT_SHIFT)
+ if (opt->bits & DF_OPT_SHIFT)
shift[0] = rcd->shift_one;
snprintf(ptr, len, " short option %c%s%c", sign, shift, opt->sopt);
@@ -451,6 +466,7 @@
{
const struct df_optdef *opt = rcd->options + i;
const int fac = (rcd->name == NULL) ? DF_PRN_DF : DF_PRN_INPUT;
+ const unsigned int opt_type = opt ? (opt->bits & DF_OPT_MASK) : 0;
if (i < 0)
opt = NULL;
@@ -458,12 +474,12 @@
logprintf(" Pushing %s for %i(=%i) at %u (%i)\n", val, i,
rcd->options[i].value, rcd->confc, line);
if (opt) {
- if (opt->type == DF_OPT_CONFIG) {
+ if (opt_type == DF_OPT_CONFIG) {
DF_DoRunControl(val, cwd, label, line, rcd);
return;
}
- if (val && opt->flags & DF_OPT_EXPAND) {
+ if (val && opt->bits & DF_OPT_EXPAND) {
char *expanded = df_rc_shell_expand(fac, val);
if (expanded) {
val = expanded;
@@ -484,13 +500,13 @@
}
rcd->config[rcd->confc].error = error;
- if ((opt == NULL || opt->type != DF_OPT_NO_ARG || opt->type == DF_OPT_SREDIR)
+ if ((opt == NULL || opt_type != DF_OPT_NO_ARG || opt_type == DF_OPT_SREDIR)
&& val != NULL)
{
rcd->config[rcd->confc].argument = (no_dup) ? (char *)val : strdup(val);
} else {
rcd->config[rcd->confc].argument = NULL;
- if (opt && opt->type == DF_OPT_ARG_RQ)
+ if (opt && opt_type == DF_OPT_ARG_RQ)
rcd->config[rcd->confc].error = DF_RC_ERR_NO_ARG;
}
@@ -629,7 +645,7 @@
/* match directive */
found = 0;
- for (i = 0; rcd->options[i].type != DF_OPT_END_OO; ++i)
+ for (i = 0; (rcd->options[i].bits & DF_OPT_MASK) != DF_OPT_END_OO; ++i)
if (strcasecmp(opt, rcd->options[i].directive) == 0) {
DF_PushConfig(i, val, DF_RC_ERR_OK, 0, cwd, label, ln, 0, rcd);
found = 1;
@@ -664,18 +680,19 @@
if (shift & SHIFT_NEXT)
shift = SHIFT_THIS | ((shift & SHIFT_ON) ^ SHIFT_ON);
- for (k = 0; rcd->options[k].type != DF_OPT_END_OO; ++k)
+ for (k = 0; (rcd->options[k].bits & DF_OPT_MASK) != DF_OPT_END_OO; ++k)
if (c == rcd->options[k].sopt) {
+ const unsigned int opt_type = rcd->options[k].bits & DF_OPT_MASK;;
/* check plus */
- if ((plus && !(rcd->options[k].flags & DF_OPT_PLUS)) ||
- (!plus && (rcd->options[k].flags & DF_OPT_PLUS)))
+ if ((plus && !(rcd->options[k].bits & DF_OPT_PLUS)) ||
+ (!plus && (rcd->options[k].bits & DF_OPT_PLUS)))
{
continue;
}
/* check shift */
- if (!(rcd->options[k].flags & DF_OPT_NOSHIFT)) {
- if (((shift & SHIFT_ON) && !(rcd->options[k].flags & DF_OPT_SHIFT)) ||
- (!(shift & SHIFT_ON) && (rcd->options[k].flags & DF_OPT_SHIFT)))
+ if (!(rcd->options[k].bits & DF_OPT_NOSHIFT)) {
+ if (((shift & SHIFT_ON) && !(rcd->options[k].bits & DF_OPT_SHIFT)) ||
+ (!(shift & SHIFT_ON) && (rcd->options[k].bits & DF_OPT_SHIFT)))
{
continue;
}
@@ -685,31 +702,28 @@
j, k, rcd->options[k].sopt, rcd->options[k].lopt,
rcd->options[k].directive, anti, shift);
- if (rcd->options[k].type == DF_OPT_CONFIG) {
+ if (opt_type == DF_OPT_CONFIG) {
shortarg[*nshortargs].position = DF_OPT_CONFIG;
shortarg[*nshortargs].required = 1;
shortarg[(*nshortargs)++].option = k;
- } else if (rcd->options[k].type == DF_OPT_SH_ONE)
+ } else if (opt_type == DF_OPT_SH_ONE)
shift |= SHIFT_NEXT;
- else if (rcd->options[k].type == DF_OPT_SH_TGL)
+ else if (opt_type == DF_OPT_SH_TGL)
shift ^= SHIFT_ON;
- else if (rcd->options[k].type == DF_OPT_ANT_OP)
+ else if (opt_type == DF_OPT_ANT_OP)
anti = !anti;
else {
- if (rcd->options[k].type == DF_OPT_SREDIR) {
+ if (opt_type == DF_OPT_SREDIR) {
DF_PushConfig(k, rcd->options[k].lopt, DF_RC_ERR_REDIR, 0, ".", NULL,
i, j, rcd);
- k = rcd->options[k].flags;
+ k = rcd->options[k].lopt[0];
}
DF_PushConfig(k, NULL, DF_RC_ERR_OK, 0, ".", NULL, i, j, rcd);
- if (rcd->options[k].type == DF_OPT_ARG_RQ ||
- rcd->options[k].type == DF_OPT_ARG_OP)
- {
+ if (opt_type == DF_OPT_ARG_RQ || opt_type == DF_OPT_ARG_OP) {
shortarg[*nshortargs].position = rcd->confc - 1;
- shortarg[*nshortargs].required =
- (rcd->options[k].type == DF_OPT_ARG_RQ) ? 1 : 0;
+ shortarg[*nshortargs].required = (opt_type == DF_OPT_ARG_RQ) ? 1 : 0;
shortarg[(*nshortargs)++].option = k;
}
}
@@ -741,9 +755,11 @@
const int fac = (rcd->name == NULL) ? DF_PRN_DF : DF_PRN_INPUT;
/* Find the shift character */
- for (i = 0; rcd->options[i].type != DF_OPT_END_OO; ++i)
- if (rcd->options[i].type == DF_OPT_SH_ONE)
+ for (i = 0; (rcd->options[i].bits & DF_OPT_MASK) != DF_OPT_END_OO; ++i)
+ if ((rcd->options[i].bits & DF_OPT_MASK) == DF_OPT_SH_ONE) {
rcd->shift_one = rcd->options[i].sopt;
+ break;
+ }
logprintf(" Shift character is %c\n", rcd->shift_one);
shortarg = (struct shortarg*)malloc(argc * sizeof(struct shortarg));
@@ -790,7 +806,9 @@
/* match long option */
found = 0;
- for (k = 0; rcd->options[k].type != DF_OPT_END_OO; ++k) {
+ for (k = 0; (rcd->options[k].bits & DF_OPT_MASK) != DF_OPT_END_OO;
+ ++k)
+ {
if (rcd->options[k].lopt != 0 && strcmp(&argv[i][2],
rcd->options[k].lopt) == 0)
{
@@ -844,7 +862,7 @@
rcd->config[shortarg[j].position].line = i;
rcd->config[shortarg[j].position].error = DF_RC_ERR_OK;
- if (rcd->options[shortarg[j].option].flags & DF_OPT_EXPAND)
+ if (rcd->options[shortarg[j].option].bits & DF_OPT_EXPAND)
rcd->config[shortarg[j].position].argument =
df_rc_shell_expand(fac, argv[i]);
else
@@ -920,7 +938,7 @@
else if (rcd->config[i].error == DF_RC_ERR_REDIR) {
df_config_error(DF_CERR_INFO, rcd->config + i, rcd,
"option %c%c deprecated (translated to %s)",
- (rcd->options[rcd->config[i].optind].flags & DF_OPT_PLUS) ? '+'
+ (rcd->options[rcd->config[i].optind].bits & DF_OPT_PLUS) ? '+'
: '-', rcd->options[rcd->config[i].optind].sopt,
rcd->config[i].argument);
} else
Modified: trunk/defile/doc/defile-input.7
===================================================================
--- trunk/defile/doc/defile-input.7 2013-03-09 05:08:59 UTC (rev 811)
+++ trunk/defile/doc/defile-input.7 2013-03-10 19:40:23 UTC (rev 812)
@@ -195,11 +195,10 @@
.fam C
struct df_optdef {
int value;
- int type;
+ unsigned int bits;
char sopt;
const char *lopt;
const char *directive;
- unsigned int flags;
const char *argname;
const char *usage;
};
@@ -211,9 +210,11 @@
.I value
is a positive number used by defile to indentify the option to the input plugin.
Bare arguments among the input options (those not associated with an option) are
-assigned a value of zero, so that should be avoided in the definition of input
+assigned a value of zero, which should be avoided in the definition of input
options. Unless two options should be treated as identical, each option should
-have a unique value. There is no requirement that consequitive values be used.
+have a unique value. There is no requirement that consequitive values be used,
+nor that the option records be sorted by value.
+.IP
After parsing the input options, defile will present the input plugin an array
of (value, argument) pairs. See the
.B Start State
@@ -222,44 +223,39 @@
for information on how the value is used. Negative values are used internally
by defile for bookkeeping purposes and should be avoided.
.TP
-.I type
+.I bits
should be one of the following symbols indicating the type of option:
.RS
.TP
.B DF_OPT_NO_ARG
-an option which takes no argument
+an option which takes no argument;
.TP
.B DF_OPT_ARG_OP
-an option which takes an optional argument
+an option which takes an optional argument;
.TP
.B DF_OPT_ARG_RQ
-an option which requires an argument
+an option which takes a required argument;
+.TP
+.B DF_OPT_CONFIG
+an option which takes the name of a run control file to parse (that is, an
+option which behaves identically to the
+.B \-\-conf
+argument recognised by the
+.BR defile (1)
+core).
.RE
-.TP
-.I sopt
-a character giving the name of the short (one character) command-line option
-which matches this option record. This should be a lower or upper-case letter
-(case is significant). If no short option is defined, this should be zero.
-.TP
-.I lopt
-a character string giving the name of the long command-line option (excluding
-the initial double dash) which matches this option record. If no long option is
-defined, this should be NULL.
-.TP
-.I directive
-a character string giving the name of the run control directive which matches
-this option record. If no directive is defined, this should be NULL.
-.TP
-.I flags
-zero or more of the following flags, bitwise or'd together, providing further
-information on how the option should be interpreted:
+.IP
+These may be optionally bitwise or'd with zero or more of the following flags
+which provide further information on how the option should be interpreted:
.RS
.TP
.B DF_OPT_EXPAND
the argument to the option should be shell expanded before being given to the
input plugin. This is typically useful for options which take filenames as
-arguments. If the option is of type
-.BR DF_OPT_NO_ARG ,
+arguments. If the option is not of type
+.BR DF_OPT_ARG_RQ
+or
+.BR DF_OPT_ARG_OP ,
this flag is ignored.
.TP
.B DF_OPT_PLUS
@@ -277,6 +273,20 @@
is ignored.
.RE
.TP
+.I sopt
+a character giving the name of the short (one character) command-line option
+which matches this option record. This should be a lower or upper-case letter
+(case is significant). If no short option is defined, this should be zero.
+.TP
+.I lopt
+a character string giving the name of the long command-line option (excluding
+the initial double dash) which matches this option record. If no long option is
+defined, this should be NULL.
+.TP
+.I directive
+a character string giving the name of the run control directive which matches
+this option record. If no directive is defined, this should be NULL.
+.TP
.I argname
the placeholder name of the argument used in usage output. For stylistic
consitency, it is recommended this consist of upper case letters only. If the
@@ -313,9 +323,8 @@
.BR \-\-conf ,
and the directive
.B Input
-which are handled specially, as they are in the defile options. See
-.BR defile (1)
-for their use. Even if an input plugin provides no input options (by setting
+which are handled specially, as they are in the defile options. Even if an
+input plugin provides no input options (by setting
.I opts
to zero, these are always accepted as input options by defile. Briefly,
.B \-\-help
@@ -327,16 +336,18 @@
.B \-\-conf
and
.B Include
-specify additional run control files to read for more configuration (see
-.BR defile (1)).
+s...
[truncated message content] |
|
From: <ket...@us...> - 2013-03-15 02:09:55
|
Revision: 821
http://sourceforge.net/p/getdata/code/821
Author: ketiltrout
Date: 2013-03-15 02:09:51 +0000 (Fri, 15 Mar 2013)
Log Message:
-----------
Map fragments across network.
Modified Paths:
--------------
trunk/defile/bin/net.c
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c 2013-03-14 23:05:29 UTC (rev 820)
+++ trunk/defile/bin/net.c 2013-03-15 02:09:51 UTC (rev 821)
@@ -47,6 +47,9 @@
static char *iobuf = NULL;
static uint64_t framecount = 0;
+/* fragment index tracking */
+int n_srv_ind = 0;
+int *srv_ind = NULL;
static char *refname = NULL;
@@ -314,7 +317,7 @@
#define DF_GET_STRING(n) DF_GET_SDATUM(string[n])
#define DF_GET_PARAM(t,n) DF_GET_DATUM(p_ ## t[n],t)
-/* get data with byte swapping */ \
+/* get data with byte swapping */
#define DF_GET_DATA(n,t,l) \
do { \
const size_t len = GD_SIZE(t) * (l); \
@@ -329,11 +332,13 @@
pos++; \
} while (0)
-
#define DF_FREE_STRINGS(n) \
do { int i; for (i = 0; i < n; ++i) free(string[i]); } while(0)
- static int df_do_rpcs(int sock)
+/* array look-up with bounds checking */
+#define df_srv_ind(n) ((n < 0 || n >= n_srv_ind) ? -1 : srv_ind[n])
+
+static int df_do_rpcs(int sock)
{
static int start = 1;
if (start) {
@@ -443,9 +448,9 @@
DF_GET_PARAM(int32,0); /* fragment */
case 3:
df_dprintf(DF_PRN_DEBUG_I, "RPC: add_alias(\"%s\", \"%s\", %i)\n",
- string[0], string[1], (int)p_int32[0]);
+ string[0], string[1], df_srv_ind(p_int32[0]));
- n = df_add_alias(string[0], string[1], (int)p_int32[0]);
+ n = df_add_alias(string[0], string[1], df_srv_ind(p_int32[0]));
DF_FREE_STRINGS(2);
if (n < 0) {
ioerr = DFIO_CALL_ERR;
@@ -643,6 +648,7 @@
if (pos > lastpos) {
df_dprintf(DF_PRN_DEBUG_I, "RPC: add_entry(<entry%x>)\n",
E.field_type);
+ E.fragment_index = df_srv_ind(E.fragment_index);
n = df_add_entry(&E);
gd_free_entry_strings(&E);
if (n < 0) {
@@ -679,7 +685,17 @@
if (n < 0) {
ioerr = DFIO_CALL_ERR;
return 1;
+ } else {
+ /* add it to the fragment index map */
+ int *ptr = (int*)realloc(srv_ind, sizeof(int) * (n_srv_ind + 1));
+ if (ptr == NULL) {
+ DF_OOM(DF_PRN_DF);
+ exit(1);
+ }
+ srv_ind = ptr;
+ srv_ind[n_srv_ind++] = n;
}
+
rpc = -1;
}
break;
@@ -725,8 +741,8 @@
n = (rpc == DF_RPC_SERVER_FRAME) ? 1 : 0;
df_dprintf(DF_PRN_DEBUG_I, "RPC: df_add_framedef(<fdef>, %i, %i)\n", n,
- (int)p_int32[0]);
- n = df_add_framedef(&fd, n, (int)p_int32[0]);
+ df_srv_ind(p_int32[0]));
+ n = df_add_framedef(&fd, n, df_srv_ind(p_int32[0]));
for (i = 0; i < fd.n_fields; ++i)
free(fd.field[i].name);
free(fd.field);
@@ -791,8 +807,8 @@
DF_GET_PARAM(int32,0); /* fragment */
case 2:
df_dprintf(DF_PRN_DEBUG_I, "RPC: df_add_spec(\"%s\", %i)\n",
- string[0], p_int32[0]);
- n = df_add_spec(string[0], p_int32[0]);
+ string[0], df_srv_ind(p_int32[0]));
+ n = df_add_spec(string[0], df_srv_ind(p_int32[0]));
DF_FREE_STRINGS(1);
if (n < 0) {
ioerr = DFIO_CALL_ERR;
@@ -1195,6 +1211,14 @@
goto client_error;
}
+ /* initialise the fragment index mapping */
+ if ((srv_ind = malloc(sizeof(int))) == NULL) {
+ DF_OOM(DF_PRN_DF);
+ goto client_error;
+ }
+ n_srv_ind = 1;
+ srv_ind[0] = 0;
+
/* figure out whether we need to run via ssh */
for (port = df->host; *port && (*port != ':'); ++port)
;
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2013-03-14 23:05:29 UTC (rev 820)
+++ trunk/defile/lib/internal.h 2013-03-15 02:09:51 UTC (rev 821)
@@ -338,7 +338,7 @@
int df_rpc_fini(struct df_rpc *slug, int debug);
int df_rpc_push_mem(struct df_rpc *slug, size_t size, const void *src);
int df_rpc_push_uint8(struct df_rpc *slug, uint8_t v);
-#define df_rpc_push_int8(s,v) df_rpc_push_uint8((s),(uint16_t)(v))
+#define df_rpc_push_int8(s,v) df_rpc_push_uint8((s),(uint8_t)(v))
int df_rpc_push_uint16(struct df_rpc *slug, uint16_t hv);
#define df_rpc_push_int16(s,v) df_rpc_push_uint16((s),(uint16_t)(v))
int df_rpc_push_uint32(struct df_rpc *slug, uint32_t hv);
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-03-14 23:05:29 UTC (rev 820)
+++ trunk/defile/lib/libdefile.c 2013-03-15 02:09:51 UTC (rev 821)
@@ -2037,15 +2037,20 @@
int df_add_fragment(const char *name, int parent, unsigned long encoding,
unsigned long byte_sex, const char *prefix, const char *suffix)
{
+ static int srv_ind = 0;
const int append = df_mode() & DF_MODE_APPEND;
int ind;
char *path, *ptr;
DF_CHECK_SEQUENCE(DF_ST_BUILD);
- if (df->mode_flags & DF_MODE_SERVER)
- return df_rpc_push(df->srv_slug, DF_RPC_ADD_FRAGMENT, name, parent,
+ if (df->mode_flags & DF_MODE_SERVER) {
+ ind = df_rpc_push(df->srv_slug, DF_RPC_ADD_FRAGMENT, name, parent,
encoding, byte_sex, prefix, suffix);
+ if (ind)
+ return ind;
+ return ++srv_ind; /* a fake fragment index */
+ }
/* make the containing directory */
ptr = strdup(name);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-16 01:16:38
|
Revision: 822
http://sourceforge.net/p/getdata/code/822
Author: ketiltrout
Date: 2013-03-16 01:16:36 +0000 (Sat, 16 Mar 2013)
Log Message:
-----------
--clobber-once
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/doc/defile.1.in
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-03-15 02:09:51 UTC (rev 821)
+++ trunk/defile/bin/defile.c 2013-03-16 01:16:36 UTC (rev 822)
@@ -943,7 +943,7 @@
DF_WOPT_VERBOSE, DF_WOPT_REMOTE, DF_WOPT_LOCAL, DF_WOPT_RSH, DF_WOPT_LISTEN,
DF_WOPT_NOLISTEN, DF_WOPT_RCMD, DF_WOPT_RDEBUG, DF_WOPT_NORDEBUG,
DF_WOPT_SRVRATE, DF_WOPT_DAEMON, DF_WOPT_CURFILE, DF_WOPT_SYMLINK,
- DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE
+ DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE, DF_WOPT_CLOBBERONE
};
static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
@@ -1060,8 +1060,10 @@
SETULONG(BLEN, buflen);
SETLINT(CLOBBER, clobber, 1);
+ SETLINT(CLOBBERONE, clobber, 2);
SETLINT(NOCLOBBER, clobber, 0);
+
SETINT(CUTDIR, cutdir);
SETMODE(NODAEMON, DF_MODE_NO_DAEMON);
@@ -2484,6 +2486,9 @@
{ DF_WOPT_CLOBBER, DF_OPT_NO_ARG, 0, "clobber", "Clobber", NULL,
"if the output dirfile already exists, overwrite it; use with caution"
},
+ { DF_WOPT_CLOBBERONE, DF_OPT_NO_ARG, 0, "clobber-once", "ClobberOnce", NULL,
+ "like --clobber, but only overwrite the first output dirfile"
+ },
{ DF_WOPT_NOCLOBBER, DF_OPT_NO_ARG, 0, "no-clobber", "NoClobber", NULL,
"don't overwrite an existing dirfile (default)"
},
@@ -2772,7 +2777,7 @@
df_subset_reset(df->ofl, DF_SUBSET_INIT);
}
if (df->clobber) {
- df_dprintf(DF_PRN_WARN, "ignoring --clobber specified "
+ df_dprintf(DF_PRN_WARN, "ignoring --clobber or --clobber-once specified "
"with --listen\n");
}
if (df->srv_rate < 1e-5) {
Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in 2013-03-15 02:09:51 UTC (rev 821)
+++ trunk/defile/doc/defile.1.in 2013-03-16 01:16:36 UTC (rev 822)
@@ -172,15 +172,18 @@
.B \-\-output
has been specified, the new dirfile will have the same name as the old one. If
.B \-\-clobber
-has been given, the old dirfile will be overwitten. If
+has been given, defile will delete the dirfile it just finished. If
.B \-\-clobber
-has not been specified, defile will exit with an error, just as if it had been
-asked to overwrite an existing dirfile. If
+has not been given, defile will exit with an error, just as if it had been asked
+to overwrite an existing dirfile. If
.B \-\-output
has not been specified, and the input plugin provides a new name, a new output
dirfile will be created in the directory specified with
.BR \-\-directory ,
-if possible.
+if possible. If you want to clobber a pre-existing, explicitly named output
+dirfile, but not allow the input plugin to cause defile to overwrite its output,
+use
+.BR \-\-clobber\-once .
.TP
.B \-\-clobber
@@ -189,6 +192,18 @@
caution! (Directive:
.BR Clobber )
.TP
+.B \-\-clobber\-once
+like
+.BR \-\-clobber ,
+but only allow overwriting an existing dirfile the
+.I first
+time defile creates a dirfile. The distinction is only meaningful if the input
+plugin tries to create multiple dirfiles in succession, and is typically only
+useful when also specifying an output dirfile name (via
+.BR \-\-output ).
+See the discussion above. (Directive:
+.BR ClobberOnce )
+.TP
.B \-\-no\-clobber
don't overwrite an existing dirfile. This is the default behaviour. (Directive:
.BR NoClobber )
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-03-15 02:09:51 UTC (rev 821)
+++ trunk/defile/lib/libdefile.c 2013-03-16 01:16:36 UTC (rev 822)
@@ -1463,9 +1463,11 @@
pthread_mutex_lock(&df_config_mx);
df->input_offset = df->offset = 0;
df->mode_flags &= ~DF_MODE_APPEND;
+ if (df->clobber > 1) /* --clobber-once */
+ df->clobber = 0;
pthread_mutex_unlock(&df_config_mx);
- /* trigger and wait for completion of the output -- in server mode, this
+ /* trigger and wait for completion of the output -- in server mode,
* the output will send a special RPC to the clients indicating reinit */
DF_INSTATE_MX(BUILD);
if (mode == DF_REINIT_DROP) {
@@ -1482,7 +1484,6 @@
* input to abort now. */
if (df->output_state != DF_ST_BUILD) {
DF_INSTATE_MX(ABORT);
- pthread_mutex_unlock(&df_state_mx);
return df_input_abort();
}
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-03-16 01:37:32
|
Revision: 823
http://sourceforge.net/p/getdata/code/823
Author: ketiltrout
Date: 2013-03-16 01:37:28 +0000 (Sat, 16 Mar 2013)
Log Message:
-----------
Doc updates.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/doc/defile.1.in
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-03-16 01:16:36 UTC (rev 822)
+++ trunk/defile/bin/defile.c 2013-03-16 01:37:28 UTC (rev 823)
@@ -2481,7 +2481,7 @@
NULL, "turn on asynchronous mode"
},
{ DF_WOPT_SYNC, DF_OPT_PLUS | DF_OPT_NO_ARG, 'a', "sync", "Synchronous",
- NULL, "turn off synchronous mode"
+ NULL, "turn off asynchronous mode (default)"
},
{ DF_WOPT_CLOBBER, DF_OPT_NO_ARG, 0, "clobber", "Clobber", NULL,
"if the output dirfile already exists, overwrite it; use with caution"
Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in 2013-03-16 01:16:36 UTC (rev 822)
+++ trunk/defile/doc/defile.1.in 2013-03-16 01:37:28 UTC (rev 823)
@@ -208,20 +208,21 @@
don't overwrite an existing dirfile. This is the default behaviour. (Directive:
.BR NoClobber )
.TP
-.BR \-D ", " \-\-directory=\fIDIR
+.BR \-D ", " \-\-directory=\fIPATH
unless
.B \-\-output
is given and specifies an absolute path, write the output dirfile as a
subdirectory under
-.IR DIR .
+.IR PATH .
If
.B \-\-output
is not specified, then name of the output dirfile created under
-.B DIR
+.B PATH
will be provided by the input plugin, or else be based on
.IR INPUT .
If
-.B \-\-output does specify an absolute path, this option is ignored. (Directive:
+.B \-\-output
+does specify an absolute path, this option is ignored. (Directive:
.BR Directory )
.TP
.BR \-o ", " \-\-output=\fIPATH
@@ -236,22 +237,10 @@
directories specified in
.IR PATH .
Using this option may have averse effects if the input causes defile to cycle as
-explained above.
-.IP
-Some input plugins may trigger defile to create more than one consecutive output
-dirfile (in cases when the input plugin notices a change in the data source
-metadata or name). When this occurs while using this option, if
-.B \-\-clobber
-has been specified, defile will truncate the old dirfile that it was just
-writing, and overwrite it with a new (empty) one at
-.IR PATH .
-If
-.B \-\-clobber
-has not been specified, defile will exit with an error, having been asked to
-overwrite an existing dirfile. (Directive:
+explained above. (Directive:
.BR Output )
.TP
-.BR \-\-cut\-dirs= N
+.BI \-\-cut\-dirs= N
When no output dirfile name is explicitly given, defile will generate one from
the
.I INPUT
@@ -271,6 +260,40 @@
.B \-\-output
is used to explicitly name the output dirfile. (Directive:
.BR CutDirs )
+.TP
+.BR \-S ", " \-\-symlink=\fIPATH
+Create a symlink called
+.I PATH
+(relative to the current directory, if not absolute) pointing to the output
+dirfile. An existing file or symlink called
+.I PATH
+will be overwritten. The symlink is updated if the input plugin causes defile
+to cycle between output dirfiles. The traditional name of the symlink is
+.IR defile.lnk .
+(Directive:
+.BR Symlink )
+.TP
+.BR +S ", " \-\-no\-symlink
+Don't create a symlink. This is the default behaviour. (Directive:
+.BR NoSymlink )
+.TP
+.BR \-C ", " \-\-curfile=\fIPATH
+Create a text file called
+.I PATH
+(relative to the current directory, if not absolute) containing the full
+pathname of the output dirfile. An existing file called
+.I PATH
+will be overwritten. The file is updated (re-written) if the input plugin
+causes defile to cycle between output dirfiles. In the Dirfile milieu, this
+is known as a "curfile", and can be used as a poor man's symlink. The
+traditional name of the file is
+.IR defile.cur .
+(Directive:
+.BR CurFile )
+.TP
+.BR +C ", " \-\-no\-curfile
+Don't create a curfile. This is the default behaviour. (Directive:
+.BR NoCurFile )
.SS Running Mode Options
The following options affect the modes in which defile runs.
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-06-21 17:19:08
|
Revision: 834
http://sourceforge.net/p/getdata/code/834
Author: ketiltrout
Date: 2013-06-21 17:19:05 +0000 (Fri, 21 Jun 2013)
Log Message:
-----------
Update for GetData-0.8.4.
Modified Paths:
--------------
trunk/defile/bin/Makefile.am
trunk/defile/bin/net.c
trunk/defile/configure.ac
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/Makefile.am
===================================================================
--- trunk/defile/bin/Makefile.am 2013-05-15 00:57:18 UTC (rev 833)
+++ trunk/defile/bin/Makefile.am 2013-06-21 17:19:05 UTC (rev 834)
@@ -24,4 +24,4 @@
AM_CPPFLAGS = ${GETDATA_CFLAGS} \
-DDEFILE_MODULEDIR="\"${moduledir}\"" ${LTDLINCL}
defile_SOURCES = defile.c rc.c net.c rc.h
-defile_LDADD = ../lib/libdefile.la -lltdl
+defile_LDADD = ../lib/libdefile.la -lltdl ${GETDATA_LIBS}
Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c 2013-05-15 00:57:18 UTC (rev 833)
+++ trunk/defile/bin/net.c 2013-06-21 17:19:05 UTC (rev 834)
@@ -555,7 +555,7 @@
case 7:
DF_GET_DATUM(E.scalar_ind[0], uint32);
case 8:
- DF_GET_DATUM(E.count_max, int32);
+ DF_GET_DATUM(E.period, int32);
case 9:
DF_GET_SDATUM(E.scalar[1]);
case 10:
Modified: trunk/defile/configure.ac
===================================================================
--- trunk/defile/configure.ac 2013-05-15 00:57:18 UTC (rev 833)
+++ trunk/defile/configure.ac 2013-06-21 17:19:05 UTC (rev 834)
@@ -136,7 +136,7 @@
echo
echo "*** Checking for external libraries"
echo
-PKG_CHECK_MODULES([GETDATA], [getdata >= 0.8.1])
+PKG_CHECK_MODULES([GETDATA], [getdata >= 0.8.4])
AC_SEARCH_LIBS([exp],[m])
AC_SEARCH_LIBS([pthread_create],[pthread])
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-05-15 00:57:18 UTC (rev 833)
+++ trunk/defile/lib/libdefile.c 2013-06-21 17:19:05 UTC (rev 834)
@@ -172,7 +172,7 @@
|| df_rpc_push_int32(slug, E->count_val)
|| df_rpc_push_string(slug, E->scalar[0])
|| df_rpc_push_uint32(slug, E->scalar_ind[0])
- || df_rpc_push_int32(slug, E->count_max)
+ || df_rpc_push_int32(slug, E->period)
|| df_rpc_push_string(slug, E->scalar[1])
|| df_rpc_push_uint32(slug, E->scalar_ind[1]);
break;
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-07-18 18:27:54
|
Revision: 837
http://sourceforge.net/p/getdata/code/837
Author: ketiltrout
Date: 2013-07-18 18:27:51 +0000 (Thu, 18 Jul 2013)
Log Message:
-----------
Use GetData's C89 api for compatibility reasons. Some BSD fixes.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/bin/net.c
trunk/defile/input/ascii.c
trunk/defile/input/dirfile.c
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
trunk/defile/test/make.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/bin/defile.c 2013-07-18 18:27:51 UTC (rev 837)
@@ -465,12 +465,12 @@
return df_rpc_push_uint64(slug, u64);
}
-int df_rpc_push_complex_double(struct df_rpc *slug, complex double v)
+int df_rpc_push_complex_double(struct df_rpc *slug, const double v[2])
{
- if (df_rpc_push_double(slug, creal(v)))
+ if (df_rpc_push_double(slug, v[0]))
return 1;
- return df_rpc_push_double(slug, cimag(v));
+ return df_rpc_push_double(slug, v[1]);
}
/* add a string to the slug */
@@ -1169,7 +1169,6 @@
double delta, rate, drate;
long long nf, nw, nr;
char si;
- struct timezone tz;
struct timeval now;
pthread_mutex_lock(&df_rate_mx);
@@ -1178,7 +1177,7 @@
nr = df->nread;
pthread_mutex_unlock(&df_rate_mx);
- gettimeofday(&now, &tz);
+ gettimeofday(&now, NULL);
if (final) {
delta = (now.tv_sec - df->start.tv_sec) +
(now.tv_usec - df->start.tv_usec) / 1e6;
@@ -1635,7 +1634,7 @@
return -1;
}
- if (setsockopt(sock, SOL_TCP, TCP_NODELAY, &n, sizeof(n)) != 0) {
+ if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, &n, sizeof(n)) != 0) {
df_dperror(DF_PRN_ERR, "setsockopt");
close(sock);
return -1;
@@ -1671,7 +1670,6 @@
static int init = 1;
static struct timeval then;
- struct timezone tz;
struct timeval now;
int i, have_fd = 0;
@@ -1697,7 +1695,7 @@
return nf;
/* rate limiting */
- gettimeofday(&now, &tz);
+ gettimeofday(&now, NULL);
if (init) {
/* be conservative */
nf = 1;
@@ -1815,7 +1813,7 @@
if (listen && pfd[0].revents & POLLIN) {
struct sockaddr_in addr;
socklen_t addrlen = sizeof(addr);
- int sock = accept(pfd[0].fd, &addr, &addrlen);
+ int sock = accept(pfd[0].fd, (struct sockaddr *)&addr, &addrlen);
if (nfds == 1 + DF_MAX_CONNECT) /* too many connections */
close(sock);
@@ -2464,9 +2462,6 @@
int main(int argc, char **argv)
{
- /* rate stuff */
- struct timezone tz;
-
sigset_t signals;
struct sigaction action;
@@ -2875,7 +2870,7 @@
if (df->quiet || (df->mode_flags & DF_MODE_SERVER)) {
/* In quiet mode, just wait until we're done */
while (DF_ThreadCheck(df, &df->then)) {
- gettimeofday(&df->then, &tz);
+ gettimeofday(&df->then, NULL);
sleep(1);
}
} else {
Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/bin/net.c 2013-07-18 18:27:51 UTC (rev 837)
@@ -18,6 +18,7 @@
* with Defile; if not, write to the Free Software Foundation, Inc.,
* 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#define GD_C89_API
#include "defile.h"
#include "internal.h"
#include <sys/time.h>
@@ -207,18 +208,15 @@
return v;
}
-static _Complex double df_rpc_get_cdouble(int sock)
+static void df_rpc_get_cdouble(double v[2], int sock)
{
- double _Complex v;
uint64_t u[2];
if (df_rpc_get_mem(sock, sizeof(v), u, NULL))
- return 0;
+ return;
u[0] = df_ntohll(u[0]);
u[1] = df_ntohll(u[1]);
- memcpy(&v, u, sizeof(v));
-
- return v;
+ memcpy(v, u, sizeof(double) * 2);
}
/* this will malloc the string -- it's a bit tricky, because we might end up
@@ -294,7 +292,6 @@
}
#define double_t double
-#define cdouble_t _Complex double
#define DF_GET_SDATUM(v) \
do { \
size_t len; \
@@ -314,6 +311,15 @@
remaining -= sizeof(t ## _t); \
} while(0)
+#define DF_GET_CDATUM(v) \
+ do { \
+ df_rpc_get_cdouble(v, sock); \
+ if (ioerr == DFIO_NO_DATA) return 0; \
+ if (ioerr) return 1; \
+ pos++; \
+ remaining -= sizeof(2 * sizeof(double)); \
+ } while(0)
+
#define DF_GET_STRING(n) DF_GET_SDATUM(string[n])
#define DF_GET_PARAM(t,n) DF_GET_DATUM(p_ ## t[n],t)
@@ -341,11 +347,7 @@
static int df_do_rpcs(int sock)
{
static int start = 1;
- if (start) {
- struct timezone tz;
- gettimeofday(&df->start, &tz);
- start = 0;
- }
+
static int rpc = -1;
static int pos = 0;
static int do_reinit_how = -1;
@@ -371,6 +373,11 @@
int i, n;
size_t part;
+ if (start) {
+ gettimeofday(&df->start, NULL);
+ start = 0;
+ }
+
/* check for the sentinal */
if (remaining == 2) {
uint16_t sentinal = df_rpc_get_uint16(sock);
@@ -479,13 +486,13 @@
case 3:
DF_GET_SDATUM(E.in_fields[0]);
case 4:
- DF_GET_DATUM(E.bitnum, int32);
+ DF_GET_DATUM(E.u.bit.bitnum, int32);
case 5:
DF_GET_SDATUM(E.scalar[0]);
case 6:
DF_GET_DATUM(E.scalar_ind[0], uint32);
case 7:
- DF_GET_DATUM(E.numbits, int32);
+ DF_GET_DATUM(E.u.bit.numbits, int32);
case 8:
DF_GET_SDATUM(E.scalar[1]);
case 9:
@@ -496,35 +503,35 @@
lastpos = 4;
switch (pos) {
case 3:
- DF_GET_DATUM(E.const_type, uint16);
+ DF_GET_DATUM(E.u.scalar.const_type, uint16);
case 4:
- DF_GET_DATUM(E.array_len, uint32);
+ DF_GET_DATUM(E.u.scalar.array_len, uint32);
}
break;
case GD_CONST_ENTRY:
lastpos = 3;
if (pos == 3)
- DF_GET_DATUM(E.const_type, uint16);
+ DF_GET_DATUM(E.u.scalar.const_type, uint16);
break;
case GD_LINCOM_ENTRY:
if (pos == 3) {
lastpos = 4;
- DF_GET_DATUM(E.n_fields, uint8);
+ DF_GET_DATUM(E.u.lincom.n_fields, uint8);
}
if (pos > 3) {
- lastpos = E.n_fields * 7 + 3;
- for (i = 0; i < E.n_fields; ++i)
+ lastpos = E.u.lincom.n_fields * 7 + 3;
+ for (i = 0; i < E.u.lincom.n_fields; ++i)
switch ((pos - 4) % 7) {
case 0:
DF_GET_SDATUM(E.in_fields[i]);
case 1:
- DF_GET_DATUM(E.cm[i], cdouble);
+ DF_GET_CDATUM(E.u.lincom.cm[i]);
case 2:
DF_GET_SDATUM(E.scalar[i]);
case 3:
DF_GET_DATUM(E.scalar_ind[i], uint32);
case 4:
- DF_GET_DATUM(E.cb[i], cdouble);
+ DF_GET_CDATUM(E.u.lincom.cb[i]);
case 5:
DF_GET_SDATUM(E.scalar[i + GD_MAX_LINCOM]);
case 6:
@@ -538,7 +545,7 @@
case 3:
DF_GET_SDATUM(E.in_fields[0]);
case 4:
- DF_GET_SDATUM(E.table);
+ DF_GET_SDATUM(E.u.linterp.table);
}
break;
case GD_MPLEX_ENTRY:
@@ -549,13 +556,13 @@
case 4:
DF_GET_SDATUM(E.in_fields[1]);
case 5:
- DF_GET_DATUM(E.count_val, int32);
+ DF_GET_DATUM(E.u.mplex.count_val, int32);
case 6:
DF_GET_SDATUM(E.scalar[0]);
case 7:
DF_GET_DATUM(E.scalar_ind[0], uint32);
case 8:
- DF_GET_DATUM(E.period, int32);
+ DF_GET_DATUM(E.u.mplex.period, int32);
case 9:
DF_GET_SDATUM(E.scalar[1]);
case 10:
@@ -578,7 +585,7 @@
case 3:
DF_GET_SDATUM(E.in_fields[0]);
case 4:
- DF_GET_DATUM(E.shift, int64);
+ DF_GET_DATUM(E.u.phase.shift, int64);
case 5:
DF_GET_SDATUM(E.scalar[0]);
case 6:
@@ -591,14 +598,14 @@
case 3:
DF_GET_SDATUM(E.in_fields[0]);
case 4:
- DF_GET_DATUM(E.poly_ord, uint8);
+ DF_GET_DATUM(E.u.polynom.poly_ord, uint8);
}
if (pos > 4) {
- lastpos = 3 * (E.poly_ord + 1) + 4;
- for (i = 0; i <= E.poly_ord; ++i)
+ lastpos = 3 * (E.u.polynom.poly_ord + 1) + 4;
+ for (i = 0; i <= E.u.polynom.poly_ord; ++i)
switch ((pos - 5) % 3) {
case 0:
- DF_GET_DATUM(E.ca[i], cdouble);
+ DF_GET_CDATUM(E.u.polynom.ca[i]);
case 2:
DF_GET_SDATUM(E.scalar[i]);
case 3:
@@ -612,7 +619,7 @@
case 3:
DF_GET_SDATUM(E.in_fields[0]);
case 4:
- DF_GET_DATUM(E.cdividend, cdouble);
+ DF_GET_CDATUM(E.u.recip.cdividend);
case 5:
DF_GET_SDATUM(E.scalar[0]);
case 6:
@@ -630,9 +637,9 @@
case 4:
DF_GET_SDATUM(E.in_fields[1]);
case 5:
- DF_GET_DATUM(E.windop, uint8);
+ DF_GET_DATUM(E.u.window.windop, uint8);
case 6:
- DF_GET_DATUM(E.threshold.u, uint64);
+ DF_GET_DATUM(E.u.window.threshold.u, uint64);
case 7:
DF_GET_SDATUM(E.scalar[0]);
case 8:
Modified: trunk/defile/input/ascii.c
===================================================================
--- trunk/defile/input/ascii.c 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/input/ascii.c 2013-07-18 18:27:51 UTC (rev 837)
@@ -1444,12 +1444,12 @@
df_exit(1, 1);
} else {
E.field_type = GD_CONST_ENTRY;
- E.const_type = ascii_conf.header[j].type;
+ E.u.scalar.const_type = ascii_conf.header[j].type;
if (df_add_entry(&E))
df_exit(1, 1);
- if (df_set_const(E.field, E.const_type,
+ if (df_set_const(E.field, E.u.scalar.const_type,
((char*)ascii_conf.header[j].data) +
GD_SIZE(ascii_conf.header[j].type) * i))
{
Modified: trunk/defile/input/dirfile.c
===================================================================
--- trunk/defile/input/dirfile.c 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/input/dirfile.c 2013-07-18 18:27:51 UTC (rev 837)
@@ -27,7 +27,6 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
-#include <complex.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
@@ -431,11 +430,11 @@
/* convert the name, if necessary */
if ((ext && !conf.link_lut) || conf.flatten) {
size_t old_lut_len = strlen(old_lut);
- free(E->table);
- E->table = malloc(old_lut_len + conf.lut_dir_len + 2);
- sprintf(E->table, "%s%s", conf.lut_dir, old_lut);
+ free(E->u.linterp.table);
+ E->u.linterp.table = malloc(old_lut_len + conf.lut_dir_len + 2);
+ sprintf(E->u.linterp.table, "%s%s", conf.lut_dir, old_lut);
lut = malloc(dirfilename_len + old_lut_len + conf.lut_dir_len + 3);
- sprintf(lut, "%s/%s", dirfilename, E->table);
+ sprintf(lut, "%s/%s", dirfilename, E->u.linterp.table);
free(old_lut);
} else
lut = old_lut;
@@ -610,11 +609,11 @@
/* set scalar values */
if (E.field_type == GD_CONST_ENTRY) {
char buffer[16];
- if (gd_get_constant(D, E.field, E.const_type, buffer)) {
+ if (gd_get_constant(D, E.field, E.u.scalar.const_type, buffer)) {
gd_free_entry_strings(&E);
df_exit(1, 1);
}
- if (df_set_const(E.field, E.const_type, buffer)) {
+ if (df_set_const(E.field, E.u.scalar.const_type, buffer)) {
gd_free_entry_strings(&E);
df_exit(1, 1);
}
@@ -646,17 +645,20 @@
}
free(buffer);
} else if (E.field_type == GD_CARRAY_ENTRY) {
- void *buffer = malloc(E.array_len * GD_SIZE(E.const_type));
+ void *buffer = malloc(E.u.scalar.array_len *
+ GD_SIZE(E.u.scalar.const_type));
if (buffer == NULL) {
gd_free_entry_strings(&E);
df_exit(1, 1);
}
- if (gd_get_carray(D, E.field, E.const_type, buffer) < 0) {
+ if (gd_get_carray(D, E.field, E.u.scalar.const_type, buffer) < 0) {
free(buffer);
gd_free_entry_strings(&E);
df_exit(1, 1);
}
- if (df_set_carray(E.field, 0, E.array_len, E.const_type, buffer)) {
+ if (df_set_carray(E.field, 0, E.u.scalar.array_len,
+ E.u.scalar.const_type, buffer))
+ {
free(buffer);
gd_free_entry_strings(&E);
df_exit(1, 1);
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/lib/internal.h 2013-07-18 18:27:51 UTC (rev 837)
@@ -26,12 +26,14 @@
#include "config.h"
#endif
+#define GD_C89_API
+#include <getdata.h>
+
#include "defile.h"
-#include <complex.h>
-#include <getdata.h>
#include <syslog.h>
#include <pthread.h>
+#include <stdint.h>
#ifdef HAVE_ENDIAN_H
#include <endian.h>
@@ -346,7 +348,7 @@
int df_rpc_push_uint64(struct df_rpc *slug, uint64_t hv);
#define df_rpc_push_int64(s,v) df_rpc_push_uint64((s),(uint64_t)(v))
int df_rpc_push_double(struct df_rpc *slug, double hv);
-int df_rpc_push_complex_double(struct df_rpc *slug, complex double v);
+int df_rpc_push_complex_double(struct df_rpc *slug, const double v[2]);
int df_rpc_push_string(struct df_rpc *slug, const char *str);
int df_rpc_push_data(struct df_rpc *slug, gd_type_t type, uint32_t n,
const void *data);
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/lib/libdefile.c 2013-07-18 18:27:51 UTC (rev 837)
@@ -136,43 +136,43 @@
case GD_BIT_ENTRY:
case GD_SBIT_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
- || df_rpc_push_int32(slug, E->bitnum)
+ || df_rpc_push_int32(slug, E->u.bit.bitnum)
|| df_rpc_push_string(slug, E->scalar[0])
|| df_rpc_push_uint32(slug, E->scalar_ind[0])
- || df_rpc_push_int32(slug, E->numbits)
+ || df_rpc_push_int32(slug, E->u.bit.numbits)
|| df_rpc_push_string(slug, E->scalar[1])
|| df_rpc_push_uint32(slug, E->scalar_ind[1]);
break;
case GD_CARRAY_ENTRY:
- r = df_rpc_push_uint16(slug, E->const_type)
- || df_rpc_push_uint32(slug, E->array_len);
+ r = df_rpc_push_uint16(slug, E->u.scalar.const_type)
+ || df_rpc_push_uint32(slug, E->u.scalar.array_len);
break;
case GD_CONST_ENTRY:
- r = df_rpc_push_uint16(slug, E->const_type);
+ r = df_rpc_push_uint16(slug, E->u.scalar.const_type);
break;
case GD_LINCOM_ENTRY:
- r = df_rpc_push_uint8(slug, E->n_fields);
- for (i = 0; !r && i < E->n_fields; ++i) {
+ r = df_rpc_push_uint8(slug, E->u.lincom.n_fields);
+ for (i = 0; !r && i < E->u.lincom.n_fields; ++i) {
r = df_rpc_push_string(slug, E->in_fields[i])
- || df_rpc_push_complex_double(slug, E->cm[i])
+ || df_rpc_push_complex_double(slug, E->u.lincom.cm[i])
|| df_rpc_push_string(slug, E->scalar[i])
|| df_rpc_push_uint32(slug, E->scalar_ind[i])
- || df_rpc_push_complex_double(slug, E->cb[i])
+ || df_rpc_push_complex_double(slug, E->u.lincom.cb[i])
|| df_rpc_push_string(slug, E->scalar[i + GD_MAX_LINCOM])
|| df_rpc_push_uint32(slug, E->scalar_ind[i + GD_MAX_LINCOM]);
}
break;
case GD_LINTERP_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
- || df_rpc_push_string(slug, E->table);
+ || df_rpc_push_string(slug, E->u.linterp.table);
break;
case GD_MPLEX_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
|| df_rpc_push_string(slug, E->in_fields[1])
- || df_rpc_push_int32(slug, E->count_val)
+ || df_rpc_push_int32(slug, E->u.mplex.count_val)
|| df_rpc_push_string(slug, E->scalar[0])
|| df_rpc_push_uint32(slug, E->scalar_ind[0])
- || df_rpc_push_int32(slug, E->period)
+ || df_rpc_push_int32(slug, E->u.mplex.period)
|| df_rpc_push_string(slug, E->scalar[1])
|| df_rpc_push_uint32(slug, E->scalar_ind[1]);
break;
@@ -183,22 +183,22 @@
break;
case GD_PHASE_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
- || df_rpc_push_int64(slug, E->shift)
+ || df_rpc_push_int64(slug, E->u.phase.shift)
|| df_rpc_push_string(slug, E->scalar[0])
|| df_rpc_push_uint32(slug, E->scalar_ind[0]);
break;
case GD_POLYNOM_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
- || df_rpc_push_uint8(slug, E->poly_ord);
- for (i = 0; !r && i <= E->poly_ord; ++i) {
- r = df_rpc_push_complex_double(slug, E->ca[i])
+ || df_rpc_push_uint8(slug, E->u.polynom.poly_ord);
+ for (i = 0; !r && i <= E->u.polynom.poly_ord; ++i) {
+ r = df_rpc_push_complex_double(slug, E->u.polynom.ca[i])
|| df_rpc_push_string(slug, E->scalar[i])
|| df_rpc_push_uint32(slug, E->scalar_ind[i]);
}
break;
case GD_RECIP_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
- || df_rpc_push_complex_double(slug, E->cdividend)
+ || df_rpc_push_complex_double(slug, E->u.recip.cdividend)
|| df_rpc_push_string(slug, E->scalar[0])
|| df_rpc_push_uint32(slug, E->scalar_ind[0]);
break;
@@ -207,8 +207,8 @@
case GD_WINDOW_ENTRY:
r = df_rpc_push_string(slug, E->in_fields[0])
|| df_rpc_push_string(slug, E->in_fields[1])
- || df_rpc_push_uint8(slug, E->windop)
- || df_rpc_push_uint64(slug, E->threshold.u)
+ || df_rpc_push_uint8(slug, E->u.window.windop)
+ || df_rpc_push_uint64(slug, E->u.window.threshold.u)
|| df_rpc_push_string(slug, E->scalar[0])
|| df_rpc_push_uint32(slug, E->scalar_ind[0]);
break;
@@ -1132,7 +1132,6 @@
int i, missing = 0, set_reference = 1;
const int append = df_mode() & DF_MODE_APPEND;
long long offset;
- struct timezone tz;
DF_CHECK_SEQUENCE(DF_ST_BUILD);
@@ -1178,14 +1177,14 @@
"derived type 0x%X\n", df->raw[i].name, E.field_type);
gd_free_entry_strings(&E);
return DF_OUTPUT;
- } else if (E.spf != df->raw[i].spf) {
+ } else if (E.u.raw.spf != df->raw[i].spf) {
df_lprintf(DF_PRN_ERR, "existing raw field %s redefined with "
- "different sample rate %u\n", df->raw[i].name, E.spf);
+ "different sample rate %u\n", df->raw[i].name, E.u.raw.spf);
gd_free_entry_strings(&E);
return DF_OUTPUT;
- } else if (E.data_type != df->raw[i].type) {
+ } else if (E.u.raw.data_type != df->raw[i].type) {
df_lprintf(DF_PRN_ERR, "existing raw field %s redefined with "
- "different data type 0x%X\n", df->raw[i].name, E.data_type);
+ "different data type 0x%X\n", df->raw[i].name, E.u.raw.data_type);
gd_free_entry_strings(&E);
return DF_OUTPUT;
}
@@ -1245,7 +1244,7 @@
}
/* record our "start" time */
- gettimeofday(&df->start, &tz);
+ gettimeofday(&df->start, NULL);
memcpy(&df->then, &df->start, sizeof(df->then));
/* switch input state */
Modified: trunk/defile/test/make.c
===================================================================
--- trunk/defile/test/make.c 2013-07-18 16:05:54 UTC (rev 836)
+++ trunk/defile/test/make.c 2013-07-18 18:27:51 UTC (rev 837)
@@ -25,6 +25,8 @@
#include "make.h"
+#include <stdint.h>
+
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
@@ -413,8 +415,8 @@
gd_discard(D);
exit(1);
}
- raws[i].type = E.data_type;
- raws[i].spf = E.spf;
+ raws[i].type = E.u.raw.data_type;
+ raws[i].spf = E.u.raw.spf;
}
} else {
char *buffer = malloc(strlen(name) + 11);
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2013-07-27 12:20:41
|
Revision: 842
http://sourceforge.net/p/getdata/code/842
Author: ketiltrout
Date: 2013-07-27 12:20:38 +0000 (Sat, 27 Jul 2013)
Log Message:
-----------
Fix server-side df_reinit.
Modified Paths:
--------------
trunk/defile/bin/net.c
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c 2013-07-25 10:55:55 UTC (rev 841)
+++ trunk/defile/bin/net.c 2013-07-27 12:20:38 UTC (rev 842)
@@ -1037,6 +1037,25 @@
}
}
break;
+ case DF_RPC_HELLO:
+ /* an extra hello, probably due to a reinit, just get and discard the
+ * whole thing */
+ switch (pos) {
+ case 0:
+ string[0] = malloc(DF_HELLO_LEN);
+ if (string[0] == NULL) {
+ DF_OOM(DF_PRN_DF);
+ exit(1);
+ }
+ framepos = 0;
+ pos++;
+ case 1:
+ DF_GET_DATA(0, GD_UINT8, DF_HELLO_LEN - 1);
+ case 2:
+ free(string[0]);
+ rpc = -1;
+ }
+ break;
default:
df_dprintf(DF_PRN_ERR, "unhandled rpc from server: %02x\n",
(unsigned int)rpc);
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2013-07-25 10:55:55 UTC (rev 841)
+++ trunk/defile/lib/internal.h 2013-07-27 12:20:38 UTC (rev 842)
@@ -119,8 +119,8 @@
#define DF_RPC_HELLO 0xFF
/* the server salutation */
-#define DF_HELLO {DF_RPC_HELLO, 0x0A, 'D', 'E', 'F', 'I', 'L', 'E', 0x0D, 0x0A, DF_SRV_SEX, DF_RPC_VERSION, \
- DF_RPC_REVISION}
+#define DF_HELLO {DF_RPC_HELLO, 0x0A, 'D', 'E', 'F', 'I', 'L', 'E', 0x0D, \
+ 0x0A, DF_SRV_SEX, DF_RPC_VERSION, DF_RPC_REVISION}
#define DF_HELLO_LEN 13
/* private mode flags */
@@ -302,6 +302,7 @@
struct df_rpc *srv_slug;
int srv_nslug;
pthread_mutex_t srv_mx;
+ int srv_ind;
int nfd;
struct df_infdef *fd;
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2013-07-25 10:55:55 UTC (rev 841)
+++ trunk/defile/lib/libdefile.c 2013-07-27 12:20:38 UTC (rev 842)
@@ -107,7 +107,7 @@
return df_rc_shell_expand(DF_PRN_INPUT, argument);
}
-/* Push a RPC ont a slug */
+/* Push a RPC onto a slug */
static int df_rpc_vpush(struct df_rpc *slug, uint8_t rpc, va_list ap)
{
const struct df_fdef *fd;
@@ -1039,10 +1039,10 @@
usleep(10000);
}
-static int df_init_server(long long length, double rate, const char *name,
- int hello)
+static int df_init_server(long long length, double rate, const char *name)
{
if (df->mode_flags & DF_MODE_SERVER) {
+ df->srv_ind = 0;
df->srv_nslug = 1;
df->srv_slug = (struct df_rpc *)malloc(sizeof(struct df_rpc));
@@ -1050,7 +1050,7 @@
return DF_OUTPUT;
/* server salutation */
- if (hello && df_rpc_push(df->srv_slug, DF_RPC_HELLO))
+ if (df_rpc_push(df->srv_slug, DF_RPC_HELLO))
return DF_OUTPUT;
if (df_rpc_push(df->srv_slug, DF_RPC_INIT, length, rate, name ? name :
@@ -1109,7 +1109,7 @@
}
/* good to go -- do the server initialisation, if necessary */
- return df_init_server(length, rate, name, 1);
+ return df_init_server(length, rate, name);
}
const char *df_input_name(int shell_expand)
@@ -1501,7 +1501,7 @@
pthread_mutex_unlock(&df_rate_mx);
/* good to go -- do the server initialisation, if necessary */
- return df_init_server(length, rate, name, 0);
+ return df_init_server(length, rate, name);
}
df_abort_func_t df_on_abort(df_abort_func_t abort_func) {
@@ -2037,7 +2037,6 @@
int df_add_fragment(const char *name, int parent, unsigned long encoding,
unsigned long byte_sex, const char *prefix, const char *suffix)
{
- static int srv_ind = 0;
const int append = df_mode() & DF_MODE_APPEND;
int ind;
char *path, *ptr;
@@ -2049,7 +2048,7 @@
encoding, byte_sex, prefix, suffix);
if (ind)
return ind;
- return ++srv_ind; /* a fake fragment index */
+ return ++df->srv_ind; /* a fake fragment index */
}
/* make the containing directory */
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2014-05-03 00:06:52
|
Revision: 894
http://sourceforge.net/p/getdata/code/894
Author: ketiltrout
Date: 2014-05-03 00:06:48 +0000 (Sat, 03 May 2014)
Log Message:
-----------
Preliminary Getdata-0.9 support. Miscellaneous doc updates.
Modified Paths:
--------------
trunk/defile/Makefile.am
trunk/defile/bin/net.c
trunk/defile/bin/rc.c
trunk/defile/doc/Makefile.am
trunk/defile/doc/defile-ascii.1
trunk/defile/doc/defile-dirfile.1
trunk/defile/doc/defile-input.7
trunk/defile/doc/defile.1.in
trunk/defile/doc/df_add_entry.3
trunk/defile/doc/df_add_fragment.3
trunk/defile/doc/df_add_framedef.3
trunk/defile/doc/df_add_lut.3
trunk/defile/doc/df_init.3
trunk/defile/doc/df_push_frame.3
trunk/defile/doc/df_ready.3
trunk/defile/doc/df_shell_expand.3
trunk/defile/lib/internal.h
Added Paths:
-----------
trunk/defile/doc/df_field_in_output.3
Removed Paths:
-------------
trunk/defile/doc/df_field_in_input.3
Modified: trunk/defile/Makefile.am
===================================================================
--- trunk/defile/Makefile.am 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/Makefile.am 2014-05-03 00:06:48 UTC (rev 894)
@@ -18,7 +18,7 @@
# along with GetData; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
#
-AUTOMAKE_OPTIONS = foreign dist-bzip2 check-news 1.10
+AUTOMAKE_OPTIONS = foreign dist-bzip2 check-news 1.11
ACLOCAL_AMFLAGS = -I m4
Modified: trunk/defile/bin/net.c
===================================================================
--- trunk/defile/bin/net.c 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/bin/net.c 2014-05-03 00:06:48 UTC (rev 894)
@@ -468,7 +468,7 @@
}
break;
case DF_RPC_ADD_ENTRY:
- E.comp_scal = 1;
+ DF_SET_COMP_SCAL(E);
switch (pos) {
case 0:
DF_GET_SDATUM(E.field);
Modified: trunk/defile/bin/rc.c
===================================================================
--- trunk/defile/bin/rc.c 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/bin/rc.c 2014-05-03 00:06:48 UTC (rev 894)
@@ -360,8 +360,13 @@
if (rcd->name)
df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "defile-%s input plugin v%s\n",
rcd->name, rcd->version);
- else
+ else {
df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "%s\n", PACKAGE_STRING);
+#ifdef GD_GETDATA_VERSION
+ df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "using GetData %s\n",
+ GD_GETDATA_VERSION);
+#endif
+ }
if (rcd->copyright)
df_iprintf(DF_PRN_CLEAR, DF_PRN_NOTE, "%s\n\n", rcd->copyright);
Modified: trunk/defile/doc/Makefile.am
===================================================================
--- trunk/defile/doc/Makefile.am 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/Makefile.am 2014-05-03 00:06:48 UTC (rev 894)
@@ -25,7 +25,7 @@
man1_MANS = defile-ascii.1 defile-dirfile.1
man7_MANS = defile-input.7
man3_MANS = df_add_alias.3 df_add_entry.3 df_add_fragment.3 df_add_framedef.3 \
- df_add_lut.3 df_check_abort.3 df_exit.3 df_field_in_input.3 \
+ df_add_lut.3 df_check_abort.3 df_exit.3 df_field_in_output.3 \
df_hide.3 df_get_offset.3 df_include.3 df_init.3 df_input_name.3 \
df_mode.3 df_nframes_allowed.3 df_on_abort.3 df_push_frame.3 \
df_puts.3 df_ready.3 df_set_const.3 df_shell_expand.3 \
Modified: trunk/defile/doc/defile-ascii.1
===================================================================
--- trunk/defile/doc/defile-ascii.1 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/defile-ascii.1 2014-05-03 00:06:48 UTC (rev 894)
@@ -112,7 +112,7 @@
.BR strtod (3))
separated by a semicolon
.RI ( ; ),
-with no intervening whitespace. This is the GetData convension; see
+with no intervening whitespace. This is the Dirfile convension; see
dirfile-format(5). The first floating-point number is the value of the real
part of the complex number and the second is the value of the imaginary part.
@@ -131,7 +131,7 @@
.BR \-\-const\-row
and
.BR \-\-string\-row ,
-which are treated in the order they are presented to defile-ascii. For each
+which are handled in the order they are presented to defile-ascii. For each
header line, including the field names, the line is parsed in the same manner as
the rest of the data: either as fixed-width columns, or else with the same
delimiters specified. By default, these header data may appear as comment lines
Modified: trunk/defile/doc/defile-dirfile.1
===================================================================
--- trunk/defile/doc/defile-dirfile.1 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/defile-dirfile.1 2014-05-03 00:06:48 UTC (rev 894)
@@ -119,7 +119,7 @@
in the subdirectory
.I DIR
under the output dirfile base directory, instead of the default "<ext>". The
-directory specified must be relative. (Directive:
+directory may not be an absolute path. (Directive:
.BR ExtDir )
.TP
.BR \-f ", " \-\-flatten
@@ -164,7 +164,7 @@
in the subdirectory
.I DIR
under the output dirfile base directory, instead of the default "<lut>". The
-directory specified must be relative. (Directive:
+directory may not be an absolute path. (Directive:
.BR LUTDir )
.TP
.BR \-X ", " \-\-link\-extern\-frags
Modified: trunk/defile/doc/defile-input.7
===================================================================
--- trunk/defile/doc/defile-input.7 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/defile-input.7 2014-05-03 00:06:48 UTC (rev 894)
@@ -38,7 +38,7 @@
can be extended easily, without having to modify defile itself.
Defile provides a C API for input plugins. Functions provided by the API are
-described in individual manual pages in section 3 of the manual. A list of
+described in individual manual pages in section 3 of this Manual. A list of
these is given below in the
.B API REFERENCE
section. The API is defined in the C header file <defile.h>. This document
@@ -48,14 +48,13 @@
.B DEFILE OPERATION
below.
-An input plugin is a dynamically-loaded shared library which asynchronously
-reads data from a data source and passes it on to defile for output as a
-dirfile. Each input plugin is identified by it's type name. The type name
-identifies the type of data the input plugin is capable of reading. The type
-name should consist of one or more lower case letters, numbers, or the
-underscore (_) character. The type name must be unique among all defile input
-plugins: defile will only ever consider the first input plugin with a given type
-name that it finds.
+An input plugin is a dynamically-loaded library which asynchronously reads data
+from a data source and passes it on to defile for output as a dirfile. Each
+input plugin is identified by it's type name. The type name identifies the type
+of data the input plugin is capable of reading. The type name should consist of
+one or more lower case letters, numbers, or the underscore (_) character. The
+type name must be unique among all defile input plugins: defile will only ever
+consider the first input plugin with a given type name that it finds.
The name of the shared library must be of the form:
.IP
@@ -66,7 +65,7 @@
where
.I anything
is optional and may contain any valid filename characters (the host platform
-may require a certain prefix for native dynamically loadable modules),
+may require a certain prefix for native dynamically-loadable modules),
.B defile-
is literaly "defile" followed by a dash,
.I type
@@ -74,7 +73,7 @@
.I -anything
contains more optional characters, separated from the type name by a dash, and
.I .ext
-is a file extension appropriate for native dynamically loadable modules on the
+is a file extension appropriate for native dynamically-loadable modules on the
host platform, if any.
.PP
The type name is also the value of the
@@ -151,7 +150,7 @@
section below.
.TP
.I licence
-indicates the licence type of the plugin. It affects what license boilerplate
+indicates the licence type of the plugin. It affects which license boilerplate
(if any) is printed along with version information. See the
.B HELP AND VERSION OUTPUT
section below for details.
@@ -289,7 +288,7 @@
.TP
.I argname
the placeholder name of the argument used in usage output. For stylistic
-consitency, it is recommended this consist of upper case letters only. If the
+consitency, it is suggested this consist of upper case letters only. If the
option is of type
.BR DF_OPT_NO_ARG ,
this value is ignored and may be NULL. Otherwise, if this is NULL, a default
@@ -300,7 +299,7 @@
usage output (see the
.B HELP AND VERSION OUTPUT
section below). It should contain no newlines; they will be inserted where
-appropriate by defile. For stylistic consistency, it is recommended this begin
+appropriate by defile. For stylistic consistency, it is suggested this begin
with a lower case letter, and have independent clauses separated by semicolons.
If this option record specifies neither
.I sopt
@@ -429,18 +428,18 @@
The input plugin's role in each state is discussed below.
The input plugin's main loop may return at any time if it's job is complete,
-or if it ran into an unrecoverable error. Returning a value of zero
+or if it ran into an irrecoverable error. Returning a value of zero
indicates to defile that the input plugin completed normally (success).
Returning any other value indicates that the input plugin encountered an
-unrecoverable error (failure).
+irrecoverable error (failure).
.SS START State
Defile begins in the
.B START
-state. In this state, defile starts by processing the defile options. It
-then determines the input plugin type to use, either as a result of an explicit
-type specified by the user, or else by using automatic type detection on the
+state. In this state, defile first processes the defile options. It then
+determines the input plugin type to use, either as a result of an explicit type
+specified by the user, or else by using automatic type detection on the
specified input (see
.B AUTOMATIC TYPE DETECTION
for information on how to get an input plugin to participate in this).
@@ -471,7 +470,9 @@
options and input directives.
.TP
.I value
-an array of integers of length
+an array of
+.IR int s
+of length
.I n_config
containing the values (see
.B INPUT OPTIONS
@@ -511,11 +512,9 @@
The input plugin is required to define at least one subframe (via the
.BR df_add_framedef (3)
call) which describes the raw data in the data objects which the input plugin
-uses to communicate with defile. It may also add derived channels to the output
-dirfile using
+uses to communicate with defile. It may also add derived channels and other objects to the output dirfile using
.BR df_add_entry (3)
-and define the rest of the output dirfile metadata.
-(See also the
+(or similar) and define the rest of the output dirfile metadata. (See also the
.B DEFILE DATA MODEL
section below for an overview of how defile expects data to be marshalled.)
This is also the earliest time at which the input plugin is able to determine
@@ -589,13 +588,14 @@
Defile enters the
.B ABORT
-state in cases when an unrecoverable error occurs during operation, or else when
+state in cases when an irrecoverable error occurs during operation, or else when
it receives an asynchronous request to cancel (for example a keyboard
interrupt). Most of the input plugin API functions check whether defile has
entered this state and return
.B DF_ABORT
when detected (see
-.BR "Standard API Return Semaphores" ).
+.B Standard API Return Semaphores
+below).
The input should either monitor the return value of these functions to
detect an unexpected transition into the
.B ABORT
@@ -694,7 +694,7 @@
The most common reason for this is the input plugin spending too long in the
.B ABORT
state, but it may happen in other exceptional circumstances. Input plugins which
-disable cancelability for long periods of time may be subject to more drastic
+disable cancellability for long periods of time may be subject to more drastic
measures without forewarning.
.SS Console I/O
@@ -784,7 +784,7 @@
A careful trade-off must be made between false negatives (returning zero in
cases where the input designator really does indicate something readable by the
input plugin) and false positives (returning non-zero in cases where the input
-designator does not, after all, indicate something readable by the plugin.
+designator does not, after all, indicate something readable by the plugin).
False negatives cause defile to be unable to automatically find a suitable
input plugin; excessive false positives can lead to multiple input plugins
claiming the input (resulting in neither being automatically given it).
@@ -798,7 +798,7 @@
The following functions (some listed multiple times) are provided by the defile
C API, grouped here by sequencing restrictions. See the indicated manual pages
-for details.
+for details on the functions themselves.
.TP
Functions which may be called at any time, regardless of current state:
Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/defile.1.in 2014-05-03 00:06:48 UTC (rev 894)
@@ -137,7 +137,7 @@
.BR +t ", " \-\-autotype
ignore any previous input type specified. If no subsequent
.B \-t
-occurs, defile with attempt to automatically determine the input type. This
+occurs, defile will attempt to automatically determine the input type. This
is the default behaviour. Defile attempts to automatically determine the
appropriate input plugin for the
.I INPUT
@@ -155,11 +155,11 @@
.SS Ouptut Location Options
The path of the created dirfile is controlled by
-.B \-\-dirfile
+.B \-\-directory
and
.BR \-\-output ,
at least one of which must be given, unless running as a server. If
-.B \-\-dirfile
+.B \-\-directory
is used without
.BR \-\-output ,
the name of the output dirfile will depend on the
@@ -424,8 +424,8 @@
.I HOST
(with the username
.IR USER ,
-if given) and transfer data from it using a remote defile. Change the (local)
-program used to log into the remote host, see
+if given) and transfer data from it using a remote defile. To change the
+(local) program used to log into the remote host, see
.BR \-\-rsh ;
to change the program executed on the remote host, see
.BR \-\-remote\-command .
@@ -483,7 +483,7 @@
Use
.I COMMAND
as the remote shell used to log into the remote host. By default this is "ssh" (see
-.BR ssh(1)),
+.BR ssh (1)),
and
.I COMMAND
must be "ssh compatible" in the sense that the second last argument passed to
@@ -531,7 +531,7 @@
.BR \-\-append ", " \-\-directory ", " \-\-output ", and " \-\-async .
Any output field list specified, as well as
.BR \-\-clobber ,
-are ignored. (Directive:
+are also ignored. (Directive:
.BR Listen )
.TP
.BR +L ", " \-\-no\-listen
@@ -934,8 +934,7 @@
the default suite of defile input plugins.
.SH ENVIRONMENT
-Defile never modifies the environment, but may consider the following
-environmental variables, if present:
+Defile considers the following environmental variables, if present:
.TP
.B DEFILE_PLUGIN_PATH
Modified: trunk/defile/doc/df_add_entry.3
===================================================================
--- trunk/defile/doc/df_add_entry.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_add_entry.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -73,7 +73,7 @@
If the field has been excluded from the output dirfile by the Output Field List
(see
.BR defile (1)),
-these functions do nothing and succeeds.
+these functions do nothing and succeed.
.SH RETURN VALUE
Upon success,
.BR df_add_entry ()
Modified: trunk/defile/doc/df_add_fragment.3
===================================================================
--- trunk/defile/doc/df_add_fragment.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_add_fragment.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -44,8 +44,8 @@
may contain path elements relative to the base output dirfile directory.
Subdirectories specified in
.I name
-will be created if they do not already exist. The new fragment will be included
-in the fragment indexed by
+will be created in the output dirfile if they do not already exist. The new
+fragment will be included in the fragment indexed by
.IR fragment ,
which should be zero (for the primary format file) or else a fragment index
returned by a previous call to
Modified: trunk/defile/doc/df_add_framedef.3
===================================================================
--- trunk/defile/doc/df_add_framedef.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_add_framedef.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -135,8 +135,8 @@
output in the order defined in the subframe definition, which may be different
than their ordering within the subframe itself.
.PP
-Specifications for field which have not appeared in any previous subframe
-definition will be added to the output fragment indexed by
+Fields which have not appeared in any previous subframe definition will be
+added to the output fragment indexed by
.IR fragment ,
which should be either zero (for the primary fragment) or else a fragment index
returned by
@@ -147,8 +147,8 @@
subframe definition, and need not be the same as the fragment in which the
field is actually defined.
.PP
-Fields which have been excluded from the output dirfile by the Output Field List
-(see
+Fields which have been excluded from the output dirfile by the user via the
+Output Field List (see
.BR defile (1))
are accepted in subframe definitions and simply ignored.
.SH RETURN VALUE
Modified: trunk/defile/doc/df_add_lut.3
===================================================================
--- trunk/defile/doc/df_add_lut.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_add_lut.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -88,6 +88,7 @@
or
.I y_type
was invalid.
+.TP
.B DF_OUTPUT
An error occured attempting to create the LUT. See
.BR mkdir (2),
Deleted: trunk/defile/doc/df_field_in_input.3
===================================================================
--- trunk/defile/doc/df_field_in_input.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_field_in_input.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -1,63 +0,0 @@
-.\" df_field_in_output.3. The df_field_in_output man page.
-.\"
-.\" Copyright (C) 2012 D. V. Wiebe
-.\"
-.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
-.\"
-.\" This file is part of the Defile package.
-.\"
-.\" Permission is granted to copy, distribute and/or modify this document
-.\" under the terms of the GNU Free Documentation License, Version 1.2 or
-.\" any later version published by the Free Software Foundation; with no
-.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
-.\" Texts. A copy of the license is included in the `COPYING.DOC' file
-.\" as part of this distribution.
-.\"
-.TH df_field_in_output 3 "27 February 2012" "Version 0.0.1" DEFILE
-.SH NAME
-df_field_in_output \(em check whether defile is writing a field to its output
-.SH SYNOPSIS
-.nh
-.ad l
-.fam C
-.B #include <defile.h>
-.HP
-.BI "int df_field_in_output(const char *" name );
-.fam
-.ad
-.hy
-.SH DESCRIPTION
-This function is part of the defile input plugin C API. See
-.BR defile-input (7)
-for an overview.
-.PP
-The
-.BR df_field_in_output ()
-function consults defile's Output Field List (see
-.BR defile (1)),
-and returns non-zero if the field specified by
-.I name
-would be writen to the output dirfile were it to be defined at some point by
-the input plugin. If the field has been excluded from the output dirfile, this
-function will return zero. Note: this function only checks for the potential
-for exclusion or inclusion: the returned value is not affected by whether a
-field called
-.I name
-actually has been (or will be) defined.
-.PP
-It is not required that an input plugin self-exclude fields not in the output
-dirfile by checking this function. Both included and excluded fields may
-be defined by the input plugin. Excluded fields will be silently ignored. This
-function is provided for use in rare cases where knowing a field is to be
-excluded may speed or simplify the input plugin's operation.
-.SH RETURN VALUE
-The
-.BR df_field_in_output ()
-function returns non-zero if a field called
-.I name
-would be included in the output dirfile if provided. It returns zero if a field
-with such a name would be excluded from the output.
-.SH SEE ALSO
-.BR defile-input (7),
-.BR defile (1),
-.BR df_add_framedef (3).
Copied: trunk/defile/doc/df_field_in_output.3 (from rev 875, trunk/defile/doc/df_field_in_input.3)
===================================================================
--- trunk/defile/doc/df_field_in_output.3 (rev 0)
+++ trunk/defile/doc/df_field_in_output.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -0,0 +1,63 @@
+.\" df_field_in_output.3. The df_field_in_output man page.
+.\"
+.\" Copyright (C) 2012 D. V. Wiebe
+.\"
+.\""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
+.\"
+.\" This file is part of the Defile package.
+.\"
+.\" Permission is granted to copy, distribute and/or modify this document
+.\" under the terms of the GNU Free Documentation License, Version 1.2 or
+.\" any later version published by the Free Software Foundation; with no
+.\" Invariant Sections, with no Front-Cover Texts, and with no Back-Cover
+.\" Texts. A copy of the license is included in the `COPYING.DOC' file
+.\" as part of this distribution.
+.\"
+.TH df_field_in_output 3 "27 February 2012" "Version 0.0.1" DEFILE
+.SH NAME
+df_field_in_output \(em check whether defile is writing a field to its output
+.SH SYNOPSIS
+.nh
+.ad l
+.fam C
+.B #include <defile.h>
+.HP
+.BI "int df_field_in_output(const char *" name );
+.fam
+.ad
+.hy
+.SH DESCRIPTION
+This function is part of the defile input plugin C API. See
+.BR defile-input (7)
+for an overview.
+.PP
+The
+.BR df_field_in_output ()
+function consults the Output Field List (see
+.BR defile (1))
+specified by the user, and returns non-zero if the field specified by
+.I name
+would be writen to the output dirfile were it to be defined at some point by
+the input plugin. If the field has been excluded from the output dirfile, this
+function returns zero. Note: this function only checks for the potential for
+exclusion or inclusion: the returned value is not affected by whether a field
+called
+.I name
+actually has been (or will be) defined.
+.PP
+It is not required that an input plugin self-exclude fields not in the output
+dirfile by checking this function. Both included and excluded fields may
+be defined by the input plugin. Excluded fields will be silently dropped by
+defile. This function is provided for use in rare cases where knowing a field
+is to be excluded may speed up or simplify the input plugin's operation.
+.SH RETURN VALUE
+The
+.BR df_field_in_output ()
+function returns non-zero if a field called
+.I name
+would be included in the output dirfile, if provided. It returns zero if a
+field with such a name would be excluded from the output.
+.SH SEE ALSO
+.BR defile-input (7),
+.BR defile (1),
+.BR df_add_framedef (3).
Modified: trunk/defile/doc/df_init.3
===================================================================
--- trunk/defile/doc/df_init.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_init.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -57,7 +57,7 @@
.BR df_update_length (3).
.TP
.I rate
-the expected frame rate for live data in follow mode (see
+the expected frame rate (in frames per second) for live data in follow mode (see
.BR defile-input (7)).
If unknown or difficult to calculate, this may be -1. This value is used to
prime the IIR filter in the rate calculator in the running information provided
@@ -120,7 +120,7 @@
case, if the input plugin is able, it may try calling the function again,
potentially with a different
.I name
-argument. In case of an irrecoverable error, a subsequent call will result in
+argument. In case of an irrecoverable error, a subsequent call will return
.BR DF_ABORT .
.IP
This will
@@ -128,8 +128,8 @@
be returned by
.BR df_reinit ()
in cases where the user has specified an explicit output dirfile name
-(--output) but not allowed overwriting of the output (--clobber), since in this
-case, both the new and old dirfiles have the same user-specified name (ie.
+(--output) but not allowed overwriting of the output (--no-clobber), since in
+this case, both the new and old dirfiles have the same, user-specified name (ie.
.I name
is ignored), and defile will refuse to truncate the dirfile it just created.
Conversely, if the user has specified both --output and --clobber, calling
Modified: trunk/defile/doc/df_push_frame.3
===================================================================
--- trunk/defile/doc/df_push_frame.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_push_frame.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -36,7 +36,7 @@
.BR df_push_frame ()
function attempts to place
.I nframes
-frames of data into the
+subframes of data into the
.BR defile (1)
buffer for conversion to the output dirfile. The
.I data
@@ -48,11 +48,11 @@
.BR gd_add_framedef (3)
when the corresponding subframe was defined.
.PP
-If
+The
.I block
-is one,
-this call will block until all data has been transferred. Otherwise, depending
-on how full defile's buffer is, fewer than
+parameter should be zero or one. If it is one, this call will block until all
+data has been transferred. Otherwise, depending on how full defile's buffer is,
+fewer than
.I nframes
frames of data may actually be buffered. In synchronous mode, care should be
taken with blocking calls to this function to avoid deadlocks. See
Modified: trunk/defile/doc/df_ready.3
===================================================================
--- trunk/defile/doc/df_ready.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_ready.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -39,8 +39,8 @@
dirfile and is ready to start providing data for conversion. (That is, it
signals a transition from the BUILD state into the RUN state; see
.BR defile-input (7).)
-If the input plugin wishes to set a reference field, it may do so now by passing
-the field's name as
+If the input plugin wishes to set a reference field, it should do so now by
+passing the field's name as
.IR reference .
If this is NULL, defile will choose a reference field for the output dirfile.
.PP
@@ -67,7 +67,8 @@
.B DF_INPUT
The field specified as
.I reference
-doesn't appear in any of the previously defined subframe definitions.
+doesn't appear as a raw field in any of the previously defined subframe
+definitions.
.TP
.B DF_SEQUENCE
The function was called while defile was not in the
Modified: trunk/defile/doc/df_shell_expand.3
===================================================================
--- trunk/defile/doc/df_shell_expand.3 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/doc/df_shell_expand.3 2014-05-03 00:06:48 UTC (rev 894)
@@ -35,8 +35,8 @@
.BR df_shell_expand ()
function returns the result of performing shell-like word expansion on the input
.IR string .
-The result is returned in as a new string, which is allocated by
-.BR malloc (3),
+The result is returned in a new string allocated by
+.BR malloc (3)
and must be freed by the caller when no longer needed. For details of
shell-like word expansion, see
.BR wordexp (3).
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2014-05-01 04:07:20 UTC (rev 893)
+++ trunk/defile/lib/internal.h 2014-05-03 00:06:48 UTC (rev 894)
@@ -181,6 +181,17 @@
#define DF_OOM(fac) \
df_iprintf(fac, DF_PRN_ERR, "out of memory at [%s,%i]", __FILE__, __LINE__)
+/* GetData version abstraction */
+#ifndef GD_GETDATA_INT_VERSION
+#define GD_GETDATA_INT_VERSION 804 /* Minimum supported version */
+#endif
+
+#if GD_GETDATA_INT_VERSION < 900
+#define DF_SET_COMP_SCAL(E) (E).comp_scal = 1
+#else
+#define DF_SET_COMP_SCAL(E) (E).flags |= GD_EN_COMPSCAL
+#endif
+
/* NList */
struct df_subset_ {
int n; /* # of elements */
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
|
From: <ket...@us...> - 2014-09-30 02:32:55
|
Revision: 922
http://sourceforge.net/p/getdata/code/922
Author: ketiltrout
Date: 2014-09-30 02:32:45 +0000 (Tue, 30 Sep 2014)
Log Message:
-----------
Output frameoffset.
Modified Paths:
--------------
trunk/defile/bin/defile.c
trunk/defile/doc/defile.1.in
trunk/defile/lib/internal.h
trunk/defile/lib/libdefile.c
Modified: trunk/defile/bin/defile.c
===================================================================
--- trunk/defile/bin/defile.c 2014-07-30 20:00:59 UTC (rev 921)
+++ trunk/defile/bin/defile.c 2014-09-30 02:32:45 UTC (rev 922)
@@ -943,7 +943,7 @@
DF_WOPT_VERBOSE, DF_WOPT_REMOTE, DF_WOPT_LOCAL, DF_WOPT_RSH, DF_WOPT_LISTEN,
DF_WOPT_NOLISTEN, DF_WOPT_RCMD, DF_WOPT_RDEBUG, DF_WOPT_NORDEBUG,
DF_WOPT_SRVRATE, DF_WOPT_DAEMON, DF_WOPT_CURFILE, DF_WOPT_SYMLINK,
- DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE, DF_WOPT_CLOBBERONE
+ DF_WOPT_NOSYMLINK, DF_WOPT_NOCURFILE, DF_WOPT_CLOBBERONE, DF_WOPT_FOFFS
};
static int DF_OutputOpt(struct df_defile *df, struct df_rc_config *config,
@@ -1049,10 +1049,19 @@
~(DF_MODE_RDEBUG_I | DF_MODE_RDEBUG_O | DF_MODE_RDEBUG_T);
break;
+ case DF_WOPT_FOFFS: /* --output-offset */
+ /* a sign means relative adjustment */
+ if (config->argument[0] != '-' && config->argument[0] != '+')
+ df->output_adjust_set = 1;
+ else
+ df->output_adjust_set = 0;
+ df->output_adjust = strtoll(config->argument, NULL, 0);
+ break;
+
SETMODE(APPEND, DF_MODE_APPEND);
CLRMODE(NOAPPEND, DF_MODE_APPEND);
@@ -2537,6 +2546,10 @@
{ DF_WOPT_OUTPUT, DF_OPT_EXPAND | DF_OPT_ARG_RQ, 'o', "output", "Output",
"PATH", "write data to a dirfile called PATH"
},
+ { DF_WOPT_FOFFS, DF_OPT_ARG_RQ, 'O', "offset-adjust", "OffsetAdjust",
+ "ADJUST", "adjust the frame offset of the output dirfile by ADJUST "
+ "(default: +0)"
+ },
{ DF_WOPT_QUIET, DF_OPT_NO_ARG, 'q', "quiet", "Quiet", NULL,
"be less verbose"
},
Modified: trunk/defile/doc/defile.1.in
===================================================================
--- trunk/defile/doc/defile.1.in 2014-07-30 20:00:59 UTC (rev 921)
+++ trunk/defile/doc/defile.1.in 2014-09-30 02:32:45 UTC (rev 922)
@@ -295,6 +295,78 @@
Don't create a curfile. This is the default behaviour. (Directive:
.BR NoCurFile )
+.SS Frame Selection Options
+The following options affect the starting frame number for input and output.
+.TP
+.BR \-s ", " \-\-skip=\fINUM
+Skip the first
+.I NUM
+frames of the input; equivalently: request the input start with the frame
+numbered
+.IR NUM .
+Even when this option is specified, defile may start on a later frame, if the
+first frame available in the input is greater than
+.IR NUM .
+Only the last
+.B \-\-skip
+specified is honoured. The default skip is zero (i.e. start with the first
+frame).
+
+Using
+.B \-\-append
+(q.v.) to append to a pre-existing output dirfile has the effect of specifying
+an implicit
+.B \-\-skip
+with
+.I NUM
+equal to the number of frames currently in the output dirfile. A
+.B \-\-skip
+which specifies a smaller number is ignored, but this option can be used to
+advance the starting frame number further (leaving a hole in the outpu
+dirfile). (Directive:
+.BR Skip )
+
+.TP
+.BR \-O ", " \-\-offset-adjust=\fIADJUST
+When not in
+.I Append Mode\fR,
+defile will, by default, set the frame offset of the output dirfile to the
+frame number of the first converted frame (which may have been adjusted with the
+.B \-\-skip
+option). This option permits modification of the output dirfile's frame offset.
+If the
+.I ADJUST
+parameter is starts with a plus (+) or minus (\-) sign, the value is taken as a
+relative adjustment to this default frame offset. If the relative offset would
+result in a negative frame offset, the output frame offset is set to zero. If
+.I ADJUST
+is a number with neither sign, the frame offset is set to that value. So:
+.RS
+.IP
+.nh
+.ad l
+\-\-offset\-adjust=100
+.hy
+.ad n
+.RE
+.IP
+sets the output dirfile's frame offset to 100, regardless of the actual starting
+frame, while:
+.RS
+.IP
+.nh
+.ad l
+\-\-offset\-adjust=+100
+.hy
+.ad n
+.RE
+.IP
+adds 100 to the default frame offset. The default is, effectively, "+0" (i.e.,
+no adjustment). In
+.I Append Mode\fR,
+this option is ignored. (Directive:
+.BR OutputAdjust )
+
.SS Running Mode Options
The following options affect the modes in which defile runs.
.TP
Modified: trunk/defile/lib/internal.h
===================================================================
--- trunk/defile/lib/internal.h 2014-07-30 20:00:59 UTC (rev 921)
+++ trunk/defile/lib/internal.h 2014-09-30 02:32:45 UTC (rev 922)
@@ -296,6 +296,8 @@
struct df_config config;
long long offset;
long long input_offset;
+ long long output_adjust;
+ int output_adjust_set;
unsigned long buflen;
unsigned int mode_flags;
Modified: trunk/defile/lib/libdefile.c
===================================================================
--- trunk/defile/lib/libdefile.c 2014-07-30 20:00:59 UTC (rev 921)
+++ trunk/defile/lib/libdefile.c 2014-09-30 02:32:45 UTC (rev 922)
@@ -1228,11 +1228,21 @@
pthread_mutex_lock(&df_config_mx);
offset = (df->input_offset > df->offset) ? df->input_offset : df->offset;
pthread_mutex_unlock(&df_config_mx);
+
+ /* output frameoffset */
+ if (!append) {
+ long long output_offset = offset;
+ if (df->output_adjust_set)
+ output_offset = df->output_adjust;
+ else if (output_offset < -df->output_adjust) /* underflow */
+ output_offset = 0;
+ else
+ output_offset += df->output_adjust;
+ if (output_offset)
+ gd_alter_frameoffset64(df->D, output_offset, GD_ALL_FRAGMENTS, 0);
+ }
+
if (offset > 0) {
- /* frameoffset */
- if (!append)
- gd_alter_frameoffset(df->D, offset, GD_ALL_FRAGMENTS, 0);
-
/* seek to the start frame */
for (i = 0; i < df->nraw; ++i)
if (gd_seek64(df->D, df->raw[i].name, offset, 0,
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|
Thanks for helping keep SourceForge clean.
X