Menu
▾
▴
[Octave-cvsupdate] SF.net SVN: octave:[7035] trunk/octave-forge/main/general/inst
|
From: <hi...@us...> - 2010-03-12 09:00:56
|
Revision: 7035
http://octave.svn.sourceforge.net/octave/?rev=7035&view=rev
Author: highegg
Date: 2010-03-12 09:00:48 +0000 (Fri, 12 Mar 2010)
Log Message:
-----------
major modifications in parcellfun/parrarrayfun; support ChunksPerProc and Vectorized options
Modified Paths:
--------------
trunk/octave-forge/main/general/inst/pararrayfun.m
trunk/octave-forge/main/general/inst/parcellfun.m
Added Paths:
-----------
trunk/octave-forge/main/general/inst/private/
trunk/octave-forge/main/general/inst/private/chunk_parcellfun.m
trunk/octave-forge/main/general/inst/private/parcellfun_opts.m
Modified: trunk/octave-forge/main/general/inst/pararrayfun.m
===================================================================
--- trunk/octave-forge/main/general/inst/pararrayfun.m 2010-03-12 05:10:15 UTC (rev 7034)
+++ trunk/octave-forge/main/general/inst/pararrayfun.m 2010-03-12 09:00:48 UTC (rev 7035)
@@ -26,6 +26,12 @@
## Argument and options handling is analogical to @code{parcellfun}, except that
## arguments are arrays rather than cells. If cells occur as arguments, they are treated
## as arrays of singleton cells.
+## Arrayfun supports one extra option compared to parcellfun: "Vectorized".
+## This option must be given together with "ChunksPerProc" and it indicates
+## that @var{fun} is able to operate on vectors rather than just scalars, and returns
+## a vector. The same must be true for @var{errfunc}, if given.
+## In this case, the array is split into chunks which are then directly served to @var{func}
+## for evaluation, and the results are concatenated to output arrays.
## @seealso{parcellfun, arrayfun}
## @end deftypefn
@@ -35,27 +41,34 @@
print_usage ();
endif
- nargs = length (varargin);
+ [nargs, uniform_output, error_handler, ...
+ verbose_level, chunks_per_proc, vectorized] = parcellfun_opts (varargin);
- recognized_opts = {"UniformOutput", "ErrorHandler", "VerboseLevel"};
-
- while (nargs >= 2)
- maybeopt = varargin{nargs-1};
- if (ischar (maybeopt) && any (strcmpi (maybeopt, recognized_opts)))
- nargs -= 2;
- else
- break;
- endif
- endwhile
-
args = varargin(1:nargs);
opts = varargin(nargs+1:end);
+ if (nargs == 0)
+ print_usage ();
+ elseif (nargs > 1)
+ [err, args{:}] = common_size (args{:});
+ if (err)
+ error ("pararrayfun: arguments size must match");
+ endif
+ endif
- args = cellfun (@num2cell, args, "UniformOutput", false,
- "ErrorHandler", @arg_class_error);
+ njobs = numel (args{1});
- [varargout{1:nargout}] = parcellfun (nproc, func, args{:}, opts{:});
+ if (vectorized && chunks_per_proc > 0 && chunks_per_proc < njobs / nproc)
+ ## If "Vectorized" is on, we apply the function directly on chunks of
+ ## arrays.
+ [varargout{1:nargout}] = chunk_parcellfun (nproc, chunks_per_proc, ...
+ func, error_handler, verbose_level, args{:});
+ else
+ args = cellfun (@num2cell, args, "UniformOutput", false,
+ "ErrorHandler", @arg_class_error);
+ [varargout{1:nargout}] = parcellfun (nproc, func, args{:}, opts{:});
+ endif
+
endfunction
function arg_class_error (S, X)
Modified: trunk/octave-forge/main/general/inst/parcellfun.m
===================================================================
--- trunk/octave-forge/main/general/inst/parcellfun.m 2010-03-12 05:10:15 UTC (rev 7034)
+++ trunk/octave-forge/main/general/inst/parcellfun.m 2010-03-12 09:00:48 UTC (rev 7035)
@@ -21,6 +21,7 @@
## @deftypefn{Function File} {[@var{o1}, @var{o2}, @dots{}] =} parcellfun (@var{nproc}, @var{fun}, @var{a1}, @var{a2}, @dots{})
## @deftypefnx{Function File} {} parcellfun (nproc, fun, @dots{}, "UniformOutput", @var{val})
## @deftypefnx{Function File} {} parcellfun (nproc, fun, @dots{}, "ErrorHandler", @var{errfunc})
+## @deftypefnx{Function File} {} parcellfun (nproc, fun, @dots{}, "ChunksPerProc", @var{val})
## Evaluates a function for multiple argument sets using multiple processes.
## @var{nproc} should specify the number of processes. A maximum recommended value is
## equal to number of CPUs on your machine or one less.
@@ -33,6 +34,9 @@
## A VerboseLevel option controlling the level output is supported.
## A value of 0 is quiet, 1 is normal, and 2 or more enables
## debugging output.
+## The ChunksPerProc option control the number of chunks which contains elementary jobs. This
+## option particularly useful when time execution of function is small. Setting this option
+## to 100 is a good choice in most cases.
##
## NOTE: this function is implemented using "fork" and a number of pipes for IPC.
## Suitable for systems with an efficient "fork" implementation (such as GNU/Linux),
@@ -60,46 +64,41 @@
error ("parcellfun: fun must be either a function handle or name")
endif
- uniform_output = true;
- error_handler = [];
- verbose_level = 1; # default to normal output level
+ [nargs, uniform_output, error_handler, ...
+ verbose_level, chunks_per_proc] = parcellfun_opts (varargin);
- args = varargin;
- nargs = length (varargin);
-
- ## parse options
- if (nargs > 1)
- do
- if (strcmp (args{nargs-1}, "UniformOutput"))
- uniform_output = args{nargs};
- nargs -= 2;
- continue;
- endif
- if (strcmp (args{nargs-1}, "ErrorHandler"))
- error_handler = args{nargs};
- nargs -= 2;
- continue;
- endif
- if (strcmp (args{nargs-1}, "VerboseLevel"))
- verbose_level = args{nargs};
- nargs -= 2;
- continue;
- endif
- break;
- until (nargs < 2);
+ args = varargin(1:nargs);
+ if (! all (cellfun ("isclass", args, "cell")))
+ error ("parcellfun: all non-option arguments except the first one must be cell arrays");
endif
-
- args = args(1:nargs);
-
- if (length (args) == 0)
+
+ if (nargs == 0)
print_usage ();
- elseif (length (args) > 1)
+ elseif (nargs > 1)
[err, args{:}] = common_size (args{:});
if (err)
error ("parcellfun: arguments size must match");
endif
endif
+ njobs = numel (args{1});
+
+ if (chunks_per_proc > 0 && chunks_per_proc < njobs / nproc)
+ ## We need chunked evaluation.
+
+ ## Function executed for a chunk.
+ if (isempty (error_handler))
+ chunk_fun = @(varargin) cellfun (fun, varargin{:}, "UniformOutput", uniform_output);
+ else
+ chunk_fun = @(varargin) cellfun (fun, varargin{:}, ...
+ "UniformOutput", uniform_output, "ErrorHandler", error_handler);
+ endif
+
+ [varargout{1:nargout}] = chunk_parcellfun (nproc, chunks_per_proc, ...
+ chunk_fun, [], verbose_level, args{:});
+ return
+ endif
+
nproc = min (nproc, numel (args{1}));
## create communication pipes.
@@ -270,7 +269,6 @@
else
## parent process.
- njobs = numel (varargin{1});
res = cell (nargout, njobs);
pjobs = 0;
@@ -316,7 +314,7 @@
fwrite (cmdw(isubp), 0, "double");
fclose (cmdw(isubp));
endif
- if( verbose_level > 0 )
+ if (verbose_level > 0)
fprintf (stderr, "\rparcellfun: %d/%d jobs done", pjobs - sum (pending != 0), njobs);
fflush (stderr);
endif
Added: trunk/octave-forge/main/general/inst/private/chunk_parcellfun.m
===================================================================
--- trunk/octave-forge/main/general/inst/private/chunk_parcellfun.m (rev 0)
+++ trunk/octave-forge/main/general/inst/private/chunk_parcellfun.m 2010-03-12 09:00:48 UTC (rev 7035)
@@ -0,0 +1,54 @@
+## Copyright (C) 2010 VZLU Prague, a.s., Czech Republic
+## Copyright (C) 2010 Jean-Benoist Leger <jb...@jb...>
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; see the file COPYING. If not, see
+## <http://www.gnu.org/licenses/>.
+
+## -*- texinfo -*-
+## @deftypefn{Function File} {} chunk_parcellfun (@dots{:})
+## Undocumented internal function.
+## @end deftypefn
+
+function varargout = chunk_parcellfun (nproc, chunks_per_proc, func,
+ error_handler, verbose_level, varargin)
+
+ args = varargin;
+
+ nchunks = chunks_per_proc * nproc;
+
+ ## Compute optimal chunk sizes.
+ N = numel (args{1});
+ len_chunk = ceil (N/nchunks);
+ chunk_sizes = len_chunk (ones(1, nchunks));
+ chunk_sizes(1:nchunks*len_chunk - N) -= 1;
+
+ ## Split argument arrays into chunks (thus making arrays of arrays).
+ chunked_args = cellfun (@(arg) mat2cell (arg(:), chunk_sizes), args, ...
+ "UniformOutput", false);
+
+ ## Attach error handler if present.
+ if (! isempty (error_handler))
+ chunked_args = [chunked_args, {"ErrorHandler", error_handler}];
+ endif
+
+ ## Main call.
+ [out_brut{1:nargout}] = parcellfun (nproc, func, chunked_args{:}, ...
+ "UniformOutput", false, "VerboseLevel", verbose_level);
+
+ ## Concatenate output args and reshape them to correct size.
+ true_size = size (args{1});
+ varargout = cellfun (@(arg) reshape (vertcat (arg{:}), true_size), out_brut, "UniformOutput", false);
+
+endfunction
+
Added: trunk/octave-forge/main/general/inst/private/parcellfun_opts.m
===================================================================
--- trunk/octave-forge/main/general/inst/private/parcellfun_opts.m (rev 0)
+++ trunk/octave-forge/main/general/inst/private/parcellfun_opts.m 2010-03-12 09:00:48 UTC (rev 7035)
@@ -0,0 +1,78 @@
+## Copyright (C) 2010 VZLU Prague, a.s., Czech Republic
+##
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 3 of the License, or
+## (at your option) any later version.
+##
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU General Public License
+## along with this program; see the file COPYING. If not, see
+## <http://www.gnu.org/licenses/>.
+
+## -*- texinfo -*-
+## @deftypefn{Function File} {} parcellfun_opts (args)
+## Undocumented internal function.
+## @end deftypefn
+
+function [nargs, uniform_output, error_handler, ...
+ verbose_level, chunks_per_proc, vectorized] = parcellfun_opts (args)
+
+ uniform_output = true;
+ error_handler = [];
+ verbose_level = 1; # default to normal output level
+ chunks_per_proc = 0; # 0 means than size of chunk is 1
+ vectorized = false;
+
+ nargs = length (args);
+
+ ## parse options
+ while (nargs > 1)
+ opt = args{nargs-1};
+ if (! ischar (opt))
+ break;
+ else
+ opt = tolower (opt);
+ val = args{nargs};
+ endif
+ switch (opt)
+ case "uniformoutput"
+ uniform_output = logical (val);
+ if (! isscalar (uniform_output))
+ error ("parcellfun: UniformOutput must be a logical scalar");
+ endif
+ case "errorhandler"
+ error_handler = val;
+ if (! isa (error_handler, "function_handle"))
+ error ("parcellfun: ErrorHandler must be a function handle");
+ endif
+ case "verboselevel"
+ verbose_level = val;
+ if (! isscalar (verbose_level))
+ error ("parcellfun: VerboseLevel must be a numeric scalar");
+ endif
+ case "chunksperproc"
+ chunks_per_proc = round (val);
+ if (! isscalar (chunks_per_proc) || chunks_per_proc <= 0)
+ error ("parcellfun: ChunksPerProc must be a positive scalar");
+ endif
+ case "vectorized"
+ vectorized = logical (val);
+ if (! isscalar (vectorized))
+ error ("parcellfun: Vectorized must be a logical scalar");
+ endif
+ otherwise
+ break;
+ endswitch
+ nargs -= 2;
+ endwhile
+
+ if (vectorized && chunks_per_proc <= 0)
+ error ("parcellfun: the ""Vectorized"" option requires also ""ChunksPerProc""");
+ endif
+
+endfunction
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|