Revision: 246
http://clustershell.svn.sourceforge.net/clustershell/?rev=246&view=rev
Author: st-cea
Date: 2010-04-07 22:51:33 +0000 (Wed, 07 Apr 2010)
Log Message:
-----------
Node Groups addons for CS1.3 (trac #41):
* added external, multiple sources group support to NodeSet using a new helper module NodeUtils (#43)
* enhanced parser to support node/nodegroups basic arithmetics (union=, difference=! intersection=& and xor=^) (#44)
* added nodeset -r (regroup/reverse) and nodeset -l (list groups) options
* modified clush to use new group mechanisms when using -a, -g or -X (-w/-x are automatically compatible with new @group syntax)
* added example groups.conf file
* added nonreg tests.
Modified Paths:
--------------
trunk/lib/ClusterShell/NodeSet.py
trunk/scripts/clush.py
trunk/scripts/nodeset.py
trunk/tests/NodeSetErrorTest.py
trunk/tests/run_testsuite.py
Added Paths:
-----------
trunk/conf/groups.conf
trunk/lib/ClusterShell/NodeUtils.py
trunk/tests/NodeSetGroupTest.py
Added: trunk/conf/groups.conf
===================================================================
--- trunk/conf/groups.conf (rev 0)
+++ trunk/conf/groups.conf 2010-04-07 22:51:33 UTC (rev 246)
@@ -0,0 +1,20 @@
+# Configuration file for ClusterShell node groups
+# This is a example, please edit to fit your needs!
+# See man groups.conf(5).
+# $Id$
+
+[Main]
+default:local
+
+[local]
+map: awk -F: '/^$GROUP:/ {print $2}' /etc/clustershell/groups
+all: awk -F: '/^all:/ {print $2}' /etc/clustershell/groups
+list: awk -F: '/^\w/ {print $1}' /etc/clustershell/groups
+#reverse:
+
+[slurm]
+map: sinfo -h -o "%N" -p $group
+all: sinfo -h -o "%N"
+list: sinfo -h -o "%P"
+reverse: sinfo -h -N -o "%P" -n $node
+
Property changes on: trunk/conf/groups.conf
___________________________________________________________________
Added: svn:keywords
+ Id
Modified: trunk/lib/ClusterShell/NodeSet.py
===================================================================
--- trunk/lib/ClusterShell/NodeSet.py 2010-03-09 13:18:22 UTC (rev 245)
+++ trunk/lib/ClusterShell/NodeSet.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -65,6 +65,15 @@
import copy
import re
+import ClusterShell.NodeUtils as NodeUtils
+
+
+# Define default GroupResolver object used by NodeSet
+DEF_GROUPS_CONFIG = "/etc/clustershell/groups.conf"
+DEF_STD_GROUP_RESOLVER = NodeUtils.GroupResolverConfig(DEF_GROUPS_CONFIG)
+STD_GROUP_RESOLVER = DEF_STD_GROUP_RESOLVER
+
+
class RangeSetException(Exception):
"""Base RangeSet exception class."""
@@ -100,7 +109,10 @@
def __init__(self, rset_exc):
NodeSetParseError.__init__(self, str(rset_exc), "bad range")
+class NodeSetExternalError(NodeSetException):
+ """Raised when an external error is encountered."""
+
class RangeSet:
"""
Advanced range sets.
@@ -201,6 +213,15 @@
inst.update(RangeSet(rg))
return inst
+ @classmethod
+ def fromone(cls, index, pad=0, autostep=None):
+ """
+ Class method that returns a new RangeSet of one single item.
+ """
+ inst = RangeSet(autostep=autostep)
+ inst.add(index, pad)
+ return inst
+
def __iter__(self):
"""
Iterate over each item in RangeSet.
@@ -500,11 +521,11 @@
return NotImplemented
return self.union(other)
- def add(self, elem):
+ def add(self, elem, pad=0):
"""
Add element to RangeSet.
"""
- self.add_range(elem, elem, step=1, pad=0)
+ self.add_range(elem, elem, step=1, pad=pad)
def update(self, rangeset):
"""
@@ -719,161 +740,50 @@
return self._fold(lst, pad1)
-def _NodeSetParse(ns, autostep):
+class NodeSetBase(object):
"""
- Internal RangeSet generator for NodeSet or nodeset string pattern
- parsing.
+ Base class for NodeSet.
"""
- # is ns a NodeSet instance?
- if isinstance(ns, NodeSet):
- for pat, rangeset in ns._patterns.iteritems():
- yield pat, rangeset
- # or is ns a string?
- elif type(ns) is str:
- single_node_re = None
- pat = str(ns)
- # avoid misformatting
- if pat.find('%') >= 0:
- pat = pat.replace('%', '%%')
- while pat is not None:
- # Ignore whitespace(s) for convenience
- pat = pat.lstrip()
-
- # What's first: a simple node or a pattern of nodes?
- comma_idx = pat.find(',')
- bracket_idx = pat.find('[')
-
- # Check if the comma is after the bracket, or if there
- # is no comma at all but some brackets.
- if bracket_idx >= 0 and (comma_idx > bracket_idx or comma_idx < 0):
-
- # In this case, we have a pattern of potentially several
- # nodes.
-
- # Fill prefix, range and suffix from pattern
- # eg. "forbin[3,4-10]-ilo" -> "forbin", "3,4-10", "-ilo"
- pfx, sfx = pat.split('[', 1)
-
- try:
- rg, sfx = sfx.split(']', 1)
- except ValueError:
- raise NodeSetParseError(pat, "missing bracket")
-
- # Check if we have a next comma-separated node or pattern
- if sfx.find(',') < 0:
- pat = None
- else:
- sfx, pat = sfx.split(',', 1)
-
- # Ignore whitespace(s)
- sfx = sfx.rstrip()
-
- # pfx + sfx cannot be empty
- if len(pfx) + len(sfx) == 0:
- raise NodeSetParseError(pat, "empty node name")
-
- # Process comma-separated ranges
- try:
- rset = RangeSet(rg, autostep)
- except RangeSetParseError, e:
- raise NodeSetParseRangeError(e)
-
- yield "%s%%s%s" % (pfx, sfx), rset
- else:
- # In this case, either there is no comma and no bracket,
- # or the bracket is after the comma, then just return
- # the node.
- if comma_idx < 0:
- node = pat
- pat = None # break next time
- else:
- node, pat = pat.split(',', 1)
- # Ignore whitespace(s)
- node = node.strip()
-
- if len(node) == 0:
- raise NodeSetParseError(pat, "empty node name")
-
- # single node parsing
- if single_node_re is None:
- single_node_re = re.compile("(\D*)(\d*)(.*)")
-
- mo = single_node_re.match(node)
- if not mo:
- raise NodeSetParseError(pat, "parse error")
- pfx, idx, sfx = mo.groups()
- pfx, sfx = pfx or "", sfx or ""
-
- # pfx+sfx cannot be empty
- if len(pfx) + len(sfx) == 0:
- raise NodeSetParseError(pat, "empty node name")
-
- if idx:
- try:
- rset = RangeSet(idx, autostep)
- except RangeSetParseError, e:
- raise NodeSetParseRangeError(e)
- p = "%s%%s%s" % (pfx, sfx)
- yield p, rset
- else:
- # undefined pad means no node index
- yield pfx, None
- else:
- raise NodeSetParseError('', "unsupported input %s" % type(ns))
-
-
-class NodeSet(object):
- """
- Iterable class of nodes with node ranges support.
-
- NodeSet creation examples:
- nodeset = NodeSet() # empty NodeSet
- nodeset = NodeSet("clustername3") # contains only clustername3
- nodeset = NodeSet("clustername[5,10-42]")
- nodeset = NodeSet("clustername[0-10/2]")
- nodeset = NodeSet("clustername[0-10/2],othername[7-9,120-300]")
-
- NodeSet provides methods like update(), intersection_update() or
- difference_update() methods, which conform to the Python Set API.
- However, unlike RangeSet or standard Set, NodeSet is somewhat not
- so strict for convenience, and understands NodeSet instance or
- NodeSet string as argument. Also, there is no strict definition of
- one element, for example, it IS allowed to do:
- nodeset.remove("blue[36-40]").
- """
- def __init__(self, pattern=None, autostep=None):
+ def __init__(self, pattern=None, rangeset=None):
"""
- Initialize a NodeSet. If no pattern is specified, an empty
- NodeSet is created.
+ Initialize an empty NodeSetBase.
"""
- self._autostep = autostep
self._length = 0
self._patterns = {}
- if pattern is not None:
- self.update(pattern)
+ if pattern:
+ self._add(pattern, rangeset)
+ elif rangeset:
+ raise ValueError("missing pattern")
- @classmethod
- def fromlist(cls, nodelist, autostep=None):
+ def _iter(self):
"""
- Class method that returns a new NodeSet with nodes from
- provided list.
+ Iterator on internal item tuples (pattern, index, padding).
"""
- inst = NodeSet(autostep=autostep)
- for node in nodelist:
- inst.update(node)
- return inst
-
- def __iter__(self):
- """
- Iterate over concret nodes.
- """
for pat, rangeset in sorted(self._patterns.iteritems()):
if rangeset:
for start, stop, step, pad in rangeset._ranges:
while start <= stop:
- yield pat % ("%0*d" % (pad, start))
+ yield pat, start, pad
start += step
else:
+ yield pat, None, None
+
+ def _iterbase(self):
+ """
+ Iterator on single, one-item NodeSetBase objects.
+ """
+ for pat, start, pad in self._iter():
+ yield NodeSetBase(pat, RangeSet.fromone(start, pad))
+
+ def __iter__(self):
+ """
+ Iterator on single nodes as string.
+ """
+ # Does not call self._iterbase() + str() for better performance.
+ for pat, start, pad in self._iter():
+ if start is not None:
+ yield pat % ("%0*d" % (pad, start))
+ else:
yield pat
def __len__(self):
@@ -890,7 +800,7 @@
def __str__(self):
"""
- Get pdsh-like, ranges-based pattern of node list.
+ Get ranges-based pattern of node list.
"""
result = ""
for pat, rangeset in sorted(self._patterns.iteritems()):
@@ -914,32 +824,23 @@
def _binary_sanity_check(self, other):
# check that the other argument to a binary operation is also
# a NodeSet, raising a TypeError otherwise.
- if not isinstance(other, NodeSet):
- raise TypeError, "Binary operation only permitted between NodeSets"
+ if not isinstance(other, NodeSetBase):
+ raise TypeError, "Binary operation only permitted between NodeSetBase"
def issubset(self, other):
"""
Report whether another nodeset contains this nodeset.
"""
- binary = None
-
- # type check is needed for this case...
- if isinstance(other, NodeSet):
- binary = other
- elif type(other) is str:
- binary = NodeSet(other)
- else:
- raise TypeError, \
- "Binary operation only permitted between NodeSets or string"
-
- return binary.issuperset(self)
+ self._binary_sanity_check(other)
+ return other.issuperset(self)
def issuperset(self, other):
"""
Report whether this nodeset contains another nodeset.
"""
+ self._binary_sanity_check(other)
status = True
- for pat, erangeset in _NodeSetParse(other, self._autostep):
+ for pat, erangeset in other._patterns.iteritems():
rangeset = self._patterns.get(pat)
if rangeset:
status = rangeset.issuperset(erangeset)
@@ -955,7 +856,7 @@
NodeSet equality comparison.
"""
# See comment for for RangeSet.__eq__()
- if not isinstance(other, NodeSet):
+ if not isinstance(other, NodeSetBase):
return NotImplemented
return len(self) == len(other) and self.issuperset(other)
@@ -984,41 +885,11 @@
"""
return list(self)[i]
- def __getslice__(self, i, j):
+ def _add(self, pat, rangeset):
"""
- Return the slice from index i to index j-1. For convenience
- only, not optimized as of version 1.0.
+ Add nodes from a (pat, rangeset) tuple. `pat' may be an existing
+ pattern and `rangeset' may be None.
"""
- return NodeSet.fromlist(list(self)[i:j])
-
- def split(self, nbr):
- """
- Split the nodeset into nbr sub-nodeset. Each sub-nodeset will have the
- same number of element more or less 1. Current nodeset remains
- unmodified.
-
- >>> NodeSet("foo[1-5]").split(3)
- NodeSet("foo[1-2]")
- NodeSet("foo[3-4]")
- NodeSet("foo5")
- """
- # XXX: This uses the non-optimized __getslice__ method.
- assert(nbr > 0)
-
- # We put the same number of element in each sub-nodeset.
- slice_size = len(self) / nbr
- left = len(self) % nbr
-
- begin = 0
- for i in range(0, nbr):
- length = slice_size + int(i < left)
- yield self[begin:begin + length]
- begin += length
-
- def _add_rangeset(self, pat, rangeset):
- """
- Add a rangeset to a new or existing pattern.
- """
# get patterns dict entry
pat_e = self._patterns.get(pat)
@@ -1046,7 +917,7 @@
Implements the | operator. So s | t returns a new nodeset with
elements from both s and t.
"""
- if not isinstance(other, NodeSet):
+ if not isinstance(other, NodeSetBase):
return NotImplemented
return self.union(other)
@@ -1060,9 +931,11 @@
"""
s.update(t) returns nodeset s with elements added from t.
"""
- for pat, rangeset in _NodeSetParse(other, self._autostep):
- self._add_rangeset(pat, rangeset)
+ self._binary_sanity_check(other)
+ for pat, rangeset in other._patterns.iteritems():
+ self._add(pat, rangeset)
+
def clear(self):
"""
Remove all nodes from this nodeset.
@@ -1101,25 +974,27 @@
s.intersection_update(t) returns nodeset s keeping only
elements also found in t.
"""
+ self._binary_sanity_check(other)
+
if other is self:
return
- tmp_ns = NodeSet()
+ tmp_ns = NodeSetBase()
- for pat, irangeset in _NodeSetParse(other, self._autostep):
+ for pat, irangeset in other._patterns.iteritems():
rangeset = self._patterns.get(pat)
if rangeset:
rs = copy.copy(rangeset)
rs.intersection_update(irangeset)
# ignore pattern if empty rangeset
if len(rs) > 0:
- tmp_ns._add_rangeset(pat, rs)
+ tmp_ns._add(pat, rs)
elif not irangeset and pat in self._patterns:
# intersect two nodes with no rangeset
- tmp_ns._add_rangeset(pat, None)
+ tmp_ns._add(pat, None)
elif not irangeset and pat in self._patterns:
# intersect two nodes with no rangeset
- tmp_ns._add_rangeset(pat, None)
+ tmp_ns._add(pat, None)
# Substitute
self._patterns = tmp_ns._patterns
@@ -1146,7 +1021,7 @@
Implement the - operator. So s - t returns a new nodeset with
elements in s but not in t.
"""
- if not isinstance(other, NodeSet):
+ if not isinstance(other, NodeSetBase):
return NotImplemented
return self.difference(other)
@@ -1156,11 +1031,12 @@
elements found in t. If strict is True, raise KeyError
if an element cannot be removed.
"""
+ self._binary_sanity_check(other)
# the purge of each empty pattern is done afterward to allow self = ns
purge_patterns = []
# iterate first over exclude nodeset rangesets which is usually smaller
- for pat, erangeset in _NodeSetParse(other, self._autostep):
+ for pat, erangeset in other._patterns.iteritems():
# if pattern is found, deal with it
rangeset = self._patterns.get(pat)
if rangeset:
@@ -1220,33 +1096,23 @@
s.symmetric_difference_update(t) returns nodeset s keeping all
nodes that are in exactly one of the nodesets.
"""
- binary = None
-
- # type check is needed for this case...
- if isinstance(other, NodeSet):
- binary = other
- elif type(other) is str:
- binary = NodeSet(other)
- else:
- raise TypeError, \
- "Binary operation only permitted between NodeSets or string"
-
+ self._binary_sanity_check(other)
purge_patterns = []
# iterate over our rangesets
for pat, rangeset in self._patterns.iteritems():
- brangeset = binary._patterns.get(pat)
+ brangeset = other._patterns.get(pat)
if brangeset:
rangeset.symmetric_difference_update(brangeset)
else:
- if binary._patterns.has_key(pat):
+ if other._patterns.has_key(pat):
purge_patterns.append(pat)
- # iterate over binary's rangesets
- for pat, brangeset in binary._patterns.iteritems():
+ # iterate over other's rangesets
+ for pat, brangeset in other._patterns.iteritems():
rangeset = self._patterns.get(pat)
if not rangeset and not self._patterns.has_key(pat):
- self._add_rangeset(pat, brangeset)
+ self._add(pat, brangeset)
# check for patterns cleanup
for pat, rangeset in self._patterns.iteritems():
@@ -1267,6 +1133,433 @@
return self.symmetric_difference_update(other)
+class NodeGroupBase(NodeSetBase):
+ """
+ """
+ def _add(self, pat, rangeset):
+ """
+ Add groups from a (pat, rangeset) tuple. `pat' may be an existing
+ pattern and `rangeset' may be None.
+ """
+ if pat and pat[0] != '@':
+ raise ValueError("NodeGroup name must begin with character '@'")
+ NodeSetBase._add(self, pat, rangeset)
+
+
+class ParsingEngine(object):
+ """
+ Class that is able to transform a source into a NodeSetBase.
+ """
+ OP_CODES = { 'update': ',',
+ 'difference_update': '!',
+ 'intersection_update': '&',
+ 'symmetric_difference_update': '^' }
+
+ def __init__(self, group_resolver):
+ """
+ Initialize Parsing Engine.
+ """
+ self.group_resolver = group_resolver
+
+ def parse(self, nsobj, autostep):
+ """
+ Parse provided object if possible and return a NodeSetBase object.
+ """
+ # is nsobj a NodeSetBase instance?
+ if isinstance(nsobj, NodeSetBase):
+ return nsobj
+
+ # or is nsobj a string?
+ if type(nsobj) is str:
+ try:
+ return self.parse_string(str(nsobj), autostep)
+ except NodeUtils.GroupSourceQueryFailed, exc:
+ raise NodeSetParseError(nsobj, str(exc))
+
+ raise TypeError("Unsupported NodeSet input %s" % type(nsobj))
+
+ def parse_string(self, nsstr, autostep):
+ """
+ Parse provided string and return a NodeSetBase object.
+ """
+ ns = NodeSetBase()
+
+ for opc, pat, rangeset in self._scan_string(nsstr, autostep):
+ # Parser main debugging:
+ #print "OPC %s PAT %s RANGESET %s" % (opc, pat, rangeset)
+ if self.group_resolver and pat[0] == '@':
+ ns_group = NodeSetBase()
+ for nodegroup in NodeGroupBase(pat, rangeset):
+ # parse/expand nodes group
+ ns_string_ext = self.parse_group_string(nodegroup)
+ if ns_string_ext:
+ # convert result and apply operation
+ ns_group.update(self.parse(ns_string_ext, autostep))
+ # perform operation
+ getattr(ns, opc)(ns_group)
+ else:
+ getattr(ns, opc)(NodeSetBase(pat, rangeset))
+
+ return ns
+
+ def parse_group(self, group, namespace=None, autostep=None):
+ """Parse provided single group name (without @ prefix)."""
+ assert self.group_resolver is not None
+ nodestr = self.group_resolver.group_nodes(group, namespace)
+ return self.parse(",".join(nodestr), autostep)
+
+ def parse_group_string(self, nodegroup):
+ """Parse provided group string and return a string."""
+ assert nodegroup[0] == '@'
+ assert self.group_resolver is not None
+ grpstr = nodegroup[1:]
+ if grpstr.find(':') < 0:
+ # default namespace
+ return ",".join(self.group_resolver.group_nodes(grpstr))
+ else:
+ # specified namespace
+ namespace, group = grpstr.split(':', 1)
+ return ",".join(self.group_resolver.group_nodes(group, namespace))
+
+ def _next_op(self, pat):
+ """Opcode parsing subroutine."""
+ op_idx = -1
+ next_op_code = None
+ for opc, idx in [(k, pat.find(v)) \
+ for k, v in ParsingEngine.OP_CODES.iteritems()]:
+ if idx >= 0 and (op_idx < 0 or idx <= op_idx):
+ next_op_code = opc
+ op_idx = idx
+ return op_idx, next_op_code
+
+ def _scan_string(self, nsstr, autostep):
+ """
+ Parsing engine's string scanner method.
+ """
+ single_node_re = None
+ pat = nsstr.strip()
+ # avoid misformatting
+ if pat.find('%') >= 0:
+ pat = pat.replace('%', '%%')
+ next_op_code = 'update'
+ while pat is not None:
+ # Ignore whitespace(s) for convenience
+ pat = pat.lstrip()
+
+ op_code, next_op_code = next_op_code, None
+ op_idx = -1
+ op_idx, next_op_code = self._next_op(pat)
+ bracket_idx = pat.find('[')
+
+ # Check if the operator is after the bracket, or if there
+ # is no operator at all but some brackets.
+ if bracket_idx >= 0 and (op_idx > bracket_idx or op_idx < 0):
+ # In this case, we have a pattern of potentially several
+ # nodes.
+ # Fill prefix, range and suffix from pattern
+ # eg. "forbin[3,4-10]-ilo" -> "forbin", "3,4-10", "-ilo"
+ pfx, sfx = pat.split('[', 1)
+ try:
+ rg, sfx = sfx.split(']', 1)
+ except ValueError:
+ raise NodeSetParseError(pat, "missing bracket")
+
+ # Check if we have a next op-separated node or pattern
+ op_idx, next_op_code = self._next_op(sfx)
+ if op_idx < 0:
+ pat = None
+ else:
+ sfx, pat = sfx.split(self.OP_CODES[next_op_code], 1)
+
+ # Ignore whitespace(s)
+ sfx = sfx.rstrip()
+
+ # pfx + sfx cannot be empty
+ if len(pfx) + len(sfx) == 0:
+ raise NodeSetParseError(pat, "empty node name")
+
+ # Process comma-separated ranges
+ try:
+ rset = RangeSet(rg, autostep)
+ except RangeSetParseError, e:
+ raise NodeSetParseRangeError(e)
+
+ yield op_code, "%s%%s%s" % (pfx, sfx), rset
+ else:
+ # In this case, either there is no comma and no bracket,
+ # or the bracket is after the comma, then just return
+ # the node.
+ if op_idx < 0:
+ node = pat
+ pat = None # break next time
+ else:
+ node, pat = pat.split(self.OP_CODES[next_op_code], 1)
+ # Ignore whitespace(s)
+ node = node.strip()
+
+ if len(node) == 0:
+ raise NodeSetParseError(pat, "empty node name")
+
+ # single node parsing
+ if single_node_re is None:
+ single_node_re = re.compile("(\D*)(\d*)(.*)")
+
+ mo = single_node_re.match(node)
+ if not mo:
+ raise NodeSetParseError(pat, "parse error")
+ pfx, idx, sfx = mo.groups()
+ pfx, sfx = pfx or "", sfx or ""
+
+ # pfx+sfx cannot be empty
+ if len(pfx) + len(sfx) == 0:
+ raise NodeSetParseError(pat, "empty node name")
+
+ if idx:
+ try:
+ rset = RangeSet(idx, autostep)
+ except RangeSetParseError, e:
+ raise NodeSetParseRangeError(e)
+ p = "%s%%s%s" % (pfx, sfx)
+ yield op_code, p, rset
+ else:
+ # undefined pad means no node index
+ yield op_code, pfx, None
+
+
+# Special constant for NodeSet's resolver parameter to avoid any group
+# resolution at all.
+NOGROUP_RESOLVER=-1
+
+
+class NodeSet(NodeSetBase):
+ """
+ Iterable class of nodes with node ranges support.
+
+ NodeSet creation examples:
+ nodeset = NodeSet() # empty NodeSet
+ nodeset = NodeSet("clustername3") # contains only clustername3
+ nodeset = NodeSet("clustername[5,10-42]")
+ nodeset = NodeSet("clustername[0-10/2]")
+ nodeset = NodeSet("clustername[0-10/2],othername[7-9,120-300]")
+
+ NodeSet provides methods like update(), intersection_update() or
+ difference_update() methods, which conform to the Python Set API.
+ However, unlike RangeSet or standard Set, NodeSet is somewhat not
+ so strict for convenience, and understands NodeSet instance or
+ NodeSet string as argument. Also, there is no strict definition of
+ one element, for example, it IS allowed to do:
+ nodeset.remove("blue[36-40]").
+ """
+ def __init__(self, nodes=None, autostep=None, resolver=None):
+ """
+ Initialize a NodeSet.
+ The `nodes' argument may be a valid nodeset string or a NodeSet
+ object. If no nodes are specified, an empty NodeSet is created.
+ """
+ NodeSetBase.__init__(self)
+
+ self._autostep = autostep
+
+ # Set group resolver.
+ self._resolver = None
+ if resolver != NOGROUP_RESOLVER:
+ self._resolver = resolver or STD_GROUP_RESOLVER
+
+ # Initialize default parser.
+ self._parser = ParsingEngine(self._resolver)
+
+ if nodes is not None:
+ self.update(nodes)
+
+ @classmethod
+ def fromlist(cls, nodelist, autostep=None, resolver=None):
+ """
+ Class method that returns a new NodeSet with nodes from
+ provided list.
+ """
+ inst = NodeSet(autostep=autostep, resolver=resolver)
+ for node in nodelist:
+ inst.update(node)
+ return inst
+
+ @classmethod
+ def fromall(cls, namespace=None, autostep=None, resolver=None):
+ """
+ Class method that returns a new NodeSet with all nodes from
+ optional namespace.
+ """
+ inst = NodeSet(autostep=autostep, resolver=resolver)
+ if not inst._resolver:
+ raise NodeSetExternalError("No node group resolver")
+ for nodes in inst._resolver.all_nodes():
+ inst.update(nodes)
+ return inst
+
+ def _find_groups(self, node, namespace, allgroups):
+ """Find groups of node by namespace."""
+ if allgroups:
+ # find node groups using in-memory allgroups
+ for grp, ns in allgroups.iteritems():
+ if node in ns:
+ yield grp
+ else:
+ # find node groups using resolver
+ for group in self._resolver.node_groups(node, namespace):
+ yield group
+
+ def regroup(self, namespace=None, autostep=None, overlap=False):
+ """
+ Regroup nodeset using groups.
+ """
+ groups = {}
+ rest = NodeSet(self, resolver=NOGROUP_RESOLVER)
+
+ try:
+ # Get a NodeSet of all groups in specified group namespace.
+ allgrpns = NodeSet.fromlist(self._resolver.grouplist(namespace),
+ resolver=NOGROUP_RESOLVER)
+ except NodeUtils.GroupSourceException:
+ # If list query failed, we still might be able to regroup
+ # using reverse.
+ allgrpns = None
+
+ allgroups = {}
+
+ # Check for external reverse presence, and also use the
+ # following heuristic: external reverse is used only when number
+ # of groups is greater than the NodeSet size.
+ if self._resolver.has_node_groups(namespace) and \
+ (not allgrpns or len(allgrpns) >= len(self)):
+ # use external reverse
+ pass
+ else:
+ if not allgrpns: # list query failed and no way to reverse!
+ return str(rest)
+ try:
+ # use internal reverse: populate allgroups
+ for grp in allgrpns:
+ nodelist = self._resolver.group_nodes(grp, namespace)
+ allgroups[grp] = NodeSet(",".join(nodelist))
+ except NodeUtils.GroupSourceQueryFailed, exc:
+ # External result inconsistency
+ raise NodeSetExternalError("Unable to map a group " \
+ "previously listed\n\tFailed command: %s" % exc)
+
+ # For each NodeSetBase in self, finds its groups.
+ for node in self._iterbase():
+ for grp in self._find_groups(node, namespace, allgroups):
+ if grp not in groups:
+ ns = self._parser.parse_group(grp, namespace, autostep)
+ groups[grp] = (0, ns)
+ i, m = groups[grp]
+ groups[grp] = (i + 1, m)
+
+ # Keep only groups that are full.
+ fulls = []
+ for k, (i, m) in groups.iteritems():
+ assert i <= len(m)
+ if i == len(m):
+ fulls.append((i, k))
+
+ regrouped = NodeSet(resolver=NOGROUP_RESOLVER)
+
+ bigalpha = lambda x,y: cmp(y[0],x[0]) or cmp(x[1],y[1])
+
+ # Build regrouped NodeSet by selecting largest groups first.
+ for num, grp in sorted(fulls, cmp=bigalpha):
+ if not overlap and groups[grp][1] not in rest:
+ continue
+ if namespace:
+ regrouped.update("@%s:%s" % (namespace, grp))
+ else:
+ regrouped.update("@" + grp)
+ rest.difference_update(groups[grp][1])
+ if not rest:
+ return str(regrouped)
+
+ if regrouped:
+ return "%s,%s" % (regrouped, rest)
+
+ return str(rest)
+
+ def issubset(self, other):
+ """
+ Report whether another nodeset contains this nodeset.
+ """
+ ns = self._parser.parse(other, self._autostep)
+ return NodeSetBase.issuperset(ns, self)
+
+ def issuperset(self, other):
+ """
+ Report whether this nodeset contains another nodeset.
+ """
+ ns = self._parser.parse(other, self._autostep)
+ return NodeSetBase.issuperset(self, ns)
+
+ def __getslice__(self, i, j):
+ """
+ Return the slice from index i to index j-1. For convenience
+ only, not optimized as of version 1.0.
+ """
+ return NodeSet.fromlist(list(self)[i:j])
+
+ def split(self, nbr):
+ """
+ Split the nodeset into nbr sub-nodeset. Each sub-nodeset will have the
+ same number of element more or less 1. Current nodeset remains
+ unmodified.
+
+ >>> NodeSet("foo[1-5]").split(3)
+ NodeSet("foo[1-2]")
+ NodeSet("foo[3-4]")
+ NodeSet("foo5")
+ """
+ # XXX: This uses the non-optimized __getslice__ method.
+ assert(nbr > 0)
+
+ # We put the same number of element in each sub-nodeset.
+ slice_size = len(self) / nbr
+ left = len(self) % nbr
+
+ begin = 0
+ for i in range(0, nbr):
+ length = slice_size + int(i < left)
+ yield self[begin:begin + length]
+ begin += length
+
+ def update(self, other):
+ """
+ s.update(t) returns nodeset s with elements added from t.
+ """
+ ns = self._parser.parse(other, self._autostep)
+ NodeSetBase.update(self, ns)
+
+ def intersection_update(self, other):
+ """
+ s.intersection_update(t) returns nodeset s keeping only
+ elements also found in t.
+ """
+ ns = self._parser.parse(other, self._autostep)
+ NodeSetBase.intersection_update(self, ns)
+
+ def difference_update(self, other, strict=False):
+ """
+ s.difference_update(t) returns nodeset s after removing
+ elements found in t. If strict is True, raise KeyError
+ if an element cannot be removed.
+ """
+ ns = self._parser.parse(other, self._autostep)
+ NodeSetBase.difference_update(self, ns, strict)
+
+ def symmetric_difference_update(self, other):
+ """
+ s.symmetric_difference_update(t) returns nodeset s keeping all
+ nodes that are in exactly one of the nodesets.
+ """
+ ns = self._parser.parse(other, self._autostep)
+ NodeSetBase.symmetric_difference_update(self, ns)
+
+
def expand(pat):
"""
Commodity function that expands a pdsh-like pattern into a list of
@@ -1281,7 +1574,14 @@
"""
return str(NodeSet(pat))
+def grouplist(namespace=None):
+ """
+ Commodity function that retrieves the list of groups for a specified
+ group namespace (or use default namespace).
+ """
+ return STD_GROUP_RESOLVER.grouplist(namespace)
+
# doctest
def _test():
Added: trunk/lib/ClusterShell/NodeUtils.py
===================================================================
--- trunk/lib/ClusterShell/NodeUtils.py (rev 0)
+++ trunk/lib/ClusterShell/NodeUtils.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -0,0 +1,315 @@
+# Copyright CEA/DAM/DIF (2010)
+# Contributors:
+# Stephane THIELL <stephane.thiell@...>
+# Aurelien DEGREMONT <aurelien.degremont@...>
+#
+# This file is part of the ClusterShell library.
+#
+# This software is governed by the CeCILL-C license under French law and
+# abiding by the rules of distribution of free software. You can use,
+# modify and/ or redistribute the software under the terms of the CeCILL-C
+# license as circulated by CEA, CNRS and INRIA at the following URL
+# "http://www.cecill.info".
+#
+# As a counterpart to the access to the source code and rights to copy,
+# modify and redistribute granted by the license, users are provided only
+# with a limited warranty and the software's author, the holder of the
+# economic rights, and the successive licensors have only limited
+# liability.
+#
+# In this respect, the user's attention is drawn to the risks associated
+# with loading, using, modifying and/or developing or reproducing the
+# software by the user in light of its specific status of free software,
+# that may mean that it is complicated to manipulate, and that also
+# therefore means that it is reserved for developers and experienced
+# professionals having in-depth computer knowledge. Users are therefore
+# encouraged to load and test the software's suitability as regards their
+# requirements in conditions enabling the security of their systems and/or
+# data to be ensured and, more generally, to use and operate it in the
+# same conditions as regards security.
+#
+# The fact that you are presently reading this means that you have had
+# knowledge of the CeCILL-C license and that you accept its terms.
+#
+# $Id$
+
+"""
+Cluster nodes utility module
+
+The NodeUtils module is a ClusterShell helper module that provides
+supplementary services to manage nodes in a cluster. It is primarily
+designed to enhance the NodeSet module providing some binding support
+to external node groups sources in separate namespaces (example of
+group sources are: files, jobs scheduler, custom scripts, etc.).
+"""
+
+import sys
+
+from ConfigParser import ConfigParser, NoOptionError, NoSectionError
+from string import Template
+from subprocess import Popen, PIPE
+
+
+class GroupSourceException(Exception):
+ """Base GroupSource exception"""
+
+class GroupSourceNoUpcall(GroupSourceException):
+ """Raised when upcall is not available"""
+
+class GroupSourceQueryFailed(GroupSourceException):
+ """Raised when a query failed (eg. no group found)"""
+
+class GroupResolverError(Exception):
+ """Base GroupResolver error"""
+
+class GroupResolverSourceError(GroupResolverError):
+ """Raised when upcall is not available"""
+
+class GroupResolverConfigError(GroupResolverError):
+ """Raised when a configuration error is encountered"""
+
+
+class GroupSource(object):
+ """
+ GroupSource class managing external calls for nodegroup support.
+ """
+ def __init__(self, name, map_upcall, all_upcall=None,
+ list_upcall=None, reverse_upcall=None):
+ self.name = name
+ self.verbosity = 0
+
+ # Cache upcall data
+ self._cache_map = {}
+ self._cache_list = []
+ self._cache_all = None
+ self._cache_reverse = {}
+
+ # Supported external upcalls
+ self.map_upcall = map_upcall
+ self.all_upcall = all_upcall
+ self.list_upcall = list_upcall
+ self.reverse_upcall = reverse_upcall
+
+ def _verbose_print(self, msg):
+ if self.verbosity > 0:
+ print >> sys.stderr, "%s<%s> %s" % \
+ (self.__class__.__name__, self.name, msg)
+
+ def _upcall_read(self, cmdtpl, vars=dict()):
+ """
+ Invoke the specified upcall command, raise an Exception if
+ something goes wrong and return the command output otherwise.
+ """
+ cmdline = Template(getattr(self, "%s_upcall" % \
+ cmdtpl)).safe_substitute(vars)
+ self._verbose_print("EXEC '%s'" % cmdline)
+ proc = Popen(cmdline, stdout=PIPE, shell=True)
+ output = proc.communicate()[0].strip()
+ self._verbose_print("READ '%s'" % output)
+ if proc.returncode != 0:
+ self._verbose_print("ERROR '%s' returned %d" % (cmdline, \
+ proc.returncode))
+ raise GroupSourceQueryFailed(cmdline)
+ return output
+
+ def resolv_map(self, group):
+ """
+ Get nodes from group 'group', using the cached value if
+ available.
+ """
+ if group not in self._cache_map:
+ self._cache_map[group] = self._upcall_read('map', dict(GROUP=group))
+
+ return self._cache_map[group]
+
+ def resolv_list(self):
+ """
+ Return a list of all group names for this group source, using
+ the cached value if available.
+ """
+ if not self.list_upcall:
+ raise GroupSourceNoUpcall("list")
+
+ if not self._cache_list:
+ self._cache_list = self._upcall_read('list')
+
+ return self._cache_list
+
+ def resolv_all(self):
+ """
+ Return the content of special group ALL, using the cached value
+ if available.
+ """
+ if not self.all_upcall:
+ raise GroupSourceNoUpcall("all")
+
+ if not self._cache_all:
+ self._cache_all = self._upcall_read('all')
+
+ return self._cache_all
+
+ def resolv_reverse(self, node):
+ """
+ Return the group name matching the provided node, using the
+ cached value if available.
+ """
+ if not self.reverse_upcall:
+ raise GroupSourceNoUpcall("reverse")
+
+ if node not in self._cache_reverse:
+ self._cache_reverse[node] = self._upcall_read('reverse', \
+ dict(NODE=node))
+ return self._cache_reverse[node]
+
+
+class GroupResolver(object):
+ """
+ Base class GroupResolver that aims to provide node/group resolution
+ from multiple GroupSource's.
+ """
+
+ GROUP_ALL_NAME = 'all'
+
+ def __init__(self, default_source=None):
+ """
+ Initialize GroupResolver object.
+ """
+ self._sources = {}
+ self._default_source = default_source
+ if default_source:
+ self._sources[default_source.name] = default_source
+
+ def set_verbosity(self, value):
+ """
+ Set debugging verbosity value.
+ """
+ for source in self._sources.itervalues():
+ source.verbosity = value
+
+ def add_source(self, group_source):
+ """
+ Add a GroupSource to this resolver.
+ """
+ if group_source.name in self._sources:
+ raise ValueError("GroupSource '%s': name collision" % \
+ group_source.name)
+ self._sources[group_source.name] = group_source
+
+ def _list(self, source, what, *args):
+ """Helper method that returns a list of result when the source
+ is defined."""
+ result = []
+ if not source:
+ raise GroupSourceQueryFailed("Unable to resolve group")
+ raw = getattr(source, 'resolv_%s' % what)(*args)
+ for line in raw.splitlines():
+ map(result.append, line.strip().split())
+
+ return result
+
+ def _source(self, namespace):
+ """Helper method that returns the source by namespace name."""
+ if not namespace:
+ source = self._default_source
+ else:
+ source = self._sources.get(namespace)
+ if not source:
+ raise GroupResolverSourceError("%s" % namespace)
+ return source
+
+ def group_nodes(self, group, namespace=None):
+ """
+ Find nodes for specified group name and optional namespace.
+ """
+ source = self._source(namespace)
+ return self._list(source, 'map', group)
+
+ def all_nodes(self, namespace=None):
+ """
+ Find all nodes. You may specify an optional namespace.
+ """
+ source = self._source(namespace)
+ try:
+ return self._list(source, 'all')
+ except GroupSourceNoUpcall:
+ return self._list(source, 'map', GroupResolver.GROUP_ALL_NAME)
+
+ def grouplist(self, namespace=None):
+ """
+ Get full group list. You may specify an optional
+ namespace.
+ """
+ source = self._source(namespace)
+ return self._list(source, 'list')
+
+ def has_node_groups(self, namespace=None):
+ """
+ Return whether finding group list for a specified node is
+ supported by the resolver (in optional namespace).
+ """
+ source = self._source(namespace)
+ return source is not None and bool(source.reverse_upcall)
+
+ def node_groups(self, node, namespace=None):
+ """
+ Find group list for specified node and optional namespace.
+ """
+ source = self._source(namespace)
+ return self._list(source, 'reverse', node)
+
+
+class GroupResolverConfig(GroupResolver):
+ """
+ GroupResolver class that is able to automatically setup its
+ GroupSource's from a configuration file. This is the default
+ resolver for NodeSet.
+ """
+
+ def __init__(self, configfile):
+ """
+ """
+ GroupResolver.__init__(self)
+
+ self.default_namespace = None
+
+ self.config = ConfigParser()
+ self.config.read(configfile)
+
+ # Get config file sections
+ group_sections = self.config.sections()
+ if 'Main' in group_sections:
+ group_sections.remove('Main')
+
+ if not group_sections:
+ return
+
+ try:
+ self.default_namespace = self.config.get('Main', 'default')
+ if self.default_namespace and self.default_namespace \
+ not in group_sections:
+ raise GroupResolverConfigError("Default namespace not found: "
+ "\"%s\"" % self.default_namespace)
+ except (NoSectionError, NoOptionError):
+ pass
+
+ # When not specified, select a random section.
+ if not self.default_namespace:
+ self.default_namespace = group_sections[0]
+
+ for section in group_sections:
+ map_upcall = self.config.get(section, 'map', True)
+ all_upcall = list_upcall = reverse_upcall = None
+ if self.config.has_option(section, 'all'):
+ all_upcall = self.config.get(section, 'all', True)
+ if self.config.has_option(section, 'list'):
+ list_upcall = self.config.get(section, 'list', True)
+ if self.config.has_option(section, 'reverse'):
+ reverse_upcall = self.config.get(section, 'reverse', True)
+
+ self.add_source(GroupSource(section, map_upcall, all_upcall,
+ list_upcall, reverse_upcall))
+
+ def _source(self, namespace):
+ return GroupResolver._source(self, namespace or self.default_namespace)
+
+
Property changes on: trunk/lib/ClusterShell/NodeUtils.py
___________________________________________________________________
Added: svn:keywords
+ Id
Modified: trunk/scripts/clush.py
===================================================================
--- trunk/scripts/clush.py 2010-03-09 13:18:22 UTC (rev 245)
+++ trunk/scripts/clush.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -34,13 +34,14 @@
# $Id$
"""
-Utility program to run commands on a cluster using the ClusterShell library.
+Utility program to run commands on a cluster using the ClusterShell
+library.
-clush is a pdsh-like command which benefits from the ClusterShell library
-and its Ssh worker. It features an integrated output results gathering
-system (dshbak-like), can get node groups by running predefined external
-commands and can redirect lines read on its standard input to the remote
-commands.
+clush is a pdsh-like command which benefits from the ClusterShell
+library and its Ssh worker. It features an integrated output results
+gathering system (dshbak-like), can get node groups by running
+predefined external commands and can redirect lines read on its
+standard input to the remote commands.
When no command are specified, clush runs interactively.
@@ -55,6 +56,7 @@
from ClusterShell.Event import EventHandler
from ClusterShell.NodeSet import NodeSet, NodeSetParseError
+from ClusterShell.NodeSet import STD_GROUP_RESOLVER
from ClusterShell.Task import Task, task_self
from ClusterShell.Worker.Worker import WorkerSimple
from ClusterShell import __version__
@@ -298,23 +300,7 @@
def get_ssh_options(self):
return self._get_optional("Main", "ssh_options")
- def get_nodes_all_command(self):
- section = "External"
- option = "nodes_all"
- try:
- return self.get(section, option)
- except ConfigParser.Error, e:
- raise ClushConfigError(section, option, e)
- def get_nodes_group_command(self, group):
- section = "External"
- option = "nodes_group"
- try:
- return self.get(section, option, 0, { "group" : group })
- except ConfigParser.Error, e:
- raise ClushConfigError(section, option, e)
-
-
def signal_handler(signum, frame):
"""Signal handler used for main thread notification"""
if signum == signal.SIGUSR1:
@@ -668,30 +654,31 @@
# Do we have nodes group?
task = task_self()
task.set_info("debug", config.get_verbosity() > 1)
+ if config.get_verbosity() > 1:
+ STD_GROUP_RESOLVER.set_verbosity(1)
if options.nodes_all:
- command = config.get_nodes_all_command()
- task.shell(command, key="all")
+ all_nodeset = NodeSet.fromall()
+ config.verbose_print(VERB_DEBUG, \
+ "Adding nodes from option -a: %s" % all_nodeset)
+ nodeset_base.add(all_nodeset)
+
if options.group:
- for grp in options.group:
- command = config.get_nodes_group_command(grp)
- task.shell(command, key="group")
+ grp_nodeset = NodeSet()
+ for grpopt in options.group:
+ for grp in grpopt.split(','):
+ addingrp = NodeSet("@" + grp)
+ config.verbose_print(VERB_DEBUG, \
+ "Adding nodes from option -g %s: %s" % (grp, addingrp))
+ nodeset_base.update(addingrp)
+
if options.exgroup:
- for grp in options.exgroup:
- command = config.get_nodes_group_command(grp)
- task.shell(command, key="exgroup")
+ for grpopt in options.exgroup:
+ for grp in grpopt.split(','):
+ removingrp = NodeSet("@" + grp)
+ config.verbose_print(VERB_DEBUG, \
+ "Excluding nodes from option -X %s: %s" % (grp, removingrp))
+ nodeset_exclude.update(removingrp)
- # Run needed external commands
- task.resume()
-
- for buf, keys in task.iter_buffers(['all', 'group']):
- for line in buf:
- config.verbose_print(VERB_DEBUG, "Adding nodes from option %s: %s" % (','.join(keys), buf))
- nodeset_base.add(line)
- for buf, keys in task.iter_buffers(['exgroup']):
- for line in buf:
- config.verbose_print(VERB_DEBUG, "Excluding nodes from option %s: %s" % (','.join(keys), buf))
- nodeset_exclude.add(line)
-
# Do we have an exclude list? (-x ...)
nodeset_base.difference_update(nodeset_exclude)
if len(nodeset_base) < 1:
Modified: trunk/scripts/nodeset.py
===================================================================
--- trunk/scripts/nodeset.py 2010-03-09 13:18:22 UTC (rev 245)
+++ trunk/scripts/nodeset.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -1,6 +1,6 @@
#!/usr/bin/env python
#
-# Copyright CEA/DAM/DIF (2008, 2009)
+# Copyright CEA/DAM/DIF (2008, 2009, 2010)
# Contributor: Stephane THIELL <stephane.thiell@...>
#
# This file is part of the ClusterShell library.
@@ -43,6 +43,10 @@
Expand nodesets to separate nodes.
--fold, -f <nodeset> [nodeset ...]
Compact/fold nodesets (or separate nodes) into one nodeset.
+ --list, -l
+ List node groups (compatible with --namespace).
+ --regroup, -r <nodeset> [nodeset ...]
+ Fold nodes using node groups (compatible with --namespace).
Options:
--autostep=<number>, -a <number>
Specify auto step threshold number when folding nodesets.
@@ -53,6 +57,8 @@
This help page.
--quiet, -q
Quiet mode, hide any parse error messages (on stderr).
+ --namespace, -n
+ Group namespace (section to use in groups.conf(5)).
--rangeset, -R
Switch to RangeSet instead of NodeSet. Useful when working on
numerical cluster ranges, eg. 1,5,18-31.
@@ -75,10 +81,19 @@
import signal
import sys
-from ClusterShell.NodeSet import NodeSet, NodeSetParseError
-from ClusterShell.NodeSet import RangeSet, RangeSetParseError
-from ClusterShell import __version__
+from ClusterShell.NodeUtils import GroupResolverConfigError
+from ClusterShell.NodeUtils import GroupResolverSourceError
+try:
+ from ClusterShell.NodeSet import NodeSet, NodeSetParseError
+ from ClusterShell.NodeSet import RangeSet, RangeSetParseError
+ from ClusterShell.NodeSet import grouplist, STD_GROUP_RESOLVER
+ from ClusterShell import __version__
+except GroupResolverConfigError, e:
+ print >> sys.stderr, \
+ "ERROR: ClusterShell Groups configuration error:\n\t%s" % e
+ sys.exit(1)
+
def process_stdin(xset, autostep):
"""Process standard input and populate xset."""
for line in sys.stdin.readlines():
@@ -116,15 +131,17 @@
"""
autostep = None
command = None
- quiet = False
+ verbosity = 1
class_set = NodeSet
separator = ' '
+ namespace = None
# Parse getoptable options
try:
- opts, args = getopt.getopt(args[1:], "a:cefhqvRS:",
- ["autostep=", "count", "expand", "fold", "help",
- "quiet", "rangeset", "version", "separator="])
+ opts, args = getopt.getopt(args[1:], "a:cdefhln:qvrRS:",
+ ["autostep=", "count", "debug", "expand", "fold", "help",
+ "list", "namespace=", "quiet", "regroup", "rangeset",
+ "version", "separator="])
except getopt.error, err:
if err.opt in [ "i", "intersection", "x", "exclude", "X", "xor" ]:
print >> sys.stderr, "option -%s not allowed here" % err.opt
@@ -141,6 +158,8 @@
print >> sys.stderr, exc
elif k in ("-c", "--count"):
command = "count"
+ elif k in ("-d", "--debug"):
+ verbosity = 2
elif k in ("-e", "--expand"):
command = "expand"
elif k in ("-f", "--fold"):
@@ -148,8 +167,14 @@
elif k in ("-h", "--help"):
print __doc__
sys.exit(0)
+ elif k in ("-l", "--list"):
+ command = "list"
+ elif k in ("-n", "--namespace"):
+ namespace = val
elif k in ("-q", "--quiet"):
- quiet = True
+ verbosity = 0
+ elif k in ("-r", "--regroup"):
+ command = "regroup"
elif k in ("-R", "--rangeset"):
class_set = RangeSet
elif k in ("-S", "--separator"):
@@ -164,7 +189,19 @@
print __doc__
sys.exit(1)
+ # The list command doesn't need any NodeSet, check for it first.
+ if command == "list":
+ for group in grouplist(namespace):
+ if namespace:
+ print "@%s:%s" % (namespace, group)
+ else:
+ print "@%s" % group
+ return
+
try:
+ if verbosity > 1:
+ STD_GROUP_RESOLVER.set_verbosity(1)
+
# Instantiate RangeSet or NodeSet object
xset = class_set()
@@ -183,11 +220,13 @@
print separator.join(xset)
elif command == "fold":
print xset
+ elif command == "regroup":
+ print xset.regroup(namespace)
else:
print len(xset)
except (NodeSetParseError, RangeSetParseError), exc:
- if not quiet:
+ if verbosity > 0:
print >> sys.stderr, "%s parse error:" % class_set.__name__, exc
sys.exit(1)
@@ -205,5 +244,8 @@
except SyntaxError:
print >> sys.stderr, "ERROR: invalid separator"
sys.exit(1)
+ except GroupResolverSourceError, e:
+ print >> sys.stderr, "ERROR: unknown group namespace: \"%s\"" % e
+ sys.exit(1)
except KeyboardInterrupt:
sys.exit(128 + signal.SIGINT)
Modified: trunk/tests/NodeSetErrorTest.py
===================================================================
--- trunk/tests/NodeSetErrorTest.py 2010-03-09 13:18:22 UTC (rev 245)
+++ trunk/tests/NodeSetErrorTest.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -63,12 +63,8 @@
def testTypeSanityCheck(self):
"""test NodeSet input type sanity check"""
- self.assertRaises(NodeSetParseError, NodeSet, dict())
- self.assertRaises(NodeSetParseError, NodeSet, list())
- try:
- ns1 = NodeSet(dict())
- except NodeSetParseError, e:
- self.assertEqual(e.part, '')
+ self.assertRaises(TypeError, NodeSet, dict())
+ self.assertRaises(TypeError, NodeSet, list())
if __name__ == '__main__':
Added: trunk/tests/NodeSetGroupTest.py
===================================================================
--- trunk/tests/NodeSetGroupTest.py (rev 0)
+++ trunk/tests/NodeSetGroupTest.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -0,0 +1,406 @@
+#!/usr/bin/env python
+# ClusterShell.Node* test suite
+# Written by S. Thiell 2010-03-18
+# $Id$
+
+
+"""Unit test for NodeSet with Group support"""
+
+import copy
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, '../lib')
+
+# Wildcard import for testing purpose
+import ClusterShell.NodeSet
+from ClusterShell.NodeSet import *
+from ClusterShell.NodeUtils import *
+
+
+class NodeSetGroupTest(unittest.TestCase):
+
+ def setSimpleStdGroupResolver(self):
+ # create 2 GroupSource objects
+ default = GroupSource("default",
+ "awk -F: '/^$GROUP:/ {print $2}' test_groups1",
+ "awk -F: '/^all:/ {print $2}' test_groups1",
+ "awk -F: '/^\w/ {print $1}' test_groups1",
+ None)
+
+ source2 = GroupSource("source2",
+ "awk -F: '/^$GROUP:/ {print $2}' test_groups2",
+ "awk -F: '/^all:/ {print $2}' test_groups2",
+ "awk -F: '/^\w/ {print $1}' test_groups2",
+ None)
+
+ ClusterShell.NodeSet.STD_GROUP_RESOLVER = GroupResolver(default)
+ ClusterShell.NodeSet.STD_GROUP_RESOLVER.add_source(source2)
+
+ def restoreStdGroupResolver(self):
+ ClusterShell.NodeSet.STD_GROUP_RESOLVER = ClusterShell.NodeSet.DEF_STD_GROUP_RESOLVER
+
+ def makeTestFile(self, text):
+ """
+ Create a temporary file with the provided text.
+ """
+ f = tempfile.NamedTemporaryFile()
+ f.write(text)
+ f.flush()
+ return f
+
+ def testGroupResolverSimple(self):
+ """test NodeSet with simple custom GroupResolver"""
+
+ source = GroupSource("simple",
+ "awk -F: '/^$GROUP:/ {print $2}' test_groups1",
+ "awk -F: '/^all:/ {print $2}' test_groups1",
+ "awk -F: '/^\w/ {print $1}' test_groups1",
+ None)
+
+ # create custom resolver with default source
+ res = GroupResolver(source)
+
+ nodeset = NodeSet("@gpu", resolver=res)
+ self.assertEqual(nodeset, NodeSet("montana[38-41]"))
+ self.assertEqual(str(nodeset), "montana[38-41]")
+
+ nodeset = NodeSet("@chassis3", resolver=res)
+ self.assertEqual(str(nodeset), "montana[36-37]")
+
+ nodeset = NodeSet("@chassis[3-4]", resolver=res)
+ self.assertEqual(str(nodeset), "montana[36-39]")
+
+ nodeset = NodeSet("@chassis[1,3,5]", resolver=res)
+ self.assertEqual(str(nodeset), "montana[32-33,36-37,40-41]")
+
+ nodeset = NodeSet("@chassis[2-12/2]", resolver=res)
+ self.assertEqual(str(nodeset), "montana[34-35,38-39,42-43,46-47,50-51,54-55]")
+
+ nodeset = NodeSet("@chassis[1,3-4,5-11/3]", resolver=res)
+ self.assertEqual(str(nodeset), "montana[32-33,36-41,46-47,52-53]")
+
+ # test recursive group gpuchassis
+ nodeset1 = NodeSet("@chassis[4-5]", resolver=res)
+ nodeset2 = NodeSet("@gpu", resolver=res)
+ nodeset3 = NodeSet("@gpuchassis", resolver=res)
+ self.assertEqual(nodeset1, nodeset2)
+ self.assertEqual(nodeset2, nodeset3)
+
+ # test also with some inline operations
+ nodeset = NodeSet("montana3,@gpuchassis!montana39,montana77^montana38",
+ resolver=res)
+ self.assertEqual(str(nodeset), "montana[3,40-41,77]")
+
+ def testGroupSyntaxes(self):
+ """test NodeSet group operation syntaxes"""
+
+ self.setSimpleStdGroupResolver()
+ try:
+ nodeset = NodeSet("@gpu")
+ self.assertEqual(str(nodeset), "montana[38-41]")
+
+ nodeset = NodeSet("@chassis[1-3,5]&@chassis[2-3]")
+ self.assertEqual(str(nodeset), "montana[34-37]")
+
+ nodeset1 = NodeSet("@io!@mds")
+ nodeset2 = NodeSet("@oss")
+ self.assertEqual(str(nodeset1), str(nodeset2))
+ self.assertEqual(str(nodeset1), "montana[4-5]")
+
+ finally:
+ self.restoreStdGroupResolver()
+
+ def testGroupListDefault(self):
+ """test group listing GroupResolver.grouplist()"""
+ self.setSimpleStdGroupResolver()
+ try:
+ groups = ClusterShell.NodeSet.STD_GROUP_RESOLVER.grouplist()
+ self.assertEqual(len(groups), 21)
+ helper_groups = grouplist()
+ self.assertEqual(len(helper_groups), 21)
+ total = 0
+ nodes = NodeSet()
+ for group in groups:
+ ns = NodeSet("@%s" % group)
+ total += len(ns)
+ nodes.update(ns)
+ self.assertEqual(total, 311)
+
+ all_nodes = NodeSet.fromall()
+ self.assertEqual(len(all_nodes), len(nodes))
+ self.assertEqual(all_nodes, nodes)
+ finally:
+ self.restoreStdGroupResolver()
+
+ def testGroupListSource2(self):
+ """test group listing GroupResolver.grouplist(source)"""
+ self.setSimpleStdGroupResolver()
+ try:
+ groups = ClusterShell.NodeSet.STD_GROUP_RESOLVER.grouplist("source2")
+ self.assertEqual(len(groups), 2)
+ total = 0
+ for group in groups:
+ total += len(NodeSet("@source2:%s" % group))
+ self.assertEqual(total, 24)
+ finally:
+ self.restoreStdGroupResolver()
+
+ def testAllNoResolver(self):
+ """test NodeSet.fromall() with no resolver"""
+ self.assertRaises(NodeSetExternalError, NodeSet.fromall,
+ resolver=NOGROUP_RESOLVER)
+
+ def testGroupResolverMinimal(self):
+ """test NodeSet with minimal GroupResolver"""
+
+ source = GroupSource("minimal",
+ "awk -F: '/^$GROUP:/ {print $2}' test_groups1",
+ None, None, None)
+
+ # create custom resolver with default source
+ res = GroupResolver(source)
+
+ nodeset = NodeSet("@gpu", resolver=res)
+ self.assertEqual(nodeset, NodeSet("montana[38-41]"))
+ self.assertEqual(str(nodeset), "montana[38-41]")
+
+ NodeSet.fromall(resolver=res)
+ #self.assertRaises(NodeSetExternalError, NodeSet.fromall)
+
+
+ def testConfigEmpty(self):
+ """test groups with an empty configuration file"""
+ f = self.makeTestFile("")
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "example[1-100]")
+ # non existant group
+ self.assertRaises(NodeSetParseError, NodeSet, "@bar", resolver=res)
+
+ def testConfigBasicLocal(self):
+ """test groups with a basic local config file"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo foo
+#reverse:
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+ self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
+
+ # regroup with rest
+ nodeset = NodeSet("example[1-101]", resolver=res)
+ self.assertEqual(nodeset.regroup(), "@foo,example101")
+
+ # regroup incomplete
+ nodeset = NodeSet("example[50-200]", resolver=res)
+ self.assertEqual(nodeset.regroup(), "example[50-200]")
+
+ # regroup no matching
+ nodeset = NodeSet("example[102-200]", resolver=res)
+ self.assertEqual(nodeset.regroup(), "example[102-200]")
+
+ def testConfigBasicLocalVerbose(self):
+ """test groups with a basic local config file (verbose)"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo foo
+#reverse:
+ """)
+ res = GroupResolverConfig(f.name)
+ res.set_verbosity(1)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+ self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
+
+ def testConfigBasicLocalAlternative(self):
+ """test groups with a basic local config file (= alternative)"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default=local
+
+[local]
+map=echo example[1-100]
+#all=
+list=echo foo
+#reverse=
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+ self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
+ # @truc?
+
+ def testConfigBasicEmptyDefault(self):
+ """test groups with a empty default namespace"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default:
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo foo
+#reverse:
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+ self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
+
+ def testConfigBasicNoMain(self):
+ """test groups with a local config without main section"""
+ f = self.makeTestFile("""
+# A comment
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo foo
+#reverse:
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+ self.assertEqual(str(NodeSet("@foo", resolver=res)), "example[1-100]")
+
+ def testConfigBasicWrongDefault(self):
+ """test groups with a wrong default namespace"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: pointless
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo foo
+#reverse:
+ """)
+ self.assertRaises(GroupResolverConfigError, GroupResolverConfig, f.name)
+
+ def testConfigQueryFailed(self):
+ """test groups with config and failed query"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: /bin/false
+#all:
+list: echo foo
+#reverse:
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertRaises(NodeSetExternalError, nodeset.regroup)
+
+ def testConfigRegroupWrongNamespace(self):
+ """test groups by calling regroup(wrong_namespace)"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo foo
+#reverse:
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertRaises(GroupResolverSourceError, nodeset.regroup, "unknown")
+
+ def testConfigNoListButReverseQuery(self):
+ """test groups with no list but reverse upcall"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example[1-100]
+#all:
+#list: echo foo
+reverse: echo foo
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+
+ def testConfigWithEmptyList(self):
+ """test groups with list upcall returning nothing"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example[1-100]
+#all:
+list: echo -n
+reverse: echo foo
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("example[1-100]", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+ self.assertEqual(nodeset.regroup(), "@foo")
+
+ def testConfigCrossRefs(self):
+ """test groups config with cross references"""
+ f = self.makeTestFile("""
+# A comment
+
+[Main]
+default: local
+
+[local]
+map: echo example[1-100]
+
+[other]
+map: echo @local:foo
+ """)
+ res = GroupResolverConfig(f.name)
+ nodeset = NodeSet("@other:foo", resolver=res)
+ self.assertEqual(str(nodeset), "example[1-100]")
+
+
+if __name__ == '__main__':
+ suite = unittest.TestLoader().loadTestsFromTestCase(NodeSetGroupTest)
+ unittest.TextTestRunner(verbosity=2).run(suite)
Property changes on: trunk/tests/NodeSetGroupTest.py
___________________________________________________________________
Added: svn:executable
+ *
Added: svn:keywords
+ Id
Modified: trunk/tests/run_testsuite.py
===================================================================
--- trunk/tests/run_testsuite.py 2010-03-09 13:18:22 UTC (rev 245)
+++ trunk/tests/run_testsuite.py 2010-04-07 22:51:33 UTC (rev 246)
@@ -41,6 +41,7 @@
"RangeSetErrorTest",
"NodeSetTest",
"NodeSetErrorTest",
+ "NodeSetGroupTest",
"NodeSetScriptTest",
"MisusageTest",
"MsgTreeTest",
This was sent by the SourceForge.net collaborative development platform, the world's largest Open Source development site.
|