[Woc-commit] CVS: woc/src/woc/cgi-src/htplpdir/editor Editor,NONE,1.1 Grammar.pm,NONE,1.1 Help.pm,NONE,1.1 INSTALL,NONE,1.1 README,NONE,1.1 conv,NONE,1.1 conv_syntax,NONE,1.1 dataConv.pm,NONE,1.1 dataEdit.pm,NONE,1.1 dataFunc.pm,NONE,1.1 dataIO.pm,NONE,1.1

[Geert J. <gj...@us...>]

Update of /cvsroot/woc/woc/src/woc/cgi-src/htplpdir/editor
In directory usw-pr-cvs1:/tmp/cvs-serv23171/woc/cgi-src/htplpdir/editor

Added Files:
	Editor Grammar.pm Help.pm INSTALL README conv conv_syntax 
	dataConv.pm dataEdit.pm dataFunc.pm dataIO.pm 
Log Message:




--- NEW FILE: Editor ---
#!/usr/local/bin/perl -w
#package editor;
# Polymorphic-data editing utility           by G.P.H. Josten         June, 1998
#
# This Perl package file contains general functions for showing, searching and
# editing data which is held in the @database list-variable. This variable, held
# in the main module, contains 'records' which are represented as single lines
# by separating the values by some string defined in $separator. The order of
# the data is defined in the hash-variable %order. This also names the fields of
# the records. Representation on screen is further defined in two hash-variables.
# The first called %distribution defines on what line each field of a record
# should be shown by giving per line all indexes in the right order. The second
# hash-variable called %keynames defines with what string each of those lines
# is preceded. This grammar is defined in a separate text-file and is read by the
# read_grammar sub-function in this module.
#
# Example:
#
# We have data that looks like:
#
#    e-mail#last name#city#first name
#
# The separator will be the guard ('#'). Then order will be:
#
#   %order =
#   ("email",     0,
#    "lastname",  1,
#    "city",      2,
#    "firstname", 3)
#
# Using this hash and identifiers like "city", you can find the index for city
# on a line of your data (@database). Likewise you define %distribution and
# %keynames:
#
#   %distribution =
#   (0, "3, 1",
#    1, "0",
#    2, "2")
#
#   %keynames =
#   (0, "name: ",
#    1, "mail: ",
#    2, "city: ")
#
# This shows that each record will be shown on screen in three lines. The first
# showing the name of a person, the second his or her e-mail address and the
# third and last the city where he or she comes from. As can be seen from the
# distribution hash-variable, the first line (indexed with 0) shows field three
# first, followed by field one. So first the first name, then the last name.
#
# Note:
# this is not the way you should put it in a grammar file. To make it easy for
# you, you don't have to index the lines explicitely. This is automatically
# derived from the precedence in the file. For the given example the file would
# look like:
#
#   dataseparator:
#   #
#   fieldseparator:
#   , 
#   order: 4
#   email
#   lastname
#   city
#   firstname
#   distribution: 3
#   3, 1
#   0
#   2
#   keynames: 3
#   name: 
#   mail: 
#   city: 
#   
# The string on the line below dataseparator is the separator for the fields
# of one line of the database hash-variable. fieldseparator is used when
# multiple fields are to be shown on one line on the screen, like first and
# last name in the exaple.
#
# Final note:
# this grammar file is quite spaces sensitive, so placing a space before or
# behind a keyname will output one!! Moreover, spaces on the line below
# dataseparator will require them exactly as separator of your fields in
# the database...
#

#use strict;
use dataConv;
use dataEdit;
use dataFunc;
use dataIO;
use Grammar;
use Help;


#
# The single argument denotes what is to be processed. It is a string
# with which the filenames are determined (just addition of .dat .grm
# .hlp and .new).
#
my ($mode, $current);
if (@ARGV) {
  $mode = $ARGV[0];
} else {
  $mode = "smoel";
}
my ($datafile) = $mode."data/$mode.dat";
my ($grammarfile) = $mode."data/$mode.grm";
$script = $0;
$script =~ s#.*\/##i;
my ($defhelpfile) = "$script.hlp";
my ($helpfile) = $mode."data/$mode.hlp";
my ($newdatafile) = $mode."data/$mode.new";

#$dataseparator = "#";
#$fieldseparator = ", ";
#$sep = "#";#old version
#$expsep = ", ";#old version



## main ##
print "$0 - \u$mode","Source Editor v2.0, Geert Josten, June 1998$/$/";

print "Reading data-grammar$/";
(*grammar) = &read_grammar ($grammarfile);

print "Reading database$/";
(*database, *commented_data) = &read_data ($datafile, "verbose");

print "Reading default help texts$/";
(*help) = &read_help ($defhelpfile, \@grammar, $datafile, $newdatafile);
if (-r $helpfile) {
  print "Reading additional help texts$/";
  (*addhelp) = &read_help ($helpfile, \@grammar, $datafile, $newdatafile);
  %help = (%help, %addhelp);
}
print $/;

#@oldstuds = @studenten;

$current = 1;
&show ([$current], \@database, \@grammar);

print "\thelp\tgeeft beschrijving commando's$/$/";
&editor;
print "Tot ziens!$/$/";


sub testje {
  return @_;
}

sub editor {
  local ($quit) = ""; #false
  local ($commando, @args);
  do {
    do {
      print "[$current] commando? ";
      chomp ($commando = <STDIN>);
    } until ($commando);
    @args = split (/ /, $commando);
    $commando = $args[0];
    shift (@args);
    if ($commando =~ /^d/i) {                # del
      @args = ($current) if (! @args);
      (*database) = &del (\@args, \@database, \@grammar, "verbose");
      $current = 1;
    } elsif ($commando =~ /^e/i) {           # edit
      @args = ($current) if (! @args);
      if ($args[0] =~ /\b\d+\b/) {
        &edit ($args[0], \@database, \@grammar, \%help);
        $current = $args[0];
      }
    } elsif ($commando =~ /^fil/i) {         # filter
      if (@args == 0) {
        &help (\%help, \@grammar, "search", "help");
      } elsif ((@args % 2) == 0) {
        (*database) = &filter (\@args, \@database, \@grammar, "verbose");
        $current = 1;
      } else {
        print "filter heeft een even aantal parameters nodig!$/";
      }
    } elsif ($commando =~ /^f/i) {           # find
      if (@args == 0) {
        &help (\%help, \@grammar, "search", "help");
      } elsif ((@args % 2) == 0) {
        (*args) = &find (\@args, \@database, \@grammar, "verbose");
        if (@args) {
          &show (\@args, \@database, \@grammar);
          print "gevonden: ", join (",", @args), $/;
          $current = $args[$#args];
        }
      } else {
        print "find heeft een even aantal parameters nodig!$/";
      }
    } elsif ($commando =~ /^h/i) {           # help
      &help (\%help, \@grammar, "commando", "help");
    } elsif ($commando =~ /^n/i) {           # new
      $last = @database;
      (*database) = &new_record (\@database, \@grammar, \%help);
      $current = @database if ($last < @database);
    } elsif ($commando =~ /^r/i) {           # read
      @args = ($datafile) if (! @args);
      (*database) = &read_data ($args[0], "verbose");
      $current = 1;
    } elsif ($commando =~ /^sa/i) {          # save
      @args = ($newdatafile) if (! @args);
      &save_data ($args[0], \@database, \@commented_data, "verbose");
    } elsif ($commando =~ /^so/i) {          # sort
      (*database) = &sort_data (\@database, "verbose");
      $current = 1;
    } elsif ($commando =~ /^s/i) {           # show
      @args = ($current) if (! @args);
      &show (\@args, \@database, \@grammar);
      $current = $args[0] if ($args[0] =~ /\b\d+\b/);
    } elsif ($commando =~ /^us/i) {          # usage
      &help (\%help, \@grammar, "commando", "usage");
    } elsif ($commando =~ /^u/i) {           # unfilter
#      &unfilter (@args);
      print "niet beschikbaar$/";
    } elsif ($commando =~ /^q/i) {           # quit
      $quit = "true";
    } elsif ($commando =~ /^\?/i) {          # ?
      &help (\%help, \@grammar, "commando", "afkort");
    } else {
      print "onbekend commando!$/";
    }
  } until ($quit);
}

--- NEW FILE: Grammar.pm ---
package Grammar;

# Polymorphic-data-grammar reading        by G.P.H. Josten          June, 1998
#
# This package contains the function to extract the data-grammar from a file.
#

use strict;
require Exporter;
@Grammar::ISA = qw(Exporter);
@Grammar::EXPORT = qw(read_grammar);

  sub read_grammar ($) {
    #
    # read in the grammar of the data from a grammar file
    #
    # PRE: $source contains name of grammar file
    # POST: %order, %distribution and %keynames contain the information
    #       necessary to retrieve fields from the data and show them on screen

    my ($source) = @_;
    my (%order, %distribution, %keynames, $regel, @regel, @lijst, $i);
    my ($dataseparator) = "#";
    my ($fieldseparator) = ", ";

    open (SOURCE, $source) || die $!;

    #
    # first read the order into %order:
    #
    defined ($regel = <SOURCE>) || die $!;
    @regel = split (" ", $regel);
    $i=0;
    @lijst = ();
    while ($i < $regel[1] &&  defined ($regel = <SOURCE>)) {
      chomp($regel);
      push (@lijst, $regel, $i);
      $i++;
    }
    %order = @lijst;

    #
    # second the distribution along multiple lines:
    #  
    defined ($regel = <SOURCE>) || die $!;
    @regel = split (" ", $regel);
    $i=0;
    @lijst = ();
    while ($i < $regel[1] && defined ($regel = <SOURCE>)) {
      chomp($regel);
      push (@lijst, $i, $regel);
      $i++;
    }
    %distribution = @lijst;

    #
    # last the names to show in front of each line:
    #
    defined ($regel = <SOURCE>) || die $!;
    @regel = split (" ", $regel);
    $i=0;
    @lijst = ();
    while ($i < $regel[1] && defined ($regel = <SOURCE>)) {
      chomp($regel);
      push (@lijst, $i, $regel);
      $i++;
    }
    push (@lijst, "comment", "/");
    %keynames = @lijst;

    close (SOURCE) || die $!;

    # return the references only!!

    return [\%order, \%distribution, \%keynames, $dataseparator, $fieldseparator];
  }

--- NEW FILE: Help.pm ---
package Help;

# Help-interface command and edit help       by G.P.H. Josten          June, 1998
# 
# This package contains functions to open the help-files, get the help texts and
# print the appropriate help on request.
#

use strict;
require Exporter;
@Help::ISA = qw(Exporter);
@Help::EXPORT = qw(help read_help);

  sub help ($$$$) {
    #
    # show the requested help
    #
    # PRE: $refhelp is a reference to a double hash table which is
    #      indexed with $function and $sublevel; $grammar is used to
    #      extract the help text from the single line format
    my ($refhelp, $grammar, $function, $sublevel) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;
    my (@help);
    @help = split (/$datasep/, $$refhelp{$function}{$sublevel});
    while (@help) {
      print shift (@help), "\t", shift (@help), $/;
    }
    print $/;
  }

  sub read_help ($$$$) {
    #
    # read in the help from a file
    #
    # PRE: $helpfile contains the name of the file to read, $grammar
    #      some information for parsing the file and $datafile and
    #      $newdatafile are used to substitute pseudo-variables with
    #      actual values
    # POST: result is a reference to a double hash table with the help
    #       texts compressed to single line format using $datasep from
    #       $grammar
    my ($helpfile, $grammar, $datafile, $newdatafile) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;
    my ($function, $sublevel, $regel, @help, %help);
    
    open (SOURCE, $helpfile) || die $!;
    
    # skip comment
    while (defined ($regel = <SOURCE>) && ($regel =~ /^$$refkeys{"comment"}/)) {
    }
    
    if (defined ($regel)) {
      do {
        chomp ($regel);
        ($function, $sublevel) = split (/\s/, $regel);
        while (defined ($regel = <SOURCE>) && ($regel !~ /^EOH/)) {
          chomp ($regel);
          $regel =~ s/\$datafile/$datafile/ig;
          $regel =~ s/\$newdatafile/$newdatafile/ig;
          push (@help, $regel);
        }
        $regel = join ($datasep, @help);
        $help{$function}{$sublevel} = $regel;
        @help = ();
      } while (defined ($regel = <SOURCE>));
    }

    close (SOURCE) || die $!;
    return (\%help);
  }

--- NEW FILE: INSTALL ---
Editor

Editor is a script to read, manipulate, search and save data. It is meant to be
self-explanatory, so for the usage I refer to the program. Requirement on the
data is, that it contains on each line one 'record' of which all have have an
equal number of fields. The length may range from zero to some physical limit.
Each field on such a line is separated from the others by some unique string (at
the moment simply a guard '#'!). The order of the fields should be the same on
all lines and this should be described in a separate file called the grammar
file. If not all these, the program may get confused from time to time.


System Requirements

-Perl 5 should do, I guess. I used the script with versions 5.003 and 5.004. Perl
4 doesn't support certain constructs, but converting all my's to local's might
make the script work under Perl 4 as well.
-Although I mostly worked on a Unix environment (SunOS 4 and 5.3), it should
work similarly on DOS, although you will probably have to do without long
filenames.
-Memory usage and file space aren't very relevant, although editing many
megabytes of data might get you into trouble. There are no mem-checks for this
script is probably far from efficient enough to edit that large quantaties.


Installation

The installation is fairly simple, but manual:
-Copy the following files to some nice directory:

        Editor		The main script
        Editor.hlp	Standard help texts
        INSTALL		This file
        README		Text file with additional information about Editor
        Grammar.pm	Module for reading and using the grammar of the data
        Help.pm		Module for initialising and using the help functions
	dataConv.pm	Module for conversion from record to basic representation
        dataFunc.pm	Module for searching and filtering records
        dataEdit.pm	Module for editing records
        dataIO.pm	Module for reading, saving and printing data
	exampledata/			dir with example data
        exampledata/example.dat		example data file
        exampledata/example.grm		grammar file of example data
        exampledata/example.hlp		additional help file for example data

-All you need now is some data and its grammar. Some additional help could be
useful too. Editor takes one argument and it uses it to identify what datafile
you want to read, search and edit and where it can find it. This first argument
is just a keyword. Editor takes this keyword and appends data. This identifies
the subdirectory where the data file and the grammar file should be located.
Editor then takes the keyword and appends .dat .grm and .hlp for respectively the
data, the grammar and the additional help file. Look at the example data for how
these three files are constructed...

-A last hint: it is most convenient to put the Editor script and its modules
somewhere in a bin (subdir editor orso) and make symbolic links to the
datadirectories you want to be able to edit. You could even put handy scripts
right next to the data, which changes the directory to the one that contains
the Editor script and then executes Editor with the right keyword. Note: don't
call that script edit, for that is probably an already existing command.


Geert
August, 1998

--- NEW FILE: README ---
August 5th, 1998

Why Htplp and Editor

Htplp and Editor are two Perl-scripts which were written to resolve problems
concerning the representation of large quantities of data on WWW as HTML. Htplp
is the Parser which generates the pages, Editor is the utility to edit the data.

Where it all started

The parent problem was a small already existing site of the student association
V.V.C.N. Sigma (http://www.sci.kun.nl/sigma/) which contained almost all 
pictures of the Chemistry students at the University of Nijmegen (most of them 
member of Sigma). Each year new students arrived and the pages had to be updated (each year of students meant three pages of each a number of kilobytes). I took 
that job on me (don't ask me why, I can't remember) and did this for one new 
year of students. My conclusion was that it was an unflexible, easily 
out-of-date and time consuming way. Everything had to be done manually. 
Especially when I (yes, one of those great ideas to do some self-torture) came 
to the idea to improve the layout by using multi-column layout for the pictures, I got tired of it. With some cut and paste it could be done, but it was boring 
and specially suited for computers to do it for you.

The first monster

The problems weren't very important, as the pages didn't needed to be very 
up-to-date. Nevertheless, it is a good visit card to do have a site with 
accurate and usable information. Besides, it was an interesting challenge which 
could make things easy in future. With my fresh knowledge of Java ("C1", the 
introductory course to programming - in Java :-( - was one of the obligatory 
courses), I set off to build a database and an engine that could read the data 
and generate the pages for me. An immense monster resulted. It worked, but it 
was ill-programmed and unreadable, only I worked with it - with reason I presume :-P -, and most of all, it was unflexible and still unmaintainable. But, a 
database was generated, one of the biggest benefits, and the page generation was automated. The engines most worth, it proved to me with it's first use outside 
it's original purpose, a use for like-wise layout, not for WWW but for a booklet with information about Sigma. I used this monster more than a year I believe, it has shown it's purpose, but now fortunately it's obsolete.

The data

The data which was held by a number of files, had to be created and maintained. 
This was a second problem, because using a text-editor isn't the best way, for 
mistakes are very easily made. Leaving some special symbol out meant that half 
the data of one year wasn't read, but skipped. Searching these files and 
extracting the useful information was another problem. Busy with the data - I 
was probably adding a new year or updating some names - I came to the idea of 
writing an editor. Actually, the idea originated because I wanted to facilitate 
the extraction of information. I was busy typing all fields that represented a 
single student. Life happened to let me stumble over Perl, I was interested and 
asked a 'colleague' student - he studied Informatics, I Chemistry at that time, 
Informatical Chemistry presently - for some advise about books about Perl. I 
bought to his advice two and with "Learning Perl" in one hand and the data files in the other, I tried to get a better and moreover faster extraction going. But 
using Unix shell-script (find, grep, sort, etc.) and Unix shell commands within 
Perl scripts didn't work and typing all information didn't facilitate it very 
much either this way. Well, why not read it in with Perl, put it in some 
variables, let Perl do some sorting and voila, my first real useful Perl-script 
was born. But with that it didn't stop. Hacking around in Perl, I wrote not only a search utility, but also some conversion utilities and another page generating script. But again, they were ill-programmed, hardly better readable, still only 
I (could) work(ed) with them and they still didn't show very much flexibility. 
The data improved though, for I typed all data, got some real regularity in it 
and got fresh ideas about how to better deal with it.

The new approach

I realised that the power of Perl could be far better utilised for this matter 
than I did. I hardly did, so that isn't much of a surprise. One benefit of Perl 
is, that normally it doesn't make a difference between words and numbers. I mean that in Perl for example not only numbers could be used to index in an array of 
items, but words as well. The Perl hash-variable the best example of this power 
of Perl in which one value is used to extract another, one to one. Not vice 
versa by the way. My fantasy got almost wild with the only enlarging space in 
which I could invent options and ideas and problems and solutions. Meanwhile, I 
had been busy with some funny command interpreting script which could represent 
the fields of data of one student in some basic way, could edit them as well and after some trouble, could even search the big lot. Discovering more and more 
about Perl, I started extracting more and more of the determining values and 
putting them into handy variables I could initialize at start and manipulated 
mid-way if necessary. This process resulted in a separate file which contained 
the full typing of the data and even the representation in the basic way. This 
information is read in and put in a few variables. In the rest of the script 
only these variables are used and not their actual value. Comparing with the 
actual values of these typing variables is only done with user input!! This 
process plus the ability of recognizing shortened versions of the actual values 
made the Editor very powerful.

The Editor evolved

I continued this work and started working on a profound way. I grouped certain 
functions and made them complete and robust. Most of them had to be converted to my new ideas and this allowed me to reconsider the solutions and improve them to their present state. This makes the Editor a full grown script, with fairly 
balanced functions and a quite user-friendly user-interface. I already had build an almost complete version of the Editor which I had subjected to some 
beta-testing (another colleague student :-). The new version could be tested 
back to back with this one and most testing did I do between all the other 
programming activities (hint: try to always test an earlier programmed part 
before you need to use it!). And still my knowledge about Perl is increasing, 
but the usefulness has converged about now - after building Htplp and Editor -.

The data structuring

The data had changed a lot in the meantime. First more regularity, second the 
typing and third cenversion. Conversion from the multi-line format (quite 
similar to the basic representation) to a single line one. The first two steps 
had been time consuming and text editor usage intensive, but are worth it now. 
The integrity of the data has very much increased as well as the managebility, 
editibility and usability. It also allowed me to experiment a little with how to structure the data. This has resulted in the separate file with the typing and 
the single-line format, with all fields of data of one student on a single line. This was done deliberately with several thoughts, no particular order, in my 
mind. Difficult to handle are fields of data of undefined number of lines. As 
linefeeds and carriage returns are of no use in HTML (it's just treated as 
whitespace) it was the first I got rid of. I stuffed everything on a single line and separated it by some specific string. At the moment I have chosen for a 
single character, the guard '#'. In near future the choice will be put in the 
typing file as well. I call these typing information together the grammar of the data, just to give it a name. Putting everything on a single line, makes the 
data easily manageble by using a simple array. This can even be sorted without 
any special work (one single command). Using the Perl functions split and join, 
the fields can be separated and glued together (single lines again). With the 
grammar of the data you can find out where a certain field remains and this can 
be use to extract, compare, manipulate and store the value if wanted. It may 
sound a bit difficult but it is just short and highly generic all at the same 
time, flexible at last and powerful indeed. It's real power became clear to me 
when I tried the Editor on a different data-file. It took two minutes to define 
a grammar and invent four lines of data. I ran the Editor and it just worked! It really worked!! Not a single change to the Editor was necessary, only the 
grammar and the data. I had underestimated my own abilities and the power of 
Perl.

What followed

With building this Editor which needs only some final commenting, addition of 
one or two extra functions and documentation on usage and maintenance, I created a profound basis of functions I saw I could use for the parent problem where I 
started my story. I had deliberately put all the Editor functions grouped in 
separate files, which could be used as nicely shielded modules. You only need to call the Perl function use and the functions are all yours. I had data, I had 
grammar and I had a whole bunch of functions, all tested and commented too. I 
took the page generating script in Perl in front of me and almost paniced in 
horror. It was awfull, HTML and Perl stood right next to each other, it was one 
big mess and in this script I had used old copies of the functions I had nicely 
defined in modules now. To get this readable, I had to get rid of the HTML 
between the Perl in some or the other way. By this time, I happened to look in 
some existing Perl script which gives some statistics, but more important, it 
read a plain text file in and substituted variables it contained (just a dollar 
sign '$' followed by an identifier). Interesting!! I though and once more I set 
off. But hey, I might could do better, I might be able to write some script 
which not only reads in those plain text files and substitutes some variables, 
but do this for any file, identified by some argument or so. Not a bad idea, I 
thought, so I hacked a bit in Perl, read in a file and substituted variables and produced the result. Hmm, well, I had to do a lot more if I wanted ever to 
produce the picture pages of Sigma. It is a tree-like structure with a lot of 
hyperlink refering to each other. Besides I had to process a lot of information 
if necessary. But, no worry, I thought, let the parser recognize filenames as 
variables as well (just dollar sign '$' plus filename) and process these too, 
Perl scripts as well. So I did, but being busy and stumbling over several
difficulties one becomes wiser and wiser.

The Htpl parser

After lots and lots of trial-and-error activities (regular expressions are very, very, very flexible in Perl :-P), this resulted in Htplp. The name is derived 
from HTml and PerL Parser, which is not completely accurate, but it will do. 
This script, or parser if you like to call it that way, starts with reading one 
argument. This argument determines what site it should parse and generate. From 
this argument some basic files are derived, a.o. the grammar file, the data 
file, the main Perl script needed too beside the parsing script and the main 
text file. The grammar and data are read, some default variables are set and 
next the main Perl script is executed. Actually, it is the only Perl script that is necessary for that particular site. This script is supposed to do something 
useful to every remaining argument the parser script received at start. With 
these arguments the script is given the opportunity to initialize, process and 
destroy variables which, and this is the trick, are automatically acessible from within the parsing script (they are just global variables, so it's no fancy 
trick at all and note that the parser script's variables are global too!!). Then the main text file is read in and the real parsing commences. This parsing is no real big deal, although it cost me a lot of fine tuning to get it working.

How the parsing works

This parsing means only that the parser tries to find a few posibilities and 
process accordingly. First, the parser could find the word "if" somewhere. If 
this "if" is followed by a variable ($varname), then it will try to determine 
wether this variable exists or not, if so, the following part remains, otherwise it is deleted. If it is followed in the same line by the word "fi", then 
everything in between might survive or not. No "fi", then if the variable isn't 
set by that site-specific script, than the full line is deleted. The if .. fi 
construction may be put halfway a line, if .... construction must commence at 
the beginning of a line. In both if constructions the variable may be directly 
preceded by an exclamation mark '!', which negates the required existence of the following variable. Other constructs are as mentioned previously, filenames 
($filename), with the only restriction that the filename must end with dot plus 
"htpl" ("$filenaam.htpl"). This is merely a handy choice which I'm not about to 
change. Final possibilities are just variable substitutions, scalar variables 
($varname) and array variables (@varname) as well. Scalar variables are just 
substituted by their content. Array variables are a bit trickier. I programmed 
the parser in such a way that it pops the first value (The Perl function shift), which is lost that way. I found it necessary to build in a last option, to not 
pop that first value, but look at it and possibly use it once again. For this, 
put a exclamation mark '!' directly behind the @ in @varname (e.g. @!varname). 
This ought to do the trick :-). After each substitution in a line, the parser 
tries again, untill it fully fails, then it produces that line. Going over each 
line exhaustively allows to parse highly nested structures as well (variable 
names in variable names in variable names in a filename in an array variable in 
an variable name in an if evaluation in...). This may sound like a lot of work, 
but Perl manages to do this for a small but difficult page, within 10 seconds 
(at our server at least, which is not the slowest around). Especially the 
variable substitutions, which are often the far most done, influence this time 
greatly. Try to keep the actions as minimal as possible, by presubstituting all 
those values if possible. This doesn't have to be done exhaustively, the script 
is fast enough (Perl is compiled before execution!!). If it takes too long, your page might be just too long or perhaps, the server is slow in executing 
scripts...

Finally

I hope that this Editor and Htpl Parser may find a lot of future applications 
and not only done by me. I build these to give them through to future members of the Internetcommittee of V.V.C.N. Sigma. For as long as I will still be around I will try to do some final modifications in order to make those future 
applications even more possible. Let me note, that the Editor script or some 
text file nearby that script will contain a more extensive explenation of the 
use of the grammar. The Editor itself contains a help function which should 
explain the Editor functions at least a little. The error messages should do the rest, I hope. Htplp should be installed by somebody with at least some Perl 
experience and he or she could take care of the main site scripts if they are 
needed. Any layouting can be done by somebody else, like some HTML wizard or so. Minimal knowledge of how to call the script with the right arguments and what 
variables are available in which situation is necessary though. A documenting 
file would be very helpfull I suppose. And now the really, really final note: I 
called the Parser HTml and PerL Parser, but, actually, the HTML part doesn't 
have to be HTML at all, anything based on plain text could do. LaTeX for example too.

May you all have a pleasent exercise of mind with scripts, may they enlighten
your life, may they relieve you of a lot of trouble and sorrow and sweat and 
pain. If they don't, I suggest: R.T.F.M.!!!!!!!!


Signed,
WoodWorm alias G.P.H. Josten

student Informatical Chemistry at the University of Nijmegen
member of student association Sigma at Nijmegen
member of the internetcommittee of Sigma

--- NEW FILE: conv ---
#!/usr/local/bin/perl -w
use strict;
my ($SmoelenSource, $SortedSmoelen, $AccessSmoelen);
$SmoelenSource = ($ARGV[0] || "SmoelenSource.Sm");
$SortedSmoelen = ($ARGV[2] || "SmoelenSource.sorted");
$AccessSmoelen = ($ARGV[1] || "SmoelenSource");

print "controleer $SmoelenSource:\n";
&controleer ($SmoelenSource);
print "converteer $SmoelenSource naar $AccessSmoelen\n";
&Sm2Acc ($SmoelenSource, $AccessSmoelen);
#print "controleer $SmoelenSource:\n";
#&controleer ($SmoelenSource);
#print "sorteer $SmoelenSource tot $SortedSmoelen\n";
#&sorteer ($SmoelenSource, $SortedSmoelen);
#print "controleer $SmoelenSource:\n";
#&controleer ($SmoelenSource);
#print "controleer $SortedSmoelen:\n";
#&controleer ($SortedSmoelen);


sub sorteer {
  my ($source, $target) = @_;
  my ($temp, $temp2);

  $temp = "/tmp/SPE.gjosten.presort.tmp";
  $temp2 = "/tmp/SPE.gjosten.postsort.tmp";
  &Sm2Acc ($source, $temp);
  !system ("sort $temp > $temp2") || die ("SPE kan sort niet aanroepen: $!");
  &Acc2Sm ($temp2, $target);
}

sub Sm2Acc {
  my ($source, $target) = @_;
  my ($tabelregel, @tabelregels, $status, $voornaam, $achternaam, $spreuk, $anker, $plaatje, $i);
  my ($jaar, $stdnr, $email);

  open (SOURCE, $source) || die ("SPE kan $source niet openen: $!");
  while (defined ($status = <SOURCE>)) {
    chomp ($status);
    while ($status =~ /^\/.*/) { #commentaar overslaan
      defined ($status = <SOURCE>) || die ("Onverwacht einde van $source: $!");  
    }

    defined ($voornaam   = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($achternaam = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($spreuk     = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($anker      = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($plaatje    = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    chomp ($status);
    $jaar = substr ($status, 1, 2);
    $stdnr = substr ($status, 4, 7);
    $status = substr ($status, 12);
    chomp ($voornaam);
    $voornaam = substr ($voornaam, 1);
    chomp ($achternaam);
    chomp ($spreuk);
    $spreuk = substr ($spreuk, 1);
    chomp ($anker);
    $anker = substr ($anker, 1);
    (($anker =~ /^@/i) && ($email = substr ($anker, 1))) || ($email = "");
    chomp ($plaatje);
    ($plaatje =~ /stimpy\.gif/i) && ($plaatje = "stimpy.gif");

    $tabelregels[@tabelregels] = join ("#", $jaar, $achternaam, $voornaam, $stdnr, $status, $spreuk, $anker, $email, $plaatje);
  }
  close (SOURCE) || die ("SPE kan $source niet sluiten: $!");

  open (TARGET, ">$target") || die ("SPE kan $target niet openen: $!");
  select (TARGET);
  for ($i = 0; $i < @tabelregels; $i++) {
    print ($tabelregels[$i]."\n") || die ("SPE kan gegevens niet naar $target schrijven: $!");
  }
  select (STDOUT);
  close (TARGET) || die ("SPE kan $target niet sluiten: $!");
}


sub Acc2Sm {
  my ($source, $target) = @_;
  my ($tabelregel, @tabelregels, $gegeven, $status, $voornaam, $achternaam, $spreuk, $anker, $plaatje, $i);

  open (SOURCE, $source) || die ("SPE kan $source niet openen: $!");
  while (defined ($tabelregel = <SOURCE>)) {
    chomp ($tabelregel);
    $tabelregels[@tabelregels] = $tabelregel;
  }
  close (SOURCE) || die ("SPE kan $source niet sluiten: $!");

  open (TARGET, ">$target") || die ("SPE kan $target niet openen: $!");
  select (TARGET);
  print ("///// $target /////\n") || die ("SPE kan gegevens niet naar $target schrijven: $!");
  for ($i = 0; $i < @tabelregels; $i++) {
    ($achternaam, $voornaam, $status, $spreuk, $anker, $plaatje) = split ("\\\\\\\\", $tabelregels[$i]);

    foreach $gegeven ("/// ".($i+1)." ///", $status, $voornaam, $achternaam, $spreuk, $anker, $plaatje) {
      (print $gegeven."\n") || die ("SPE kan gegevens niet naar $target schrijven: $!");
    }
  }
  select (STDOUT);
  close (TARGET) || die ("SPE kan $target niet sluiten: $!");
}


sub controleer {
  my ($source) = @_;
  my ($status, $voornaam, $achternaam, $spreuk, $anker, $plaatje, $regelnr, $prev, $next);

  open (SOURCE, $source) || die ("SPE kan $source niet openen: $!");
  $regelnr = 0;
  while (defined ($status = <SOURCE>)) {
    $regelnr++;
    chomp ($status);
    while ($status =~ /^\/.*/) { #commentaar overslaan
      $prev = $status;
      $regelnr++;
      defined ($status = <SOURCE>) || die ("Onverwacht einde van $source regelnr $regelnr: $!");
      chomp ($status);
    }

    if ($status =~ /^:..:.......:.*/) {
      #mooi!
      $regelnr++;
      defined ($voornaam = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
      chomp ($voornaam);
    }# else {
      #probeer resynchronisatie
   #   if ($prev =~ /^:..:.......:.*/) {
   #     print ("SPE $source: STAPJE TERUG");
   #     $status = $prev;
   #     $voornaam = $status;
   #   } else {
   #     $regelnr++;
   #     defined ($next = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
   #     chomp ($next);
   #     if ($next =~ /^:..:.......:.*/) {
   #       print ("SPE $source: STAPJE NAAR VOREN");
   #       $status = $next;
   #       $regelnr++;
   #       defined ($voornaam = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
   #       chomp ($voornaam);
   #     } else {
   #       print ("SPE $source: RESYNCHRONISATIE MISLUKT");
   #       print ("SPE $source: status syntaxfout op $regelnr\n");
   #       $voornaam = $next;
   #     }
   #   }
   # }

    $voornaam =~ /^~.*/             || print ("SPE $source: voornaam syntaxfout op $regelnr\n");

    $regelnr++;
    defined ($achternaam = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
    
    $regelnr++;
    defined ($spreuk     = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
    chomp ($spreuk);
    $spreuk   =~ /^\#.*/             || print ("SPE $source: spreuk syntaxfout op $regelnr\n");

    $regelnr++;
    defined ($anker      = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
    chomp ($anker);
    $anker    =~ /^\@.*/             || print ("SPE $source: anker syntaxfout op $regelnr\n");

    $regelnr++;
    defined ($plaatje    = <SOURCE>) || die ("Onverwacht einde van $source $regelnr: $!");
    chomp ($plaatje);
    $plaatje =~ /^.*\.(gif|jpg).*/ || print ("SPE $source: status syntaxfout op $regelnr\n");
    $prev = $plaatje;
  }
  close (SOURCE) || die ("SPE kan $source niet sluiten: $!");
}

--- NEW FILE: conv_syntax ---
#!/usr/local/bin/perl -w
my ($source, $target) = ($ARGV[0], $ARGV[1]);
my ($tabelregel, @tabelregels, $status, $voornaam, $achternaam, $spreuk, $anker, $plaatje, $i);

$i = 0;

  open (SOURCE, $source) || die ("SPE kan $source niet openen: $!");
  open (TARGET, ">$target") || die ("SPE kan $target niet openen: $!");

  while (defined ($status = <SOURCE>)) {
    $i++;
    chomp ($status);
    while ($status =~ /^\/.*/) { #commentaar overslaan
      defined ($status = <SOURCE>) || die ("Onverwacht einde van $source: $!");  
    }

    defined ($voornaam   = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($achternaam = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($spreuk     = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($anker      = <SOURCE>) || die ("Onverwacht einde van $source: $!");
    defined ($plaatje    = <SOURCE>) || die ("Onverwacht einde van $source: $!");

    $status =~ s/^://;
    $status =~ s/:[^0-9]{7}:/,, /;
    $status =~ s/:/, /;
    $status =~ s/:/, /;
    chomp ($achternaam);
    $voornaam =~ s/^~//;
    $spreuk =~ s/^#//;
    $anker =~ s/^@//;

    print STDOUT "/// $i ///$/cd: ${status}nm: $achternaam, ${voornaam}sp: ${spreuk}an: ${anker}ft: $plaatje$/" || die ("SPE kan gegevens niet naar $target schrijven: $!");

  }
  close (SOURCE) || die ("SPE kan $source niet sluiten: $!");
  close (TARGET) || die ("SPE kan $target niet sluiten: $!");

--- NEW FILE: dataConv.pm ---
package dataConv;

# Polymorphic-data convert and show         by G.P.H. Josten          June, 1998
# 
# This package contains functions convert some database data from single line
# format to multiple line format and show it this way on screen.
#

use strict;
require Exporter;
@dataConv::ISA = qw(Exporter);
@dataConv::EXPORT = qw(compress expand show);

  sub by_number {
    #
    # function used by sort to sort numerical instead of alphanumerical
    #
    if ($a < $b) {
      return -1;
    } elsif ($b < $a) {
      return 1;
    } else {
      return 0;
    }
  }

  sub compress ($$) {
    #
    # compress data fields from over multi lines onto single lines
    #
    # PRE: $refdata, a reference to the data in multiline format to be
    #      compressed, $reforder, $refdistr and $refkeys, references to
    #      the hash variables that define how to translate from multi to
    #      single line and $datasep and $fieldsep the strings that
    #      separate the fields in the single resp. multi line format
    # POST: the data in single line format

    my ($refdata, $grammar) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;
    my (@result, $num_lines, $line, $i, @pos, @gegevens);

    $num_lines = scalar (keys (%$refdistr));

    while (@$refdata) {
      do {
        $line = shift (@$refdata);
      } while ($line =~ /$$refkeys{"comment"}/i);
      for ($i = 0; $i < $num_lines; $i++) {
        $line =~ tr/^$$refkeys{$i}//;
        @pos = split(/$fieldsep/, $$refdistr{$i});
        @gegevens[@pos] = split(/$fieldsep/, $line);
      }
      shift (@$refdata); # empty line
      push (@result, join ("$datasep", @gegevens));
    }
    return \@result;
  }

  sub expand ($$$) {
    #
    # expand one single line of data to multiline format
    #
    # PRE: $refdata, a reference to the data in single line format from
    #      which one record is to be expanded; $reforder, $refdistr and
    #      $refkeys, references to the hash variables that define how to
    #      translate from single to multi line format and $datasep and
    #      $fieldsep the strings that separate the fields in the single
    #      resp. multi line format
    # POST: one data record in multi line format

    my ($index, $refdata, $grammar) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;
    my (@data, @result, @lines, @fields) = ((), (), (), ());
    my ($line);    # show each of the list by expanding them one by one

    @data = split (/$datasep/, $$refdata[$index-1]);
    @lines = (0..(scalar (keys (%$refdistr)) - 1));

    # add numbering of output
    push (@result, "/// ".$index." ///$/");
    # add each line with the apropriate datafields
    foreach (@lines) {
      # which fields?
      @fields = split(/$fieldsep/, $$refdistr{$_});
      # add line identifying string
      $line = $$refkeys{$_};
      # add the fields
      $line .= join ("$fieldsep", @data[@fields]);
      push (@result, "$line$/");
    }

    return @result;
  }

  sub show ($$$) {
    #
    # show a given number of record from the data
    #
    # PRE: $reflist, a reference to a list with all indices to the records
    #      from the data to be shown;
    #      $refdata, a reference to the data in single line format from
    #      which one record is to be expanded; $reforder, $refdistr and
    #      $refkeys, references to the hash variables that define how to
    #      translate from single to multi line format and $datasep and
    #      $fieldsep the strings that separate the fields in the single
    #      resp. multi line format

    my ($reflist, $refdata, $grammar) = @_;
    my (@listtoshow) = @$reflist;

    # got a number?
    if (@listtoshow) {
      # show all?
      if ($listtoshow[0] =~ /^all/i) {
        @listtoshow = (1..@$refdata);
      } else {
      # else sort the received list
        @listtoshow = (sort by_number (@$reflist));
      }

      # show each of the list by expanding them one by one
      foreach (@listtoshow) {
        print &expand ($_, $refdata, $grammar), $/;
      }
    }
  }

--- NEW FILE: dataEdit.pm ---
package dataEdit;

# Polymorphic-data edit                      by G.P.H. Josten         June, 1998
# 
# This package contains the function to access your data (@database) and
# change any value you like. Includes help handling.
#

use strict;
require Exporter;
@dataEdit::ISA = qw(Exporter);
@dataEdit::EXPORT = qw(edit);

use dataConv;
use Help;

  sub edit ($$$$) {
    #
    # show data and provide an interface to edit all fields
    #
    # PRE: 
    #

    my ($nr, $refdata, $grammar, $refhelp) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;

    my (@gegevens) = split (/$datasep/, $$refdata[$nr-1]);
    my (@args, $gegeven, @known);
    my (@keynames) = sort (keys (%$reforder));

    &show ([$nr], $refdata, $grammar);

    do {
      # get a user command
      print "[$nr] edit? ";
      chomp ($gegeven = <STDIN>);

      if ($gegeven) {
        # extract command from arguments
        ($gegeven, @args) = split (/\s/, $gegeven);
        
        if ($gegeven =~ /^h/i) {
        # help

          &help ($refhelp, $grammar, "edit", "help");

        } elsif ($gegeven =~ /^\?/i) {
        # ?

          &help ($refhelp, $grammar, "edit", "afkort");

        } elsif ($gegeven =~ /^q/i) {
        # quit

          $gegeven = "";
        } else {
        # fieldname?

          # look how many fieldnames match the given description
          @known = grep {$_ =~ /^$gegeven/i} @keynames;

          if (@known > 1) {
          # more than one match
      
            print "meer mogelijkheden:$/@known$/";
          } elsif (@known > 0) {
          # one match
      
            if (@args) {
              $gegevens[$$reforder{$known[0]}] = join (" ", @args);
              $$refdata[$nr-1] = join ("$datasep", @gegevens);
            } else {
              print $known[0],"? ";
              chomp ($gegevens[$$reforder{$known[0]}] = <STDIN>);
              $$refdata[$nr-1] = join ("$datasep", @gegevens);
            }
            &show ([$nr], $refdata, $grammar);

          } else {
          # no matches
      
            print "mogelijkheden:$/@keynames$/$/";
            print "help\tgeeft beschrijving gegevens$/$/";
          }
        }
      }
    } while ($gegeven);
    # quit or no command
  }

--- NEW FILE: dataFunc.pm ---
package dataFunc;

# Polymorphic-data del, find, filter, get, new  by G.P.H. Josten     June, 1998
# 
# This package contains functions to delete, find records, filter them out or
# build new ones.
#

use strict;
require Exporter;
@dataFunc::ISA = qw(Exporter);
@dataFunc::EXPORT = qw(del filter find get new_record sort_data undo);

use dataConv;

  sub by_number {
    #
    # function used by sort to sort numerical instead of alphanumerical
    #
    if ($a < $b) {
      return -1;
    } elsif ($b < $a) {
      return 1;
    } else {
      return 0;
    }
  }

  sub del ($$$$) {
    #
    # delete records from the database, given some indices
    #
    # PRE: $reflist is a reference to the list with indices to the
    #      records that should be deleted, $refdata the reference to
    #      the data list; if $verbose is set true, give some output
    # POST: reference to the updated data list
    
    my ($reflist, $refdata, $grammar, $verbose) = @_;
    my ($nr, $antwoord, $i, $j, @checked);
    
    foreach (sort by_number (@$reflist)) {
      &show ([$_], $refdata, $grammar);
      print "verwijderen? ";
      chomp ($antwoord = <STDIN>);
      if ($antwoord =~ /^(j|y)/i) {
        push (@checked, $_);
      }
    }

    my (@new) = ();
    $j = 1;
    foreach (@checked) {
      for ($i = $j; $i < $_; $i++) {
        push (@new, $$refdata[$i-1]);
      }
      $j = $_+1;
    }
    for ($i = $j; $i <= @$refdata; $i++) {
      push (@new, $$refdata[$i-1]);
    }
    
    print "verwijderd zijn: ", join (",", @checked), " overgebleven aantal: ",
           $#new+1, $/ if (@checked && $verbose);

    return (\@new);
  }

  sub filter ($$$$) {
    #
    # grep records out of the database given some criteria
    #
    # PRE: $reffilters is a reference to a number of pairs strings
    #      of which the first identifies a unique field and the second
    #      is used to match the value; $grammar is used to extract the
    #      fields from the data, $verbose to show some output on screen
    # POST: reference to the remaining data
    
    my ($reffilters, $refdata, $grammar, $verbose) = @_;
    my ($reflist, $gedelete, $one, $other);
    my (@result) = ();

    ($reflist) = &find ($reffilters, $refdata, $grammar, $verbose);

    $one = 1;
    while (@$reflist) {
      if ($one < $$reflist[0]) {
        push (@result, $one);
        $one++;
      } else {
        shift @$reflist;
        $one++;
      }
    }
    while ($one <= @$refdata) {
      push (@result, $one);
      $one++;
    }
    
    @result = @$refdata[map {--$_} @result] if @result;

    $gedelete = @$refdata-@result;
    print "aantal weggefilterd: ", $gedelete, ", aantal overgebleven: ",$#result+1,$/
      if (($gedelete > 0) && $verbose);
    return \@result;
  }

  sub find ($$$$) {
    #
    # find records given some criteria
    #
    # PRE: $reffilters is a reference to a number of pairs strings
    #      of which the first identifies a unique field and the second
    #      is used to match the value; $grammar is used to extract the
    #      fields from the data, $verbose to show some output on screen
    # POST: reference to the list with found matches
    
    my ($reffilters, $refdata, $grammar, $verbose) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;
    my (%filters) = @$reffilters;
    
    my (@known, @filtered);
    my (@result) = (0..$#$refdata);
    my ($failure) = "";
    my (@keynames) = sort (keys (%$reforder));
    my ($gegeven, $string, $waarde, @gegevens);
    

    while (!$failure && (($gegeven, $string) = each (%filters))) {
      @filtered = ();

      # look how many fieldnames match the given description
      @known = grep {$_ =~ /^$gegeven/i} @keynames;

      if (@known > 1) {
      # more than one match
      
        print "meer mogelijkheden met $gegeven:$/@known$/" if $verbose;
        @result = ();
        $failure = "true";
      
      } elsif (@known > 0) {
      # one match
      
        foreach (@result) {
          @gegevens = split (/$datasep/, $$refdata[$_]);
          # select the fieldname that matched the description
          ($waarde = $gegevens[$$reforder{$known[0]}])
            || ($waarde = "");
          # keep line if value contains string
          push (@filtered, $_) if ($waarde =~ /$string/i);
        }
        @result = @filtered;
        @filtered = ();
      
      } else {
      # no matches
      
        print "mogelijkheden:$/@keynames$/$/help\tgeeft beschrijving gegevens$/$/" if $verbose;
        $failure = "true";
        @result = ();
      
      }
    }

    # add to all indices one to get a numbering from 1 to # data
    @result = map {++$_} @result;

    if (@result && $verbose) {
      print "gevonden zijn: ", join (",", @result), $/;
    }
    return \@result;
  }

  sub get ($$$$) {
    #
    # returns a list of all selected values
    #
    # PRE: $reflist is a reference to a list of all fieldnames of which
    #      values must be returned, $refdata a reference to the data;
    #      $grammar is used to extract the fields from the data, $verbose
    #      to show some output on screen
    # POST: reference to the list with all matching values
    
    my ($reflist, $refdata, $grammar, $verbose) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;
    my (@fields) = @$reflist;

    my (@result) = ();
    my ($failure) = "";
    my (@keynames) = sort (keys (%$reforder));
    my ($field, @gegevens, @known);

    my (@positions) = ();

    if ($fields[0] =~ /^all/i) {

      @positions = sort (values (%$reforder));

    } else {
      while (!$failure && ($field = shift (@fields))) {
        # look how many fieldnames match the given description
        @known = grep {$_ =~ /^$field/i} @keynames;

        if (@known > 1) {
        # more than one match
      
          print "meer mogelijkheden met $field:$/@known$/" if $verbose;
          @positions = ();
          $failure = "true";
      
        } elsif (@known > 0) {
        # one match
      
          push (@positions, $$reforder{$known[0]});

        } else {
        # no matches
      
          print "mogelijkheden:$/@keynames$/$/help\tgeeft beschrijving gegevens$/$/" if $verbose;
          $failure = "true";
          @positions = ();
      
        }
      }
    }

    foreach (0..(@$refdata - 1)) {
      @gegevens = split (/$datasep/, $$refdata[$_]);
      push (@result, join ("$datasep", @gegevens[@positions]));
    }
    
    return \@result;
  } 

  sub new_record ($$) {
    #
    # returns updated data
    #
    # PRE: $refdata is a reference to the datalist that should be sorted;
    #      $grammar is used to extract the fields from the data, $verbose
    #      to show some output on screen
    # POST: reference to the sorted datalist
    
    my ($refdata, $grammar, $refhelp) = @_;
    my ($reforder, $refdistr, $refkeys, $datasep, $fieldsep) = @$grammar;

    my (%invorder, $key, $aantal, $i, $notdone, $record, $value, $last, $newdata);
    
    foreach $key (keys(%$reforder)) {
      $invorder{$$reforder{$key}} = $key;
    }

    $last = @$refdata+1;
    $record = "";
    $aantal = keys(%$reforder);
    $i = 0;
    $notdone = "true";
    print "[$last] geef de waarden:$/";
    while ($i < $aantal && $notdone) {
      print $invorder{$i},"? ";
      if (defined($value = <STDIN>)) {
        chomp ($value);
        $value = " " if !$value;
        $record .= $value;
        $record .= $datasep if ($i < $aantal-1);
      } else {
        $notdone = "";
      }
      $i++;
    }
    
    if ($notdone) {
      $newdata = [@$refdata, $record];
      $last = @$newdata;
      print $/;
      &show ([$last], $newdata, $grammar);
      print "toevoegen? ";
      defined ($value = <STDIN>) || ($value = "nee");
      if ($value =~ /^(j|y)/i) {
        $refdata = $newdata;
        print "$/1 record toegevoegd...$/";
      } else {
        print "$/geen wijzigingen aangebracht...$/";
      }
    } else {
      print "$/geen wijzigingen aangebracht...$/";
    }
    
    return $refdata;
  }

  sub sort_data ($$) {
    #
    # returns alphanumerically sorted data
    #
    # PRE: $refdata is a reference to the datalist that should be sorted;
    #      $grammar is used to extract the fields from the data, $verbose
    #      to show some output on screen
    # POST: reference to the sorted datalist
    
    my ($refdata, $verbose) = @_;

    (print (@$refdata+1," records alphanumeriek gesorteerd$/")) if ($verbose);
    return [sort (@$refdata)];
  }

#sub unfilter {
#  @studenten = @unfilter;
#}

--- NEW FILE: dataIO.pm ---
package dataIO;

# Polymorphic-data read, save and print      by G.P.H. Josten          June, 1998
# 
# This package contains functions to read, save and print the database data line
# per line.
#

use strict;
require Exporter;
@dataIO::ISA = qw(Exporter);
@dataIO::EXPORT = qw(read_data save_data print_data);
@dataIO::EXPORT_OK = qw(read_data save_data print_data);

  sub read_data ($$){
    #
    # read in data from a file
    #
    # PRE: $bron contains the name of the data file and $verbose is set to true
    #      if the number of data read must be shown

    my ($bron, $verbose) = @_;
    my (@studenten, @commented_data, $regel, $comment);
    open (SOURCE, $bron) || die "$!: $bron";

    @commented_data = ();
    @studenten = ();
    $comment = "\/\/";
    while (defined ($regel = <SOURCE>)) {
      chomp ($regel);
      if ($regel =~ /^$comment/i) {
        push (@commented_data, $regel);
      } else {
        push (@studenten, $regel);
      }
    }

    close (SOURCE) || die "$!: $bron";
    print $#studenten+1," studenten ingelezen uit $bron$/" if $verbose;

    return (\@studenten, \@commented_data);
  }

  sub save_data {
    #
    # save given data to a file
    #
    # PRE: $doel contains the name of the file to which the data must be saved
    #      *studenten is a reference to a data list and $verbose determines
    #      wether the number of lines written is shown

    my ($doel, $refstuds, $refcommentdata, $verbose) = @_;
    open (TARGET, ">$doel") || die "$!: $doel";

    foreach (@$refcommentdata) {
      print TARGET $_,$/;
    }

    foreach (@$refstuds) {
      print TARGET $_,$/;
    }

    close (TARGET) || die "$!: $doel";
    print @$#refstuds+1," studenten weggeschreven naar $doel$/" if $verbose;
  }

  sub print_data {
    print @_,$/;
  }