Re: [Ocaml-lib-devel] test harness generator

[Janne H. <ja...@hy...>]

Hi John,

this looks promising.  After thinking about this a little while, I've 
come to the conclusion that the test harness generator is not such a bad 
idea after all.

I have a few suggestions though (roughly in order of priority):

1. Create and build tests in some tmp build directory.  How about 
building a library of the tests and then link that into the main testing 
suite to create an executable?  The main motivation behind is just to 
not clutter the test suite source directory.

2. Are you sure we need the .tst extension?  I'd like to put all the 
tests into a separate directory and just use .ml extension?  .ml is nice 
so that Emacs automatically selects tuareg-mode (and I don't want to 
change my Emacs confs :)). IIRC you also wanted to put the actual tests 
into a subdir.  If you believe .tst is needed, then how about naming 
them "test_xyz_*.ml.tst"?

3. More thorough error checking and logging.  I understand there's not 
much error checking since this is the first version.  Could you print 
out the command line for each ocamlc call?

4. Filtering of test cases.  Could you make a command line option for 
specifying what tests are included in the test?  This would be useful 
when developing the actual tests (e.g., I would only turn on "jh*" tests 
  when writing my own tests).

5. Add some options for generating ocamlprofs out of the testing suite 
and ExtLib?  Code coverage reports are really helpful when trying to 
make the test suite more extensive.

As for the near future.. Once 1&2 are done/agreed, I'd like to integrate 
this into the suite.

ciao,
janne

skaller wrote:
> I have attached first cut of a test harness generator.
> (On my box it all runs, but one of the test cases now segfaults).
> 
> to run it use this:
> 
> ocamlc -o mktest unix.cma
> str.cma mktest.ml && ./mktest && ./extlib_test
> 
> from a COPY (PLEASE!!) of extlib-test directory.
> 
> This program makes a check for ALL extlib modules.
> It searches for files in the current (extlib-test)
> directory with this exact format:
> 
> test_<author>_<module>_<test_name>.tst
> 
> Note the test name is mandatory, and the extension must be .tst.
> 
> It generates files:
> 
> extlib_test.ml
> mtest_<module>.ml
> itest_<author>_<module>_<test_name>.ml
> 
> and compiles and links everything together, including
> the utility util.ml. Bytecode only at the moment.
> 
> To make this work, all individual tests mut be renamed
> to the correct format.
> 
> Note: at present the only thing done to a *.tst file
> is to copy it to a *.ml file without touching anything inside.
> 
> This code is *not supposed* to clobber any existing files.
> 
> 
> 
> 
> ------------------------------------------------------------------------
> 
> (* This program generates the test harness *)
> (* This program must be executed from the directory
> 
>    extlib-test
> 
>    and extlib itself must be located in the directory
> 
>    extlib-dev
> *)
> 
> (* Step 1: find a list of all ExtLib modules *)
> 
> let extlib_dev_dir = 
>   Filename.concat Filename.parent_dir_name "extlib-dev"
> ;;
> 
> (* get all the files in a directory as a list, the names
>   do not include the directory pathname
> *)
> let find_files dirname =
>   let dir_handle = Unix.opendir dirname in
>   let all_files = 
>     let files = ref [] in
>       let rec loop () =
>         let filename = Unix.readdir dir_handle in
>         files := filename :: !files;
>         loop()
>       in try loop() with End_of_file -> !files
>   in
>   Unix.closedir dir_handle;
>   all_files
> ;;
> 
> (* filter a list of strings with a regular expression *)
> let filter_files rexp files =
>   let crexp = Str.regexp rexp in
>   List.filter
>   (fun s -> Str.string_match crexp s 0)
>   files
> ;;
> 
> (* regexp for finding *.mli files *)
> let mli_re = "^.+\\.mli$"
> ;;
> 
> (* regexp for eliding .mli extension *)
> let mli_rex = "^\\(.+\\)\\.mli$"
> ;;
> 
> (* find the modules in a directory, assuming
> each .mli file represents a module. The names
> are have leading character capitalised.
> *)
> 
> let modules_of dirname =
>   let crexp = Str.regexp mli_rex in
>   List.map
>   (fun s-> 
>     ignore(Str.string_match crexp s 0); 
>     let s = Str.matched_group 1 s in
>     s.[0] <- Char.uppercase s.[0];
>     s
>   )
>   (filter_files mli_re (find_files dirname))
> ;;
> 
> let all_modules = (modules_of extlib_dev_dir)
> ;;
> 
> List.iter
> (fun s-> print_endline s)
> all_modules
> ;;
> 
> (* now make the top level test harness *)
> let mk_top () =
>   let f = open_out "extlib_test.ml" in
> 
>   output_string f "let main() = \n";
>   output_string f "  Util.log \"ExtLib tester started\";\n";
>   List.iter
>   (fun s->
>     output_string f ("  Mtest_" ^ s ^ ".test ();\n")
>   )
>   all_modules
>   ;
>   output_string f "  Util.log \"ExtLib tests completed\"\n";
>   output_string f ";;\n";
>   output_string f "\n";
>   output_string f "main();;\n";
>   close_out f
> ;;
> 
> mk_top()
> ;;
> 
> (* now make the individual module tests *)
> 
> (* regexp for finding *.tst files *)
> let tst_re = "^test_.+_.+_.+\\.tst$"
> ;;
> 
> (* regexp for decoding test names *)
> let tst_rex = "^test_\\(.+\\)_\\(.+\\)_\\(.+\\)\\.tst$"
> ;;
> 
> 
> let tests_of dirname =
>   let h = Hashtbl.create 97 in
>   let crexp = Str.regexp tst_rex in
>   List.iter
>   (fun s-> 
>     ignore(Str.string_match crexp s 0); 
>     let author_key = Str.matched_group 1 s in
>     let module_key = Str.matched_group 2 s in
>     let test_key = Str.matched_group 3 s in
>     Hashtbl.add h module_key (author_key,test_key)
>   )
>   (filter_files tst_re (find_files dirname));
>   h
> ;;
> 
> let all_tests = (tests_of Filename.current_dir_name)
> ;;
> 
> 
> 
> let mtest mname =
>   let f = open_out ("mtest_" ^ mname ^ ".ml") in
> 
>   output_string f "let test() = \n";
> 
>   let tests = Hashtbl.find_all all_tests mname in
>   output_string f ("  Util.log \"Checking module " ^ mname ^ "\";\n");
>   List.iter
>   (fun (author,test) ->
>     output_string f ("  Itest_" ^ author ^ "_" ^mname^"_"^test^".test ();\n")
>   )
>   tests
>   ;
>   output_string f "  ()\n";
>   output_string f ";;\n";
>   output_string f "\n";
>   output_string f "test();;\n";
>   close_out f;
> ;;
> 
> let mk_mtests () = List.iter mtest all_modules
> ;;
> 
> mk_mtests()
> ;;
> 
> let patch_test mname author test =
>   let input_filename = "test_" ^ author ^ "_" ^mname^"_"^test^".tst" in
>   let output_filename = "itest_" ^ author ^ "_" ^mname^"_"^test^".ml" in
>   let fi = open_in input_filename in
>   let fo = open_out output_filename in
>   let rec loop() =
>     let s = input_line fi in
>     output_string fo (s ^ "\n");
>     loop()
>   in try loop() with End_of_file ->
>   output_string fo "\n";
>   close_out fo;
>   close_in fi
> ;;
>   
> let patch_tests () =
>   Hashtbl.iter
>   (fun mname (author,test) -> 
>     patch_test mname author test
>   )
>   all_tests
> ;;
> 
> patch_tests()
> ;;
> 
> let xqt cmd msg =
>   let result = Unix.system(cmd) in
>   let result = 
>     match result with
>     | Unix.WEXITED i -> i == 0
>     | Unix.WSIGNALED _ -> false
>     | Unix.WSTOPPED _ -> false
>   in 
>     if not result 
>     then 
>       failwith ("FAILURE: msg=" ^ msg ^ ", cmd=" ^cmd)
> ;;
> 
> let compile_file filename =
>   let cmd = "ocamlc -I "^extlib_dev_dir^" -c " ^ filename in
>   xqt cmd ("Compilation of " ^ filename)
> ;;
> 
> let compile_tests () =
>   (* compile individual tests *)
>   Hashtbl.iter
>   (fun mname (author,test) ->
>     let filename = "itest_" ^ author ^ "_" ^mname^"_"^test^".ml" in
>     compile_file filename
>   )
>   all_tests
>   ;
>   (* compile generated module level thunks *)
>   List.iter
>   (fun s ->
>     let filename = "mtest_" ^ s ^ ".ml" in
>     compile_file filename
>   )
>   all_modules
>   ;
>   (* compile mainline *)
>   compile_file "extlib_test.ml"
> ;;
> 
> compile_tests()
> ;;
> 
> compile_file "util.ml"
> ;;
> 
> let link_tests () =
>   (* individual tests *)
>   let linkstring = ref (
>     "ocamlc -I "^extlib_dev_dir ^
>     " -o extlib_test extLib.cma util.cmo " 
>   )
>   in 
>   Hashtbl.iter
>   (fun mname (author,test) ->
>     let filename = "itest_" ^ author ^ "_" ^mname^"_"^test^".cmo" in
>     linkstring := !linkstring ^ " " ^ filename
>   )
>   all_tests
>   ;
>   (* compile generated module level thunks *)
>   List.iter
>   (fun s ->
>     let filename = "mtest_" ^ s ^ ".cmo" in
>     linkstring := !linkstring ^ " " ^ filename
>   )
>   all_modules
>   ;
>   (* compile mainline *)
>   linkstring := !linkstring ^ " extlib_test.cmo"
>   ;
>   xqt !linkstring "Linking extlib_test"
> ;;
> 
> link_tests()
> ;;
> 
> print_endline "extlib_test generated"
> ;;
>