BenchmarksConstraints_2013_03_30

benchmarking with constraints

date of execution: 2013/03/30 10:06:04

number of runs: 5

CASA: Covering Arrays by Simulated Annealing (CASA) 1.1

ACTS: acts_cmd_beta_v2_r2.7.jar

model
generator
time avg.
time stddev.
size avg.
size stddev.

gcc
ACTS IPOG
0.622400
0.482116
25.0000
0.00000

ACTS IPOF
0.363600
0.0966780
24.0000
0.00000

CASA
780.000
395.810
20.8000
0.748331

benchmark_25_2way
ACTS IPOG
3.68340
1.21563
52.0000
0.00000

ACTS IPOF
2.90420
0.0526551
52.0000
0.00000

CASA
1515.56
962.334
48.4000
1.01980

benchmark_29_2way
ACTS IPOG
0.601200
0.0749330
27.0000
0.00000

ACTS IPOF
0.793400
0.0760463
32.0000
0.00000

CASA
48.3086
21.6143
31.6000
0.800000

benchmark_19_2way
ACTS IPOG
ERROR class java.lang.OutOfMemoryError

ACTS IPOF
ERROR class java.lang.OutOfMemoryError

CASA
607.065
195.504
48.2000
0.400000

CCA4
ACTS IPOG
0.0352000
0.00888594
39.0000
0.00000

ACTS IPOF
0.0212000
0.00365513
38.0000
0.00000

CASA
0.349600
0.0465472
36.2000
0.400000

benchmark_3_2way
ACTS IPOG
0.0118000
0.00760000
18.0000
0.00000

ACTS IPOF
0.00900000
0.00000
18.0000
0.00000

CASA
2.49020
1.75314
18.4000
0.489898

CCA5
ACTS IPOG
0.0158000
0.00813388
53.0000
0.00000

ACTS IPOF
0.0108000
0.00240000
53.0000
0.00000

CASA
0.476000
0.0333047
51.0000
0.00000

CCA3
ACTS IPOG
0.0102000
0.00741350
27.0000
0.00000

ACTS IPOF
0.00620000
0.000400000
28.0000
0.00000

CASA
0.438000
0.127689
26.0000
0.00000

benchmark_2_2way
ACTS IPOG
1.21480
0.130395
35.0000
0.00000

ACTS IPOF
1.14120
0.00908626
33.0000
0.00000

CASA
19.3378
2.54685
32.6000
0.800000

benchmark_24_2way
ACTS IPOG
2.66240
0.109525
48.0000
0.00000

ACTS IPOF
2.54780
0.165693
46.0000
0.00000

CASA
175.520
63.2069
45.2000
2.99333

bbs
ACTS IPOG
0.00360000
0.00320000
12.0000
0.00000

ACTS IPOF
0.00220000
0.000400000
12.0000
0.00000

CASA
0.137000
0.0116790
11.0000
0.00000

benchmark_15_2way
ACTS IPOG
0.0380000
0.0150067
31.0000
0.00000

ACTS IPOF
0.0414000
0.000489897
31.0000
0.00000

CASA
31.7728
33.9349
31.0000
0.894427

benchmark_16_2way
ACTS IPOG
1.04120
0.0513786
24.0000
0.00000

ACTS IPOF
1.29760
0.102915
29.0000
0.00000

CASA
18.9234
6.56443
24.4000
0.489898

benchmark_11_2way
ACTS IPOG
2.03640
0.0887009
43.0000
0.00000

ACTS IPOF
2.19100
0.104186
44.0000
0.00000

CASA
367.311
264.775
42.8000
1.60000

benchmark_28_2way
ACTS IPOG
ERROR class java.lang.OutOfMemoryError

ACTS IPOF
ERROR class java.lang.OutOfMemoryError

CASA
2053.67
755.718
50.2000
0.400000

spin_verifier
ACTS IPOG
0.0482000
0.0296742
43.0000
0.00000

ACTS IPOF
0.0462000
0.00515364
44.0000
0.00000

CASA
380.087
624.675
38.6000
3.20000

mobile_phone
ACTS IPOG
0.00660000
0.00520000
12.0000
0.00000

ACTS IPOF
0.00380000
0.000748331
10.0000
0.00000

CASA
0.310400
0.0270525
9.00000
0.00000

benchmark_21_2way
ACTS IPOG
1.41320
0.0153154
37.0000
0.00000

ACTS IPOF
1.75000
0.0833523
38.0000
0.00000

CASA
70.3848
59.7680
36.8000
1.16619

benchmark_30_2way
ACTS IPOG
0.0346000
0.0183368
21.0000
0.00000

ACTS IPOF
0.0372000
0.00342929
23.0000
0.00000

CASA
10.3238
2.76646
20.2000
0.400000

phone
ACTS IPOG
0.00580000
0.00312410
18.0000
0.00000

ACTS IPOF
0.00400000
0.00000
18.0000
0.00000

CASA
ERROR class citlab.core.ext.NotConvertableModel

benchmark_20_2way
ACTS IPOG
ERROR class java.lang.OutOfMemoryError

ACTS IPOF
ERROR class java.lang.OutOfMemoryError

CASA
2931.06
2369.11
54.2000
1.93907

apache
ACTS IPOG
0.0950000
0.0189737
32.0000
0.00000

ACTS IPOF
0.202600
0.0101311
33.0000
0.00000

CASA
68.7880
8.37035
35.0000
1.89737

benchmark_5_2way
ACTS IPOG
5.77760
0.306963
51.0000
0.00000

ACTS IPOF
5.10660
0.353042
48.0000
0.00000

CASA
1780.08
1612.35
48.2000
3.05941

bugzilla
ACTS IPOG
0.0118000
0.00348712
18.0000
0.00000

ACTS IPOF
0.0130000
0.00126491
18.0000
0.00000

CASA
9.16840
2.81743
16.2000
0.400000

benchmark_23_2way
ACTS IPOG
0.00840000
0.00634350
15.0000
0.00000

ACTS IPOF
0.00540000
0.000489898
13.0000
0.00000

CASA
4.05420
1.20458
12.6000
0.489898

CCA6
ACTS IPOG
0.00900000
0.00451664
30.0000
0.00000

ACTS IPOF
0.00660000
0.000800000
29.0000
0.00000

CASA
1.31760
1.10584
27.4000
0.800000

benchmark_13_2way
ACTS IPOG
1.40400
0.116681
39.0000
0.00000

ACTS IPOF
1.27360
0.126041
37.0000
0.00000

CASA
43.2804
10.6386
38.4000
1.01980

benchmark_9_2way
ACTS IPOG
0.0238000
0.0122213
21.0000
0.00000

ACTS IPOF
0.0228000
0.00213542
23.0000
0.00000

CASA
10.9808
0.449318
20.2000
0.400000

benchmark_1_2way
ACTS IPOG
1.44340
0.0851859
43.0000
0.00000

ACTS IPOF
1.59300
0.0917366
45.0000
0.00000

CASA
987.115
1010.90
38.2000
1.16619

benchmark_12_2way
ACTS IPOG
2.48980
0.0671279
41.0000
0.00000

ACTS IPOF
2.40540
0.114507
39.0000
0.00000

CASA
915.020
1200.15
40.0000
2.19089

benchmark_14_2way
ACTS IPOG
0.0890000
0.00950789
38.0000
0.00000

ACTS IPOF
0.125600
0.0103073
38.0000
0.00000

CASA
167.062
192.792
37.0000
1.09545

benchmark_7_2way
ACTS IPOG
0.00680000
0.00462169
11.0000
0.00000

ACTS IPOF
0.00560000
0.00149666
12.0000
0.00000

CASA
0.942800
0.0743758
9.00000
0.00000

benchmark_8_2way
ACTS IPOG
3.50820
0.145936
45.0000
0.00000

ACTS IPOF
3.82920
0.213308
45.0000
0.00000

CASA
336.946
257.792
41.0000
1.89737

benchmark_4_2way
ACTS IPOG
0.0190000
0.00572713
21.0000
0.00000

ACTS IPOF
0.0238000
0.00263818
25.0000
0.00000

CASA
13.8756
9.17619
21.2000
0.748331

CCA7
ACTS IPOG
0.0122000
0.00256125
41.0000
0.00000

ACTS IPOF
0.0108000
0.000979796
43.0000
0.00000

CASA
2.85660
2.34712
39.0000
1.09545

CCA8
ACTS IPOG
0.0292000
0.00584466
57.0000
0.00000

ACTS IPOF
0.0270000
0.00244949
57.0000
0.00000

CASA
2.26420
1.41302
54.8000
2.31517

SmartHome
ACTS IPOG
25.7342
0.532551
11.0000
0.00000

ACTS IPOF
24.0302
0.235091
11.0000
0.00000

CASA
ERROR class java.lang.NullPointerException

spin_simulator
ACTS IPOG
0.0106000
0.00422374
23.0000
0.00000

ACTS IPOF
0.00880000
0.000748331
23.0000
0.00000

CASA
2.92300
2.93384
20.4000
1.35647

benchmark_22_2way
ACTS IPOG
0.0406000
0.00840476
36.0000
0.00000

ACTS IPOF
0.0550000
0.00109544
36.0000
0.00000

CASA
13.7196
3.55276
36.2000
0.400000

benchmark_18_2way
ACTS IPOG
2.04680
0.0243179
49.0000
0.00000

ACTS IPOF
1.74200
0.109375
47.0000
0.00000

CASA
1318.10
807.775
41.2000
0.979796

benchmark_10_2way
ACTS IPOG
4.88740
0.0631081
44.0000
0.00000

ACTS IPOF
5.02860
0.163569
44.0000
0.00000

CASA
1387.07
1935.21
44.6000
1.85472

CCA2
ACTS IPOG
0.00480000
0.00360000
17.0000
0.00000

ACTS IPOF
0.00280000
0.000400000
18.0000
0.00000

CASA
0.119800
0.0132876
17.0000
0.00000

benchmark_27_2way
ACTS IPOG
0.0254000
0.00621611
37.0000
0.00000

ACTS IPOF
0.0368000
0.00354401
37.0000
0.00000

CASA
37.6914
48.1906
36.0000
0.00000

benchmark_26_2way
ACTS IPOG
1.70880
0.0586017
34.0000
0.00000

ACTS IPOF
1.67380
0.0958653
34.0000
0.00000

CASA
56.5342
40.2055
34.2000
2.92575

benchmark_17_2way
ACTS IPOG
2.78620
0.0841151
41.0000
0.00000

ACTS IPOF
2.37560
0.119440
39.0000
0.00000

CASA
290.241
150.218
39.6000
1.20000

benchmark_6_2way
ACTS IPOG
0.0280000
0.00831865
24.0000
0.00000

ACTS IPOF
0.0382000
0.00391918
29.0000
0.00000

CASA
21.1742
7.32762
24.0000
0.00000

CCA1
ACTS IPOG
0.00440000
0.00382623
10.0000
0.00000

ACTS IPOF
0.00280000
0.000400000
9.00000
0.00000

CASA
0.117000
0.0139427
9.00000
0.00000