Analyse the Normalized Benchmarks by Category

DKF: Here is some code that I have used to analyse the Tcl Normalized Extended Benchmarks by benchmark category. This is far more illuminating than trying to guess what is going on from the overall figures.

To use, cut-and-paste the normalized benchmark results (just the lines that correspond to an individual result) into a file called speeds.txt and run this script on them. After a bit of time, you'll get some nice geometric averages printed out by category. MISC corresponds to the category names that aren't alphanumeric.

set f [open speeds.txt]
set data [read $f]
close $f

array set vl {}
array set svl {}

# For non-UNIX
catch {console show}

proc parseData {data {bestIndex 0}} {
    global vl svl typeNames

    array set types {}
    foreach line [split $data \n] {
       set head [string trim [string range $line 0 45]]
       set tail [string trim [string range $line 45 end]]
       set n 0
       set best [lindex $tail $bestIndex]
       set type [lindex [split $head] 1]
       if {![regexp {^\w+$} $type]} {
          set type MISC
       }
       if {[string length $type] > 7} {
          set type [string range $type 0 4][string range $type end-1 end]
       }
       set types($type) 1
       foreach datum $tail {
          incr n
          if {[string is double -strict $datum]} {
             scan $datum %f d
             lappend vl($n) [expr {$d/$best}]
             lappend svl($type,$n) [expr {$d/$best}]
          }
       }
    }

    set typeNames [lsort [array names types]]
}

proc average {list} {
    set total 0
    foreach item $list {
       set total [expr {$total + log($item)}]
    }
    return [expr {exp($total / [llength $list])}]
}

proc printAverages {} {
    global vl svl typeNames
    puts "\tTotal\t[join $typeNames \t]"
    foreach n [lsort -integer [array names vl]] {
       puts -nonewline [format "%d:\t%.3f" $n [average $vl($n)]]
       foreach t $typeNames {
          if {[catch {
             puts -nonewline [format "\t%.3f" [average $svl($t,$n)]]
          }]} then {
             puts -nonewline \t
          }
       }
       puts ""
    }
}

parseData $data 6  ;# 6 corresponds to 8.4.2 in the dataset
printAverages

To use harmonic means instead of geometric means, replace average with this:

proc average {list} {
    set total 0
    foreach item $list {
       set total [expr {$total + 1/$item}]
    }
    return [expr {1/($total / [llength $list])}]
}

Category Example