Downloading your utility usage from Pacific Gas and Electric using TCL

AF: Pacific Gas and Electric aka PG&E has installed smart gas and electric meters throughout northern california which collect hourly and daily usage data. However they provide no easy way to access this data besides through their website. If you are like me and like to own your data then you can use this to download and save the comma delimited (CSV) usage information.

 package require http
 package require tdom
 package require tls
 package require vfs::zip

 http::register https 443 [list ::tls::socket]

 set tz "America/Los_Angeles"
 set datadir /usr/backup/power
 set user USER
 set pass PASSWORD

 if {$argc > 0} {
    set date [clock scan [lindex $argv 0]]
 } else {
    set date [clock seconds]
 }
 set month [clock format $date -format "%Y-%m"]

 proc cookies {t} {
    set cookies [list]

    foreach {k v} [http::meta $t] {
        if {[string tolower $k] eq "set-cookie"} {
            set v [split $v \;]
            set v [lindex $v 0]
            if {[lindex [split $v =] 1] == ""} continue
            lappend cookies $v
        }
    }
    return [list Cookie [join $cookies "; "]]
 }

 proc get_usage {month user pass} {
    set t [http::geturl "https://www.pge.com/eum/login" -query [http::formatQuery USER $user PASSWORD $pass pass_placeholder Password TARGET https://www.pge.com/myenergyweb/appmanager/pge/customer]]
 
    set cookies [cookies $t]
    http::cleanup $t

    # all this should need is EUMSessionID and SMSESSION
    set t [http::geturl "https://www.pge.com/affwebservices/public/saml2sso?SPID=sso.opower.com&RelayState=https%3A%2F%2Fpge.opower.com%2Fei%2Fapp%2FmyEnergyUse" -headers $cookies]
    set body [http::data $t]
    #parray $t
    http::cleanup $t

    set query [list]
    set html [dom parse -html $body]
    set root [$html documentElement]
    set action [[$root getElementsByTagName form] getAttribute action]
    foreach input [$root getElementsByTagName input] {
        if {[$input getAttribute type] != "hidden"} continue
        lappend query [$input getAttribute name] [$input getAttribute value]
    }
    
    
    set t [http::geturl $action -query [http::formatQuery {*}$query]]
    set body [http::data $t]
    http::cleanup $t

    set query [list]
    set html [dom parse -html $body]
    set root [$html documentElement]
    set action [[$root getElementsByTagName form] getAttribute action]
    foreach input [$root getElementsByTagName input] {
        if {[$input getAttribute type] != "hidden"} continue
        lappend query [$input getAttribute name] [$input getAttribute value]
    }
    
    set t [http::geturl $action -query [http::formatQuery {*}$query]]
    set cookies [cookies $t]
    http::cleanup $t

    set t [http::geturl "https://pge.opower.com/ei/app/myEnergyUse" -headers $cookies]
    #<a data-trigger-dialog="href" href="/ei/app/modules/customer/610138/bill_periods/export-dialog">Export your data</a>
    regexp {modules/customer/(\d+)/bill_periods} [http::data $t] -> customer
    http::cleanup $t

    set t [http::geturl "https://pge.opower.com/ei/app/modules/customer/$customer/energy/download?billing=false&bill=$month" -headers $cookies]

    if {[http::ncode $t] != 200} {
        return
    }

    set zip "/tmp/usage-$month.zip"
    set fh [open $zip w+]
    fconfigure $fh -translation binary
    puts -nonewline $fh [http::data $t]
    close $fh
    http::cleanup $t
    return $zip
 }

 proc write_csv {zip} {
    global datadir tz
    set mnt_file [vfs::zip::Mount $zip $zip]
    
    set in [open "$zip/DailyNaturalGasUsage.csv"]
    set csv [read $in]
    close $in
    #TYPE,DATE,USAGE,UNITS,NOTES
    #Natural gas usage,2011-10-27,1.03,therms,

    array set data {}
    foreach line [split $csv \n] {
        set line [split $line ,]
        if {[lindex $line 0] != "Natural gas usage"} continue
        set start [clock scan [lindex $line 1] -timezone $tz]
        set day [clock format $start -format "%Y-%m-%d"]
        lappend data($day) "$start,[lindex $line 2]"
    }
    foreach {day vals} [array get data] {
        set fn "$datadir/gas-$day.csv"
        if {[file exists $fn] && [file size $fn] != 0} continue
        set fh [open $fn w+]
        puts "writing $fn"
        puts $fh [join $vals \n]
        close $fh
    }


    set in [open "$zip/DailyElectricUsage.csv"]
    set csv [read $in]
    close $in
    #TYPE,DATE,START TIME,END TIME,USAGE,UNITS,NOTES
    #Electric usage,2011-09-28,00:00,00:59,0.16,kWh,
    
    unset -nocomplain data
    array set data {}
    foreach line [split $csv \n] {
        set line [split $line ,]
        if {[lindex $line 0] != "Electric usage"} continue
        set start [clock scan "[lindex $line 1] [lindex $line 2]" -timezone $tz]
        set day [clock format $start -format "%Y-%m-%d"]
        lappend data($day) "$start,[lindex $line 4]"
    }
    foreach {day vals} [array get data] {
        set fn "$datadir/electric-$day.csv"
        if {[file exists $fn] && [file size $fn] != 0} continue
        set fh [open $fn w+]
        puts "writing $fn"
        puts $fh [join $vals \n]
        close $fh
    }
    
    vfs::unmount $zip
 }

 set zip [get_usage $month $user $pass]
 if {$zip == ""} exit 1
 write_csv $zip
 file delete $zip