#!/usr/bin/perl
# hp2000_ - Munin plugin for HP P2000 StorageWorks
# Copyright (C) 2012 Redpill Linpro AS
#
# Author: Trygve Vea <tv@redpill-linpro.com>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# Graph stats from the HP P2000 StorageWorks unit
#
# Parameters supported:
#
#     config
#
# Configurable variables
#
#     env.host     - http-host
#     env.secret   - username_password
#     env.aspect   - vdisk-statistics, controller-statistics, disk-statistics,
#                    volume-statistics or port-statistics.  If not specified,
#                    the plugin will attempt to detect the aspect from the
#                    symlink-name using regex (/vdisk/, /controller/,
#                    /volume/, /disk/ and /port/)
#
# Suggested config:
#
#   [hp2000_san1_*]
#   host_name san1
#   env.host 10.0.0.1
#   env.secret username_password
#
# Then create the following symlinks, pointing to the plugin:
#
#  hp2000_san1_vdisk
#  hp2000_san1_disk
#  hp2000_san1_controller
#  hp2000_san1_volume
#  hp2000_san1_port
#
# And direct your munin.conf to the node configured with this plugin, using
# node 'san1'.
#
#    TODO:
#
#       * Graph sensor-data
#       * Graph disk errors
#       * Graph cache hit vs. misses in one single descriptive graph
#       * Set thresholds
#       * Add useful descriptions
#
# Magic markers:
#%# family=contrib

use Digest::MD5 qw(md5_hex);
use XML::LibXML;
use Munin::Plugin::HTTP;

need_multigraph();

my $cmd = 'vdisk-statistics'; # Default.

if ( $0 =~ /vdisk/ ) { $cmd = 'vdisk-statistics'; }
elsif ( $0 =~ /controller/ ) { $cmd = 'controller-statistics'; }
elsif ( $0 =~ /volume/ ) { $cmd = 'volume-statistics'; }
elsif ( $0 =~ /disk/ ) { $cmd = 'disk-statistics'; }
elsif ( $0 =~ /port/ ) { $cmd = 'host-port-statistics'; }

if ( exists $ENV{'aspect'} ) { $cmd = $ENV{'aspect'}; }
my $host = $ENV{'host'};
my $secret = $ENV{'secret'};

my @plugins = ('vdisk-statistics', 'controller-statistics', 'disk-statistics', 'volume-statistics', 'host-port-statistics');
my @graph_parameters = ('title','total','order','scale','vlabel','args', 'category');
my @field_parameters = ('graph', 'min', 'max', 'draw', 'cdef', 'warning', 'colour', 'info', 'type');

my %aspects = (
    'host-port-statistics' => {
        'port_iops' => {
            'title' => 'IOPS by Hostport',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) ops per second',
            'min' => 0,
            'values' => {
                'number-of-reads' => {
                    'short' => 'nor',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bps',
                    'min' => 0,
                    'graph' => 'no',
                },
                'number-of-writes' => {
                    'short' => 'now',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bps',
                    'min' => 0,
                    'negative' => 'nor',
                },
            }
        },
        'port_bandwidth' => {
            'title' => 'Bandwidth by Hostport',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) bytes per second',
            'min' => 0,
            'values' => {
                'data-read-numeric' => {
                    'short' => 'dr',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bps',
                    'min' => 0,
                    'graph' => 'no'
                },
                'data-written-numeric' => {
                    'short' => 'dw',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bps',
                    'min' => 0,
                    'negative' => 'dr',
                },
            }
        },
        'port_response_time' => {
            'title' => 'Response time by Hostport',
            'category' => 'disk',
            'vlabel' => '(- read / + write) response time in micros.',
            'min' => 0,
            'values' => {
                'avg-rsp-time' => {
                    'short' => 'wrt',
                    'type' => 'GAUGE',
                    'draw' => 'LINE1',
                    'label' => 'micros.',
                    'min' => 0,
                    'graph' => 'no'
                },
                'avg-read-rsp-time' => {
                    'short' => 'rrt',
                    'type' => 'GAUGE',
                    'draw' => 'LINE1',
                    'label' => 'micros.',
                    'min' => 0,
                    'negative' => 'wrt',
                }
            }
        },
    },
    'volume-statistics' => {
        'volume_bandwidth' => {
            'title' => 'Bandwidth',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) bytes per second',
            'min' => 0,
            'values' => {
                'data-read-numeric' => {
                    'short' => 'br',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'read bps',
                    'min' => 0,
                    'graph' => 'no',
                },
                'data-written-numeric' => {
                    'short' => 'bw',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bps',
                    'min' => 0,
                    'negative' => 'br',
                }
            }
        },
        'volume_iops' => {
            'title' => 'I/O Operations',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) ops per second',
            'min' => 0,
            'values' => {
                'number-of-reads' => {
                    'short' => 'ior',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'number-of-writes' => {
                    'short' => 'iow',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'iops',
                    'min' => 0,
                    'negative' => 'ior'
                }
            }
        },
        'volume_cache_hits' => {
            'title' => 'Cache hits',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) hits per second',
            'min' => 0,
            'values' => {
                'read-cache-hits' => {
                    'short' => 'rch',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'write-cache-hits' => {
                    'short' => 'wch',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'cache hits',
                    'min' => 0,
                    'negative' => 'rch'
                }
            }
        },
        'volume_cache_misses' => {
            'title' => 'Cache misses',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) misses per second',
            'min' => 0,
            'values' => {
                'read-cache-misses' => {
                    'short' => 'rcm',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'write-cache-misses' => {
                    'short' => 'wcm',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'cache misses',
                    'min' => 0,
                    'negative' => 'rcm'
                }
            }
        },
        'volume_small_destages' => {
            'title' => 'Small Destages',
            'category' => 'disk',
            'vlabel' => 'destages per second',
            'min' => 0,
            'values' => {
                'small-destages' => {
                    'short' => 'destages',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'destages',
                    'min' => 0,
                }
            }
        },
        'volume_stripe_destages' => {
            'title' => 'Full Stripe Write Destages',
            'category' => 'disk',
            'vlabel' => 'destages per second',
            'min' => 0,
            'values' => {
                'full-stripe-write-destages' => {
                    'short' => 'destages',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'destages',
                    'min' => 0,
                }
            }
        },
        'volume_readahead' => {
            'title' => 'Read-Ahead Operations',
            'category' => 'disk',
            'vlabel' => 'ops per second',
            'min' => 0,
            'values' => {
                'read-ahead-operations' => {
                    'short' => 'ops',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'ops',
                    'min' => 0,
                }
            }
        },
    },
    'vdisk-statistics' => {
        'vdisk_bandwidth' => {
            'title' => 'Bandwidth',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) bytes per second',
            'min' => 0,
            'values' => {
                'data-read-numeric' => {
                    'short' => 'br',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'read bps',
                    'min' => 0,
                    'graph' => 'no',
                },
                'data-written-numeric' => {
                    'short' => 'bw',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bps',
                    'min' => 0,
                    'negative' => 'br',
                }
            }
        },
        'vdisk_iops' => {
            'title' => 'I/O Operations',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) ops per second',
            'min' => 0,
            'values' => {
                'number-of-reads' => {
                    'short' => 'ior',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'number-of-writes' => {
                    'short' => 'iow',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'iops',
                    'min' => 0,
                    'negative' => 'ior'
                }
            }
        }
    },
    'controller-statistics' => {
        'cpu_load' => {
            'title' => 'CPU Usage',
            'category' => 'CPU',
            'vlabel' => 'usage in %',
            'min' => 0,
            'values' => {
                'cpu-load' => {
                    'short' => 'cpu',
                    'type' => 'GAUGE',
                    'draw' => 'LINE1',
                    'label' => '% load',
                    'min' => 0,
                },
            }
        },
        'uptime' => {
            'title' => 'Controller Uptime',
            'category' => 'System',
            'vlabel' => 'seconds up',
            'min' => 0,
            'values' => {
                'power-on-time' => {
                    'short' => 'uptime',
                    'type' => 'GAUGE',
                    'draw' => 'LINE1',
                    'label' => 'uptime',
                    'min' => 0,
                },
            }
        },
        'controller_iops' => {
            'title' => 'Controller I/O Operations',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) ops per second',
            'min' => 0,
            'values' => {
                'number-of-reads' => {
                    'short' => 'ior',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'number-of-writes' => {
                    'short' => 'iow',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'iops',
                    'min' => 0,
                    'negative' => 'ior'
                }
            }
        },
        'controller_bandwidth' => {
            'title' => 'Bandwidth',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) bytes per second',
            'min' => 0,
            'values' => {
                'data-read-numeric' => {
                    'short' => 'br',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'read bytes',
                    'min' => 0,
                    'graph' => 'no'
                },
                'data-written-numeric' => {
                    'short' => 'bw',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'write bytes',
                    'min' => 0,
                    'negative' => 'br'
                }
            }
        },
        'controller_cache_hits' => {
            'title' => 'Cache hits',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) hits per second',
            'min' => 0,
            'values' => {
                'read-cache-hits' => {
                    'short' => 'rch',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'write-cache-hits' => {
                    'short' => 'wch',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'cache hits',
                    'min' => 0,
                    'negative' => 'rch'
                }
            }
        },
        'controller_cache_misses' => {
            'title' => 'Cache misses',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) misses per second',
            'min' => 0,
            'values' => {
                'read-cache-misses' => {
                    'short' => 'rcm',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'write-cache-misses' => {
                    'short' => 'wcm',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'cache misses',
                    'min' => 0,
                    'negative' => 'rcm'
                }
            }
        },
    },
    'disk-statistics' => {
        'disk_bandwidth' => {
            'title' => 'Bandwidth',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) bytes per second',
            'min' => 0,
            'values' => {
                'data-read-numeric' => {
                    'short' => 'br',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'read bytes',
                    'min' => 0,
                    'graph' => 'no'
                },
                'data-written-numeric' => {
                    'short' => 'bw',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'bytes',
                    'min' => 0,
                    'negative' => 'br'
                }
            }
        },
        'disk_iops' => {
            'title' => 'Disk I/O Operations',
            'category' => 'disk',
            'vlabel' => '(- reads / + writes) ops per second',
            'min' => 0,
            'values' => {
                'number-of-reads' => {
                    'short' => 'ior',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'reads',
                    'min' => 0,
                    'graph' => 'no'
                },
                'number-of-writes' => {
                    'short' => 'iow',
                    'type' => 'DERIVE',
                    'draw' => 'LINE1',
                    'label' => 'iops',
                    'min' => 0,
                    'negative' => 'ior'
                }
            }
        },

    }
);


# Authenticate with the controller, get session key.
my $md5_hash = md5_hex( $secret );

$ua = Munin::Plugin::HTTP->new;
$url = "http://$host/api/login/" . $md5_hash;
$res = $ua->get($url);

# Parse the XML content using LibXML to obtain the session key
my $parser  = XML::LibXML->new();
my $doc     = $parser->parse_string( $res->content );
my $root    = $doc->getDocumentElement;
my @objects = $root->getElementsByTagName('OBJECT');
my @props   = $objects[0]->getElementsByTagName('PROPERTY');
my $sessionKey;

foreach my $prop ( @props ) {
     my $name = $prop->getAttribute('name');
     # print "Property = " . $name . "\n";
     if( $name eq 'response' ) {
         $sessionKey = $prop->textContent;
     }
}

# Got $sessionKey, use in subsequent requests.

$url = "http://$host/api/show/$cmd";

$ua = Munin::Plugin::HTTP->new;
$req = HTTP::Request->new(GET => $url);
$req->header('sessionKey' => $sessionKey );
$req->header('dataType' => 'ipa' );
$res = $ua->request($req);

#print $res->content;

$doc     = $parser->parse_string( $res->content );
$root    = $doc->getDocumentElement;
@objects = $root->getElementsByTagName('OBJECT');

foreach my $obj ( @objects ) {
    next unless ( $obj->getAttribute('basetype') eq $cmd );
    my @splitname;
    my $name;

    @props = $obj->getElementsByTagName('PROPERTY');

    foreach my $prop ( @props ) {
        my $attr = $prop->getAttribute('name');

        if ( $attr eq 'name' or $attr eq 'durable-id' or $attr eq 'sensor-name' or $attr eq 'volume-name' ){
            $name = $prop->textContent();
            $name =~ s/-/_/g;
            if ( length($name) > 12 ) {
                $name = "v".substr($name, length($name) - 11, 11);
            }

            @splitname = split(/\./, $name);

            if ( $splitname[0] ne $name ) {
                $datas{'nodes'}{$splitname[0]} = 1;
                $datas{'subnodes'}{$name} = 1;
            }
            else {
                $datas{'nodes'}{$name} = 1;
            }
        }
        else {
            if ( $splitname[0] ne $name ) {
                $datas{'stats'}{$cmd}{$attr}{$splitname[0]} += $prop->textContent();
                $datas{'substats'}{$cmd}{$attr}{$name} += $prop->textContent();
            } else {
                $datas{'stats'}{$cmd}{$attr}{$name} += $prop->textContent();
            }
        }
    }
}

foreach $key (sort keys %{$aspects{$cmd}}) {
    my $val;
    print "multigraph $key\n";
    if ( $ARGV[0] eq 'config' ) {
        foreach (@graph_parameters) {
            if (!defined($aspects{$cmd}{$key}{$_})) {
                next;
            }
            print "graph_$_ $aspects{$cmd}{$key}{$_}\n";
        }
        print "\n";
        foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {

            foreach $name (sort keys %{$datas{'stats'}{$cmd}{$val}}) {
                my $basename = $aspects{$cmd}{$key}{'values'}{$val}{'short'}."_"."$name";

                print "$basename.label $name\n";

                if ( defined($aspects{$cmd}{$key}{'values'}{$val}{'negative'}) ) {
                    print "$basename.negative $aspects{$cmd}{$key}{'values'}{$val}{'negative'}_$name\n";
                }

                foreach (@field_parameters) {
                    if (!defined($aspects{$cmd}{$key}{'values'}{$val}{$_})) {
                        next;
                    }
                    print "$basename.$_ $aspects{$cmd}{$key}{'values'}{$val}{$_}\n";
                }
            }
        }
        print "\n";
    }
    elsif ( $ARGV[0] eq 'debug' ) {
        foreach $key (sort keys %datas) {
            print "$key = $datas{$key}\n";
        }
    }
    elsif ( $ARGV[0] eq 'autoconf' ) {
        print "no (needs manual config)\n";
        exit 0;
    }
    else {
        foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {
            foreach $name (sort keys %{$datas{'stats'}{$cmd}{$val}}) {
                print $aspects{$cmd}{$key}{'values'}{$val}{'short'}."_"."$name.value $datas{'stats'}{$cmd}{$val}{$name}\n";
            }
        }
        print "\n";
    }

    if ( $cmd ne 'disk-statistics' ) {
        foreach $name (sort keys %{$datas{'nodes'}}) {

            print "multigraph $key.$name\n";

            if ( $ARGV[0] eq 'config' ) {
                foreach (@graph_parameters) {
                    if (!defined($aspects{$cmd}{$key}{$_})) {
                        next;
                    }

                    if ( $_ eq 'title' ) {
                        print "graph_$_ $aspects{$cmd}{$key}{$_}: $name\n";
                        next;
                    }
                    print "graph_$_ $aspects{$cmd}{$key}{$_}\n";
                }

                print "\n";

                foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {

                    my $basename = $aspects{$cmd}{$key}{'values'}{$val}{'short'};

                    print "$basename.label $aspects{$cmd}{$key}{'values'}{$val}{'label'}\n";

                    if ( defined($aspects{$cmd}{$key}{'values'}{$val}{'negative'}) ) {
                        print "$basename.negative $aspects{$cmd}{$key}{'values'}{$val}{'negative'}\n";
                    }

                    foreach (@field_parameters) {
                        if (!defined($aspects{$cmd}{$key}{'values'}{$val}{$_})) {
                            next;
                        }
                        print "$basename.$_ $aspects{$cmd}{$key}{'values'}{$val}{$_}\n";
                    }
                }
                print "\n";
            }
            else {
                foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {
                    print $aspects{$cmd}{$key}{'values'}{$val}{'short'}.".value $datas{'stats'}{$cmd}{$val}{$name}\n";
                }
                print "\n";
            }
        }
    } else {
        # Enclosure graphs
        foreach $name (sort keys %{$datas{'nodes'}}) {

            print "multigraph $key.$name\n";

            if ( $ARGV[0] eq 'config' ) {
                foreach (@graph_parameters) {
                    if (!defined($aspects{$cmd}{$key}{$_})) {
                        next;
                    }

                    if ( $_ eq 'title' ) {
                        print "graph_$_ $aspects{$cmd}{$key}{$_}: Enclosure $name\n";
                        next;
                    }
                    print "graph_$_ $aspects{$cmd}{$key}{$_}\n";
                }

                print "\n";


                foreach $disk (sort keys %{$datas{'subnodes'}}) {
                    my @diskname = split(/\./, $disk);
                    if ( $diskname[0] ne $name ) {
                        next;
                    }

                    foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {

                        my $basename = "disk_".$diskname[1]."_".$aspects{$cmd}{$key}{'values'}{$val}{'short'};

                        print "$basename.label d$diskname[1]\n";

                        if ( defined($aspects{$cmd}{$key}{'values'}{$val}{'negative'}) ) {
                            print "$basename.negative disk_".$diskname[1]."_$aspects{$cmd}{$key}{'values'}{$val}{'negative'}\n";
                        }

                        foreach (@field_parameters) {
                            if (!defined($aspects{$cmd}{$key}{'values'}{$val}{$_})) {
                                next;
                            }
                            print "$basename.$_ $aspects{$cmd}{$key}{'values'}{$val}{$_}\n";
                        }
                    }
                }
                print "\n";
            }
            else {
                foreach $disk (sort keys %{$datas{'subnodes'}}) {
                    my @diskname = split(/\./, $disk);
                    if ( $diskname[0] ne $name ) {
                        next;
                    }
                    foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {
                        print "disk_".$diskname[1]."_".$aspects{$cmd}{$key}{'values'}{$val}{'short'}.".value $datas{'substats'}{$cmd}{$val}{$disk}\n";
                    }
                }
                print "\n";
            }
        }

        # Subgraphs on disk level
        foreach $name (sort keys %{$datas{'subnodes'}}) {
            my $graphname = $name;
            $graphname =~ s/\./\.d/;

            print "multigraph $key.$graphname\n";

            if ( $ARGV[0] eq 'config' ) {
                foreach (@graph_parameters) {
                    if (!defined($aspects{$cmd}{$key}{$_})) {
                        next;
                    }

                    if ( $_ eq 'title' ) {
                        print "graph_$_ $aspects{$cmd}{$key}{$_}: $name\n";
                        next;
                    }
                    print "graph_$_ $aspects{$cmd}{$key}{$_}\n";
                }

                print "\n";

                foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {

                    my $basename = $aspects{$cmd}{$key}{'values'}{$val}{'short'};

                    print "$basename.label $aspects{$cmd}{$key}{'values'}{$val}{'label'}\n";

                    if ( defined($aspects{$cmd}{$key}{'values'}{$val}{'negative'}) ) {
                        print "$basename.negative $aspects{$cmd}{$key}{'values'}{$val}{'negative'}\n";
                    }

                    foreach (@field_parameters) {
                        if (!defined($aspects{$cmd}{$key}{'values'}{$val}{$_})) {
                            next;
                        }
                        print "$basename.$_ $aspects{$cmd}{$key}{'values'}{$val}{$_}\n";
                    }
                }
                print "\n";
            }
            else {
                foreach $val (sort keys %{$aspects{$cmd}{$key}{'values'}}) {
                    print $aspects{$cmd}{$key}{'values'}{$val}{'short'}.".value $datas{'substats'}{$cmd}{$val}{$name}\n";
                }
                print "\n";
            }
        }

    }
}