#!/usr/bin/perl

use strict;
use warnings;

use File::Basename qw(dirname basename);
use File::Copy;
use Log::Log4perl qw(:easy);
use Params::Validate;

use FindBin qw($Bin);
use lib "$Bin/../../lib";

use perfSONAR_PS::NPToolkit::Config::RegularTesting;

use OWP::Conf;

my $pinger_landmarks = "/opt/perfsonar_ps/PingER/etc/pinger-landmarks.xml";
my $owmesh_conf = "/opt/perfsonar_ps/perfsonarbuoy_ma/etc/owmesh.conf";
my $regular_test_conf = "/opt/perfsonar_ps/regular_testing/etc/regular_testing.conf";

Log::Log4perl->easy_init($DEBUG);

my $logger = get_logger();

unless ( -e $pinger_landmarks and -e $owmesh_conf ) {
    print "No tests to upgrade\n";
    exit 0;
}

# Backup the existing files
backup_file({ file => $pinger_landmarks });
backup_file({ file => $owmesh_conf });
backup_file({ file => $regular_test_conf });

my $regular_tests = perfSONAR_PS::NPToolkit::Config::RegularTesting->new();
my ($status, $res) = $regular_tests->init(regular_testing_config_file => $regular_test_conf);
if ($status != 0) {
    die("Problem initializing regular tests: $res");
}

($status, $res) = parse_pinger_landmarks_file($regular_tests, { file => $pinger_landmarks });
if ($status != 0) {
    die("Problem parsing PingER configuration: $res");
}

($status, $res) = parse_owmesh_conf($regular_tests, { file => $owmesh_conf });
if ($status != 0) {
    die("Problem parsing perfSONARBUOY configuration: $res");
}

my $raw_owmesh = $res;

($status, $res) = generate_pinger_landmarks_file({});

my $pinger_output = $res;

($status, $res) = generate_owmesh_conf({ raw_owmesh_conf => $raw_owmesh });

my $owmesh_output = $res;

($status, $res) = $regular_tests->generate_regular_testing_config(include_mesh_tests => 1);

my $regular_tests_output = $res;

open(NEW_FILE, ">".$pinger_landmarks);
print NEW_FILE $pinger_output;
close(NEW_FILE);

open(NEW_FILE, ">".$owmesh_conf);
print NEW_FILE $owmesh_output;
close(NEW_FILE);

open(NEW_FILE, ">".$regular_test_conf);
print NEW_FILE $regular_tests_output;
close(NEW_FILE);

exit(0);

sub backup_file {
    my $parameters = validate( @_, { file => 1, } );
    my $file = $parameters->{file};

    my $backup_file;

    my $i = 1;
    do {
        $backup_file = $file;
        $backup_file .= "-".$i;
        $i++;
        $logger->debug("Checking if $backup_file exists");
    } while (-e $backup_file);

    copy($file, $backup_file);    
}

=head2 parse_pinger_landmarks_file ({ file => 1 })
    Reads in the PingER landmarks file, converts it into a normalized form and
    loads it into this object's configuration.  Returns (0, "") on success and
    (-1, $error_msg) on failure.
    
    In the PingER model, every address is a separate test. To get the model
    into this module's test model , the module makes use of the fact that
    PingER's config file expects domain, node and port elements, but doesn't
    actually use that information.  Each test is modelled as a domain with a
    random identifier associated with it.  There's a node element
    "profile_node" which contains the test profile (description, packet size,
    etc). It has no port, so PingER ignores it. The rest of the members are
    added as 'node' elements like normal. The parameters for them, when parsed
    by this module, will be replaced with the "profile_node" parameters. For an
    existing PingER configuration, the various elements will be merged into new
    tests. A new test is added for each unique set of test properties (i.e. all
    PingER test members with the same parameters will end up in the same new
    test).
=cut

sub parse_pinger_landmarks_file {
    my ( $regular_tests, @params ) = @_;
    my $parameters = validate( @params, { file => 1, } );
    my $file = $parameters->{file};

    return ( 0, "" ) unless ( -e $parameters->{file} );

    eval {
        my $dom = XML::LibXML->load_xml(location => $file);

        die " Failed to open landmarks $file" unless $dom;

        $logger->debug("Looking for domains");

        foreach my $domain ($dom->findnodes('*[local-name()="topology"]/*[local-name()="domain"]')) {
            my $domain_id = $domain->getAttribute("id");

            $logger->debug("Found test: $domain_id");

            my $test_id;

            my $profile_node;
            foreach my $node ($domain->findnodes('*[local-name()="node"]')) {
                my $node_id = $node->getAttribute("id");
                next unless $node_id and $node_id =~ /:node=profile_node$/;

                $logger->debug("Found profile node");
                $profile_node = $node;
                last;
            }

            my $test_description;

            if ($profile_node) {
                $test_description = $profile_node->findvalue('*[local-name()="description"]');
            }

            unless ($test_description) {
                $test_description = $domain->findvalue('*[local-name()="comments"]');
            }

            unless ($test_description) {
                $test_description = "PingER test";
            }

            $logger->debug("Test description: $test_description");

            foreach my $node ( $domain->findnodes('*[local-name()="node"]') ) {
                my $node_id = $node->getAttribute("id");

                $logger->debug("Found node: $node_id");

                unless ($test_id) {
                    my $packet_size = $node->findvalue('*[local-name()="parameters"]/*[local-name()="parameter" and @name="packetSize"]');
                    my $packet_count = $node->findvalue('*[local-name()="parameters"]/*[local-name()="parameter" and @name="count"]');
                    my $packet_interval = $node->findvalue('*[local-name()="parameters"]/*[local-name()="parameter" and @name="packetInterval"]');
                    my $test_interval = $node->findvalue('*[local-name()="parameters"]/*[local-name()="parameter" and @name="measurementPeriod"]');
                    my $test_offset = $node->findvalue('*[local-name()="parameters"]/*[local-name()="parameter" and @name="measurementOffset"]');
                    my $ttl = $node->findvalue('*[local-name()="parameters"]/*[local-name()="parameter" and @name="ttl"]');

                    my $added_by_mesh;
                    $added_by_mesh = 1 if $domain_id =~ /domain=mesh_agent_/;
    
                    my ( $status, $res ) = $regular_tests->add_test_pinger(
                        {
                            description     => $test_description,
                                packet_size     => $packet_size,
                                packet_count    => $packet_count,
                                packet_interval => $packet_interval,
                                test_interval   => $test_interval,
                                test_offset     => $test_offset,
                                ttl             => $ttl,
                                added_by_mesh   => $added_by_mesh,
                            }
                        );

                        die( "Couldn't create PingER test: $res" ) unless ( $status == 0 );

                        $test_id = $res;
                    }

                    next if ( $node_id =~ /profile_node/ );

                    foreach my $port ($node->findnodes('*[local-name()="port"]')) {
                        my $description = "";

                        $description = $node->findvalue('*[local-name()="description"]');
                        $description = $node->findvalue('*[local-name()="hostName"]') unless $description;

                        my $ip = $port->findvalue('*[local-name()="ipAddress"]');
                        my $name = $node->findvalue('*[local-name()="hostName"]');
                        $name = $ip unless $name;

                        $logger->debug( "Parsed: $description/$name/$ip" );

                        my ( $status, $res ) = $regular_tests->add_test_member( { test_id => $test_id, name => $name, description => $description, address => $ip, receiver => 1, sender => 1 } );

                        die( "Couldn't add host to PingER test: $res" ) unless ( $status == 0 );
                    }
                }
            }
    };
    if ( $@ ) {
        return ( -1, "Failed to load landmarks file: $@ " );
    }

    return ( 0, "" );
}

=head2 generate_pinger_landmarks_file 
    Generates a string representation of the PingER landmarks file based on the
    passed-in tests. Each test is converted into a domain, containing a node
    "profile_node" that has the test parameters. Each test member is then added
    to the domain.
=cut

sub generate_pinger_landmarks_file {
    my ( $regular_tests, @params ) = @_;
    my $parameters = validate( @params, { } );

    my $content = '<pingertopo:topology xmlns:pingertopo="http://ogf.org/ns/nmwg/tools/pinger/landmarks/1.0/"/>';

    return ( 0, $content );
}

=head2 parse_owmesh_conf
    Parses the specified owmesh file, and loads the tests into the object's
    configuration.

    In the perfSONAR-BUOY model, the source address must be specified in the
    configuration file and must be either IPv4 or IPv6. This is the reasoning
    behind the "center" having "ipv4_address" and "ipv6_address" options in
    tests.  When written out, each test is written as a new
    group/testspec/measurement set, and each test member is written out as a
    new node. The only exception to this is the center, local, node which is
    written out as a single node no matter how many test groups there are. To
    handle the case where a single test contains both ipv4 and ipv6 addresses,
    two tests will be written out, one for the ipv4 test and one for the ipv6
    test. These tests must be merged when read back in.
=cut

sub parse_owmesh_conf {
    my ( $regular_tests, @params ) = @_;
    my $parameters = validate( @params, { file => 1, } );

    return ( 0, "" ) unless ( -e $parameters->{file} );

    my $owmesh_conf;

    eval {
        $logger->debug( "Parsing: " . $parameters->{file} );

        # We can't specify the file directly with pSB currently.
        my $confdir = dirname( $parameters->{file} );

        my $conf = OWP::Conf->new( CONFDIR => $confdir );

        $owmesh_conf = __parse_owmesh_conf({ existing_configuration => $conf });

        my %localnodes = map { $_ => 1 } @{ $owmesh_conf->{LOCALNODES} };

        use Data::Dumper;

        $logger->debug("owmesh.conf before removing all the GUI measurement sets: ".Dumper($owmesh_conf));

        # For star configurations, we allow ipv4 and ipv6 sites to co-exist in
        # a single schedule-able test. Since that doesn't work with the owmesh
        # file format, we may name measurement sets something like "[id].IPV4"
        # and "[id].IPV6". When we see those names come up, we add them to the
        # test mapping so that we don't add multiple tests for that case.
        my %test_mapping = ();

        foreach my $measurement_set (keys %{ $owmesh_conf->{MEASUREMENTSET} }) {
            my $measurement_set_desc = $owmesh_conf->{MEASUREMENTSET}->{$measurement_set};

            my $group_name = $measurement_set_desc->{GROUP};
            my $test       = $measurement_set_desc->{TESTSPEC};
            my $addr_type  = $measurement_set_desc->{ADDRTYPE};

            my $test_id;
            my $test_name;

            $logger->debug( "Measurement Set Name: $measurement_set" );

            # Handle the test
            if ( $measurement_set =~ /(.*).IPV[46]/ ) {
                $logger->debug( "Checking if test $1 has previous mapping: " . Dumper(\%test_mapping) );
                $test_id   = $test_mapping{$1};
                $test_name = $1;
            }
            else {
                $test_name = $measurement_set;
            }

            $logger->debug("Read in tests name: ".$test_name);

            unless ( $test_id ) {
                my $tool = $owmesh_conf->{TESTSPEC}->{$test}->{'TOOL' };
                if ( $tool eq "powstream" ) {
                    my $packet_interval           = $owmesh_conf->{TESTSPEC}->{$test}->{'OWPINTERVAL' };
                    my $loss_threshold            = $owmesh_conf->{TESTSPEC}->{$test}->{'OWPLOSSTHRESH' };
                    my $session_count             = $owmesh_conf->{TESTSPEC}->{$test}->{'OWPSESSIONCOUNT' };
                    my $sample_count              = $owmesh_conf->{TESTSPEC}->{$test}->{'OWPSAMPLECOUNT' };
                    my $packet_padding            = $owmesh_conf->{TESTSPEC}->{$test}->{'OWPPACKETPADDING' };
                    my $bucket_width              = $owmesh_conf->{TESTSPEC}->{$test}->{'OWPBUCKETWIDTH' };
                    my $added_by_mesh             = $measurement_set_desc->{'ADDED_BY_MESH' };

                    my $description = $owmesh_conf->{TESTSPEC}->{$test}->{'DESCRIPTION' };
                    $description = $group_name unless ( $description );

                    my ( $status, $res ) = $regular_tests->add_test_owamp(
                        {
                            description               => $description,
                            name                      => $test_name,
                            packet_interval           => $packet_interval,
                            loss_threshold            => $loss_threshold,
                            session_count             => $session_count,
                            sample_count              => $sample_count,
                            packet_padding            => $packet_padding,
                            bucket_width              => $bucket_width,
                            added_by_mesh             => $added_by_mesh,
                        }
                    );

                    die( "Couldn't add new test: $res" ) unless ( $status == 0 );

                    $test_id = $res;
                }elsif ( $tool eq "traceroute" ) {
                    my $test_interval = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACETESTINTERVAL' };
                    my $packet_size = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACEPACKETSIZE' };
                    my $timeout     = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACETIMEOUT' };
                    my $waittime    = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACEWAITTIME' };
                    my $first_ttl   = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACEFIRSTTTL' };
                    my $max_ttl     = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACEMAXTTL' };
                    my $pause       = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACEPAUSE' };
                    my $icmp        = $owmesh_conf->{TESTSPEC}->{$test}->{'TRACEICMP' };
                    my $description = $owmesh_conf->{TESTSPEC}->{$test}->{'DESCRIPTION' };
                    $description = $group_name unless ( $description );
                    my $added_by_mesh = $measurement_set_desc->{'ADDED_BY_MESH' };

                    my ( $status, $res ) = $regular_tests->add_test_traceroute(
                        {
                            description   => $description,
                            name          => $test_name,
                            test_interval => $test_interval,
                            packet_size   => $packet_size,
                            timeout       => $timeout,
                            waittime      => $waittime,
                            first_ttl     => $first_ttl,
                            max_ttl       => $max_ttl,
                            pause         => $pause,
                            protocol      => ($icmp ? 'icmp' : 'udp'),
                            added_by_mesh => $added_by_mesh
                        }
                    );

                    die( "Couldn't add new test: $res" ) unless ( $status == 0 );

                    $test_id = $res;
                }
                elsif ( $tool =~ /bwctl\/(thrulay|nuttcp|iperf)/ ) {
                    my $protocol;
                    if ( $conf->get_val( TESTSPEC => $test, ATTR => 'BWTCP' ) ) {
                        $protocol = "tcp";
                    }
                    elsif ( $conf->get_val( TESTSPEC => $test, ATTR => 'BWUDP' ) ) {
                        $protocol = "udp";
                    }
                    else {
                        die( "No protocol specified" );
                    }

                    my $test_interval             = $owmesh_conf->{TESTSPEC}->{$test}->{'BWTestInterval' };
                    my $duration                  = $owmesh_conf->{TESTSPEC}->{$test}->{'BWTestDuration' };
                    my $window_size               = $owmesh_conf->{TESTSPEC}->{$test}->{'BWWindowSize' };
                    my $report_interval           = $owmesh_conf->{TESTSPEC}->{$test}->{'BWReportInterval' };
                    my $udp_bandwidth             = $owmesh_conf->{TESTSPEC}->{$test}->{'BWUDPBandwidthLimit' };
                    my $buffer_length             = $owmesh_conf->{TESTSPEC}->{$test}->{'BWBufferLen' };
                    my $test_interval_start_alpha = $owmesh_conf->{TESTSPEC}->{$test}->{'BWTestIntervalStartAlpha' };
                    my $tos_bits                  = $owmesh_conf->{TESTSPEC}->{$test}->{'BWTosBits' };
                    my $added_by_mesh = $measurement_set_desc->{'ADDED_BY_MESH' };

                    my $description = $owmesh_conf->{TESTSPEC}->{$test}->{DESCRIPTION};
                    $description = $group_name unless ( $description );

                    $tos_bits = undef if $tos_bits and $tos_bits eq "NaN";

                    # Convert window size to megabytes and UDP bandwidth to Mbps
                    if ( defined $window_size ) {
                        if ( $window_size =~ /^(\d+)[gG]$/ ) {
                            $window_size = ( $1 * 1024 );
                        }
                        elsif ( $window_size =~ /^(\d+)$/ ) {
                            $window_size = ( $1 / 1024 );
                        }
                        elsif ( $window_size =~ /^(\d+)[mM]$/ ) {
                            $window_size = $1;
                        }
                        else {
                            die( "Invalid window size: $window_size" );
                        }
                    }

                    if ( defined $udp_bandwidth ) {
                        if ( $udp_bandwidth =~ /^(\d+)[gG]$/ ) {
                            $udp_bandwidth = ( $1 * 1000 );
                        }
                        elsif ( $udp_bandwidth =~ /^(\d+)$/ ) {
                            $udp_bandwidth = ( $1 / 1000 );
                        }
                        elsif ( $udp_bandwidth =~ /^(\d+)m$/ ) {
                            $udp_bandwidth = $1;
                        }
                        else {
                            die( "Invalid udp bandwidth: $udp_bandwidth" );
                        }
                    }

                    my ( $status, $res ) = $regular_tests->add_test_bwctl_throughput(
                        {
                            description               => $description,
                            tool                      => $1,
                            name                      => $test_name,
                            protocol                  => $protocol,
                            test_interval             => $test_interval,
                            duration                  => $duration,
                            window_size               => $window_size,
                            report_interval           => $report_interval,
                            udp_bandwidth             => $udp_bandwidth,
                            buffer_length             => $buffer_length,
                            test_interval_start_alpha => $test_interval_start_alpha,
                            tos_bits                  => $tos_bits,
                            added_by_mesh             => $added_by_mesh
                        }
                    );

                    die( "Couldn't add new test: $res" ) unless ( $status == 0 );

                    $test_id = $res;
                }
                else {
                    die( "Unknown tool" );
                }

                # Save the unique id so we can correlate the other ipv*
                # measurement set with this one.
                if ( $measurement_set =~ /(.*)\.IPV[46]/ ) {
                    $test_mapping{$1} = $test_id;
                }
            }

            my $group_desc = $owmesh_conf->{GROUP}->{$group_name};

            # Handle the group
            my $group_type = $group_desc->{GROUPTYPE};

            if ( $group_type ne "STAR" ) {
                die( "Can only handle 'star' groups currently" );
            }

            my @node_sets = ();

            push @node_sets, { type => "include", nodes => $group_desc->{NODES}, sender => 1, receiver => 1 };
            push @node_sets, { type => "include", nodes => $group_desc->{INCLUDE_RECEIVERS}, sender => 0, receiver => 1 };
            push @node_sets, { type => "exclude", nodes => $group_desc->{EXCLUDE_RECEIVERS}, receiver => 0 };
            push @node_sets, { type => "include", nodes => $group_desc->{INCLUDE_SENDERS}, sender => 1, receiver => 0 };
            push @node_sets, { type => "exclude", nodes => $group_desc->{EXCLUDE_SENDERS}, sender => 0 };

            my %senders   = ();
            my %receivers = ();

            my %member_ids = ();

            foreach my $node_set ( @node_sets ) {
                foreach my $node ( @{ $node_set->{nodes} } ) {
                    my $node_desc = $owmesh_conf->{NODE}->{$node};
                    my $node_addr = $node_desc->{$addr_type.'ADDR'};

                    next unless ( $node_addr );

                    my $node_description = $node_desc->{LONGNAME};
                    $node_description = $node unless ( $node_description );

                    my $node_owp_test_ports = $node_desc->{OWPTESTPORTS};

                    $logger->debug( "Parsing: $node -> $node_addr" );
                    my ( $addr, $port );
                    if ( $node_addr =~ /^[(.*)]:(\d+)$/ ) {
                        $addr = $1;
                        $port = $2;
                    }
                    elsif ( $node_addr =~ /^[(.*)]$/ ) {
                        $addr = $1;
                    }
                    elsif ( $node_addr =~ /^(.*):(\d+)$/ ) {
                        $addr = $1;
                        $port = $2;
                    }
                    else {
                        $addr = $node_addr;
                    }
                    $logger->debug( "Result: $addr" );

                    if ( $localnodes{$node} ) {
                        if ($node_owp_test_ports) {
                            my ($min_port, $max_port) = split('-', $node_owp_test_ports);

                            $regular_tests->set_local_port_range({ test_type => "owamp", min_port => $min_port, max_port => $max_port });
                        }
                    }

                    if ( $node_set->{type} eq "include" ) {
                        my ($status, $res) = $regular_tests->add_test_member( { test_id => $test_id, name => $node, description => $node_description, address => $addr, port => $port, receiver => $node_set->{receiver}, sender => $node_set->{sender} } );
                        if ($status == 0) {
                            $member_ids{$addr} = $res;
                        }
                    }
                    else {
                        $regular_tests->update_test_member( { test_id => $test_id, member_id => $member_ids{$addr}, description => $node_description, port => $port, receiver => $node_set->{receiver}, sender => $node_set->{sender} } );
                    }
                }
            }

            # Set the center after we've added everything.

            # We should really scan the address to see if it's ipv4 or ipv6.
            my $ip_type = "ipv4";
            if ( $measurement_set =~ /\.IPV6/ ) {
                $ip_type = "ipv6";
            }

            if ( $group_type eq "STAR" ) {
                my $center = $owmesh_conf->{GROUP}->{$group_name}->{HAUPTNODE};
                my $node_addr = $owmesh_conf->{NODE}->{$center}->{$addr_type.'ADDR'};

                die( "Couldn't find address for center node" ) unless ( $node_addr );

                if ($member_ids{$node_addr}) {
                    $regular_tests->remove_test_member({ test_id => $test_id, member_id => $member_ids{$node_addr} });
                }
            }

            $logger->debug( "Test id: " . $test_id );

            __owmesh_conf_delete_measurement_set({ measurement_set => $measurement_set, owmesh_conf => $owmesh_conf });
        }

        $logger->debug("owmesh.conf after removing all the GUI measurement sets: ".Dumper($owmesh_conf));
    };
    if ( $@ ) {
        return ( -1, $@ );
    }

    return ( 0, $owmesh_conf );
}

=head2 generate_owmesh_conf({ tests => 1 })
    Generates a string representation of the perfSONAR-BUOY configuration file
    based on the passed-in tests. A template is used to ensure that much of the
    boiler plate stays the same. This function converts the test
    representations into the representation expected by the template, and
    passes it to Template Toolkit to render.
=cut
sub generate_owmesh_conf {
    my $parameters = validate( @_, { raw_owmesh_conf => 1 } );
    my $raw_owmesh_conf = $parameters->{raw_owmesh_conf};

    my $content = __build_owmesh_conf($raw_owmesh_conf);

    return (0, $content);
}

sub __parse_owmesh_conf {
    my $parameters = validate( @_, { existing_configuration => 1, } );
    my $existing_configuration = $parameters->{existing_configuration};

    my @top_level_prefixes = ("", "BW", "OWP", "TRACE");
    my @top_level_variables = (
           "ConfigVersion", "SyslogFacility", "GroupName", "UserName", "DevNull", # Generic variables applicable to everything
           "CGIDBUser", "CGIDBPass",  # We need these values to be able to use bwdb.pl and owdb.pl
           "CentralDBName", "CentralDBPass", "CentralDBType", "CentralDBUser",  # We don't autogenerate a collector configuration so copy all those variables over
           "SessionSumCmd", "CentralDataDir", "CentralArchDir", # We don't autogenerate a collector configuration so copy all those variables over
           "CentralHost", "CentralHostTimeout", "SendTimeout", # Copy this over since we don't have a better use for it.
           "SecretName", # Copy the SecretName for now, but we need to figure out how to impart this in the future
           "DataDir", "SessionSuffix", "SummarySuffix", "BinDir", "Cmd" # Used by the master, but generic, or specific to the host it's running on.
    );
    my @measurementset_attrs = ('TESTSPEC', 'ADDRTYPE', 'GROUP', 'DESCRIPTION', 'EXCLUDE_SELF', 'ADDED_BY_MESH');
    my @group_attrs = ('GROUPTYPE','NODES','SENDERS','RECEIVERS','INCLUDE_RECEIVERS','EXCLUDE_RECEIVERS','INCLUDE_SENDERS','EXCLUDE_SENDERS','HAUPTNODE');
    my @node_attrs  = ('ADDR', 'LONGNAME', 'OWPTESTPORTS', 'NOAGENT', 'CONTACTADDR');
    my @testspec_attrs  = (
        'TOOL', 'DESCRIPTION',
        'OWPINTERVAL', 'OWPLOSSTHRESH', 'OWPSESSIONCOUNT', 'OWPSAMPLECOUNT', 'OWPPACKETPADDING', 'OWPBUCKETWIDTH',
        'TRACETESTINTERVAL', 'TRACEPACKETSIZE', 'TRACETIMEOUT', 'TRACEWAITTIME', 'TRACEFIRSTTTL', 'TRACEMAXTTL', 'TRACEPAUSE', 'TRACEICMP',
        'BWTCP', 'BWUDP', 'BWTestInterval', 'BWTestDuration', 'BWWindowSize', 'BWReportInterval', 'BWUDPBandwidthLimit', 'BWBufferLen', 'BWTestIntervalStartAlpha', 'BWTosBits'
    );

    my %top_level_variables = ();
    my %nodes            = ();
    my %groups           = ();
    my %testspecs        = ();
    my %measurement_sets = ();
    my @addrtypes        = ();
    my @localnodes       = ();

    eval {
        foreach my $variable_prefix (@top_level_prefixes) {
            my @variables = @top_level_variables;

            foreach my $variable (@variables) {
                $logger->debug("Checking ".$variable_prefix.$variable);

                my $value = $existing_configuration->get_val(ATTR => $variable, TYPE => $variable_prefix);

                $logger->debug($variable." is defined: ".$value) if defined $value;
    
                if ($variable_prefix ne "") {
                    my $higher_value = $existing_configuration->get_val(ATTR => $variable);

                    if ($higher_value and $value eq $higher_value) {
                        $logger->debug("Existing higher value $higher_value for $variable is the same");
                        next;
                    }
                }
    
                # Pull the existing owmesh configuration
                $top_level_variables{$variable_prefix.$variable} = $value if defined $value;

                # SecretName is a special case...
                if ($variable_prefix.$variable eq "SecretName" and $value) {
                    push @variables, $value;
                }
            }
        }

        my %addrtypes        = ();

        # Only include the local nodes that were for tests that we didn't add.
        my @measurement_sets = $existing_configuration->get_sublist( LIST => 'MEASUREMENTSET' );

        foreach my $measurement_set ( @measurement_sets ) {
            next if ($measurement_sets{$measurement_set});

            $measurement_sets{$measurement_set} = {};

            my $measurement_set_desc = $measurement_sets{$measurement_set};

            foreach my $attr (@measurementset_attrs) {
                __get_ref( $existing_configuration, $measurement_set_desc, $attr, { MEASUREMENTSET => $measurement_set });
            }

            my $addrtype = $measurement_set_desc->{ADDRTYPE};

            $addrtypes{$addrtype} = 1;

            my $group    = $measurement_set_desc->{GROUP};

            unless ($groups{$group}) {
                $groups{$group} = {};

                my $group_desc = $groups{$group};

                foreach my $attr (@group_attrs) {
                    __get_ref($existing_configuration, $group_desc, $attr, { GROUP => $group });
                }

                foreach my $node ( @{ $group_desc->{NODES} } ) {
                    $nodes{$node} = {} unless $nodes{$node};

                    my $node_desc = $nodes{$node};

                    foreach my $attr (@node_attrs) {
                        __get_ref($existing_configuration, $node_desc, $attr, { NODE => $node });
                        __get_ref($existing_configuration, $node_desc, $measurement_set_desc->{ADDRTYPE}.$attr, { NODE => $node });
                    }
                }
            }

            my $testspec = $measurement_set_desc->{TESTSPEC};
            unless ($testspecs{$testspec}) {
                $testspecs{$testspec} = {};

                my $testspec_desc = $testspecs{$testspec};

                foreach my $attr (@testspec_attrs) {
                    __get_ref($existing_configuration, $testspec_desc, $attr, { TESTSPEC => $testspec });
                }
            }
        }

        # Only include the local nodes that were for tests that we didn't add.
        my @temp_local_nodes = $existing_configuration->get_val(  ATTR => 'LOCALNODES'  );
        foreach my $node (@temp_local_nodes) {
            push @localnodes, $node if ($nodes{$node});
        }

        @addrtypes = keys %addrtypes;
    };
    if ( $@ ) {
        return ( -1, $@ );
    }

    my %owmesh_config = ();
    %owmesh_config = %top_level_variables; # Copy the top-level variables over

    $owmesh_config{MEASUREMENTSET} = \%measurement_sets;
    $owmesh_config{NODE}           = \%nodes;
    $owmesh_config{GROUP}          = \%groups;
    $owmesh_config{TESTSPEC}       = \%testspecs;
    $owmesh_config{ADDRTYPES}      = \@addrtypes;
    $owmesh_config{LOCALNODES}     = \@localnodes;

    # Add some variables back in on upgrade
    $owmesh_config{CGIDBUser} = "readonly" unless $owmesh_config{CGIDBUser};
    $owmesh_config{CGIDBPass} = "readonly" unless $owmesh_config{CGIDBPass};

    return ( 0, \%owmesh_config );
}

sub __get_ref {
    my ( $conf, $hash, $attr, $params ) = @_;

    my %params = %$params;
    $params{ATTR} = $attr;

    eval {
        my $val = $conf->get_ref( %params );
        $hash->{$attr} = $val if defined ($val);
    };

    return;
}

sub __owmesh_conf_delete_measurement_set {
    my $parameters = validate( @_, { measurement_set => 1, owmesh_conf => 1 });
    my $measurement_set = $parameters->{measurement_set};
    my $owmesh_conf     = $parameters->{owmesh_conf};

    my $addrtype = $owmesh_conf->{MEASUREMENTSET}->{$measurement_set}->{ADDRTYPE};
    my $group    = $owmesh_conf->{MEASUREMENTSET}->{$measurement_set}->{GROUP};
    my $testspec = $owmesh_conf->{MEASUREMENTSET}->{$measurement_set}->{TESTSPEC};

    delete($owmesh_conf->{MEASUREMENTSET}->{$measurement_set});

    my ($delete_group, $delete_testspec, $delete_addrtype) = (1, 1, 1);

    foreach my $curr_measurement_set (values %{ $owmesh_conf->{MEASUREMENTSET} }) {
        $delete_addrtype = 0 if ($curr_measurement_set->{ADDRTYPE} eq $addrtype);
        $delete_group    = 0 if ($curr_measurement_set->{GROUP} eq $group);
        $delete_testspec = 0 if ($curr_measurement_set->{TESTSPEC} eq $testspec);
    }

    __owmesh_conf_delete_group({ group => $group, owmesh_conf => $owmesh_conf }) if $delete_group;
    __owmesh_conf_delete_testspec({ testspec => $testspec, owmesh_conf => $owmesh_conf }) if $delete_testspec;
    __owmesh_conf_delete_addrtype({ addrtype => $addrtype, owmesh_conf => $owmesh_conf }) if $delete_addrtype;

    return;
}

sub __owmesh_conf_get_node {
    my $parameters  = validate( @_, { id => 1, owmesh_conf => 1 });
    my $id          = $parameters->{id};
    my $owmesh_conf = $parameters->{owmesh_conf};

    return $owmesh_conf->{NODE}->{$id};
}

sub __owmesh_conf_get_group_members {
    my $parameters  = validate( @_, { group => 1, owmesh_conf => 1 });
    my $group       = $parameters->{group};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my @referenced_nodes = @{ $owmesh_conf->{GROUP}->{$group}->{NODES} };

    my $hauptnode = $owmesh_conf->{GROUP}->{$group}->{HAUPTNODE};
    push @referenced_nodes, $hauptnode if ($hauptnode);

    return \@referenced_nodes;
}

sub __owmesh_conf_group_add_node {
    my $parameters  = validate( @_, { group => 1, node => 1, owmesh_conf => 1 });
    my $group       = $parameters->{group};
    my $node        = $parameters->{node};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my $group_desc = $owmesh_conf->{GROUP}->{$group};
    $group_desc->{NODES} = [] unless $group_desc->{NODES};

    my %existing = map { $_ => 1 } @{ $group_desc->{NODES} };

    push @{ $group_desc->{NODES} }, $node unless $existing{$node};

    return;
}

sub __owmesh_conf_group_add_exclude_senders {
    my $parameters  = validate( @_, { group => 1, node => 1, owmesh_conf => 1 });
    my $group       = $parameters->{group};
    my $node        = $parameters->{node};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my $group_desc = $owmesh_conf->{GROUP}->{$group};
    $group_desc->{EXCLUDE_SENDERS} = [] unless $group_desc->{EXCLUDE_SENDERS};

    my %existing = map { $_ => 1 } @{ $group_desc->{EXCLUDE_SENDERS} };

    push @{ $group_desc->{EXCLUDE_SENDERS} }, $node unless $existing{$node};

    return;
}

sub __owmesh_conf_group_add_exclude_receivers {
    my $parameters  = validate( @_, { group => 1, node => 1, owmesh_conf => 1 });
    my $group       = $parameters->{group};
    my $node        = $parameters->{node};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my $group_desc = $owmesh_conf->{GROUP}->{$group};
    $group_desc->{EXCLUDE_RECEIVERS} = [] unless $group_desc->{EXCLUDE_RECEIVERS};

    my %existing = map { $_ => 1 } @{ $group_desc->{EXCLUDE_RECEIVERS} };

    push @{ $group_desc->{EXCLUDE_RECEIVERS} }, $node unless $existing{$node};

    return;
}


sub __owmesh_conf_delete_group {
    my $parameters  = validate( @_, { group => 1, owmesh_conf => 1 });
    my $group       = $parameters->{group};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my $referenced_nodes = __owmesh_conf_get_group_members({ group => $group, owmesh_conf => $owmesh_conf });

    my %nodes_to_delete = map { $_ => 1 } @$referenced_nodes;

    delete($owmesh_conf->{GROUP}->{$group});

    foreach my $curr_group (keys %{ $owmesh_conf->{GROUP} }) {
        my $curr_group_nodes = __owmesh_conf_get_group_members({ group => $curr_group, owmesh_conf => $owmesh_conf });
        foreach my $node (@$curr_group_nodes) {
            delete($nodes_to_delete{$node});
        }
    }

    foreach my $node (keys %nodes_to_delete) {
        delete($owmesh_conf->{NODE}->{$node});
    }

    my @new_local_nodes = ();
    foreach my $node (@{ $owmesh_conf->{LOCALNODES} }) {
        push @new_local_nodes, $node unless ($nodes_to_delete{$node});
    }

    $owmesh_conf->{LOCALNODES} = \@new_local_nodes;

    return;
}

sub __owmesh_conf_add_node {
    my $parameters = validate( @_, { id => 1, owmesh_conf => 1 });
    my $id         = $parameters->{id};
    my $owmesh_conf = $parameters->{owmesh_conf};

    $owmesh_conf->{NODE}->{$id} = { ID => $id };

    return $owmesh_conf->{NODE}->{$id};
}

sub __owmesh_conf_add_group {
    my $parameters = validate( @_, { id => 1, owmesh_conf => 1 });
    my $id         = $parameters->{id};
    my $owmesh_conf = $parameters->{owmesh_conf};

    $owmesh_conf->{GROUP}->{$id} = { ID => $id };

    return $owmesh_conf->{GROUP}->{$id};
}

sub __owmesh_conf_add_measurement_set {
    my $parameters = validate( @_, { id => 1, owmesh_conf => 1 });
    my $id         = $parameters->{id};
    my $owmesh_conf = $parameters->{owmesh_conf};

    $owmesh_conf->{MEASUREMENTSET}->{$id} = { ID => $id };

    return $owmesh_conf->{MEASUREMENTSET}->{$id};
}

sub __owmesh_conf_add_testspec {
    my $parameters = validate( @_, { id => 1, owmesh_conf => 1 });
    my $id         = $parameters->{id};
    my $owmesh_conf = $parameters->{owmesh_conf};

    $owmesh_conf->{TESTSPEC}->{$id} = { ID => $id };

    return $owmesh_conf->{TESTSPEC}->{$id};
}

sub __owmesh_conf_add_localnode {
    my $parameters = validate( @_, { node => 1, owmesh_conf => 1 });
    my $node       = $parameters->{node};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my %existing = map { $_ => 1 } @{ $owmesh_conf->{LOCALNODES} };

    push @{ $owmesh_conf->{LOCALNODES} }, $node unless $existing{$node};

    return;
}

sub __owmesh_conf_delete_addrtype {
    my $parameters = validate( @_, { addrtype => 1, owmesh_conf => 1 });
    my $addrtype   = $parameters->{addrtype};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my @new_addrtypes = ();

    foreach my $existing_addrtype (@{ $owmesh_conf->{ADDRTYPES} }) {
        push @new_addrtypes, $existing_addrtype if ($existing_addrtype ne $addrtype);
    }

    $owmesh_conf->{ADDRTYPES} = \@new_addrtypes;

    # Get rid of the addresses associated with that addrtype
    foreach my $node (values %{ $owmesh_conf->{NODE} }) {
        delete($node->{$addrtype."ADDR"});
    }

    return;
}

sub __owmesh_conf_delete_testspec {
    my $parameters = validate( @_, { testspec => 1, owmesh_conf => 1 });
    my $testspec   = $parameters->{testspec};
    my $owmesh_conf = $parameters->{owmesh_conf};

    delete($owmesh_conf->{TESTSPEC}->{$testspec});

    return;
}


sub __owmesh_conf_add_addrtype {
    my $parameters = validate( @_, { addrtype => 1, owmesh_conf => 1 });
    my $addrtype   = $parameters->{addrtype};
    my $owmesh_conf = $parameters->{owmesh_conf};

    my %addrtypes = map { $_ => 1 } @{ $owmesh_conf->{ADDRTYPES} };

    push @{ $owmesh_conf->{ADDRTYPES} }, $addrtype unless $addrtypes{$addrtype};

    return;
}

sub __build_owmesh_conf {
    my ($owmesh_desc) = @_;

    my $text = "";

    foreach my $key (sort keys %$owmesh_desc) {
        if (ref($owmesh_desc->{$key}) eq "ARRAY") {
            $text .= $key."\t";
            $text .= "[[ ".join("  ", @{ $owmesh_desc->{$key} })." ]]";
        }
        elsif (ref($owmesh_desc->{$key}) eq "HASH") {
            foreach my $subkey (sort keys %{ $owmesh_desc->{$key} }) {
                $text .= "<$key=$subkey>\n";
                $text .= __build_owmesh_conf($owmesh_desc->{$key}->{$subkey});
                $text .= "</$key>\n";
            }
        }
        else {
            if (defined $owmesh_desc->{$key}) {
                $text .= $key."\t".$owmesh_desc->{$key};
            }
            else {
                $text .= "!".$key;
            }
        }

        $text .= "\n";
    }

    return $text;
}


