[RADIATOR] Promethues Metrics Exporter/Conflicts with Forks

[Kelsey Cummings]

It sure would be nice if Radiator used IPC::Sharable or other similar 
method to have shared stats structures across the forks so none of this 
was necessary but here's my workaround.  Thankfully it is possible, this 
seemed like a better option that trying to do similar tricks with other 
stats exporters.  (And now I can get back to triggering alerts based on 
per-client counters.)

TLDR; close/reactive management server port incrementing per fork, use 
standalone http server to combine stats for each fork naively 
incrementing from starting port until it encounters an error.

YMMV, but at least we can get aggregate stats via a single instance.

****
FarmSize 10
FarmChildHook file:"/etc/radiator/farmchild.pl"

<ServerHTTP>
         Identifier MetricsHack
         Port 8000
         DefaultPrivilegeLevel 1
         PageNotFoundHook file:"/etc/radiator/metrics-handler.pl"
</ServerHTTP>
*****

*****
/etc/radiator/farmchild.pl
sub {
         my $server = 
Radius::Configurable::find('ServerHTTP','MetricsHack');
         my $port = $server->{Port} + $main::farmInstance;
         &main::log($main::LOG_INFO, "FarmInstance $main::farmInstance 
assigning ServerHTTP port to $port");
         $server->{Port} = $port;
         $server->activate();
}
*****

*****
/etc/radiator/metrics-handler.pl
sub {
   my $uri = $_[0];

   if($uri !~ /^\/metrics$/) {
     return;
   }

   my $stats;

   foreach ( keys %Radius::StatsLogGeneric::statistic_names) {
     my $name = "radiator_$_";
     my $value = $main::config->{Statistics}{$_};
     $stats .= "$name $value\n";
   }

   for my $client (@{$main::config->{Client}})  {
         my $name = $client->{Name};
         for my $stat (keys $client->{Statistics} ) {
                 next if $stat eq 'Name';
                 $stats .= "radiatorClient_${stat}{client=\"$name\"} " . 
$client->{Statistics}{$stat} . "\n";
         }
   }

   return (200, $stats);
}
*****

*****
#!/usr/bin/perl

use strict;
use LWP::Simple;
use List::Util qw(sum);
use HTTP::Daemon;
use HTTP::Status;
use Sys::Syslog;

my $start_port = 8000;
my $server_port = 7999;

my $server = HTTP::Daemon->new( LocalPort => $server_port, Reuse => 1 ) 
or die;
while (my $client = $server->accept) {
         while ( my $request = $client->get_request) {
                 if ( $request->method eq 'GET' and $request->uri->path 
eq '/metrics' ) {
                         my $metrics = aggregate_metrics();
 
$client->send_response(HTTP::Response->new(200,undef,undef,$metrics));
                 } else {
                         $client->send_error(RC_FORBIDDEN);
                 }
         }
         $client->close;
}

sub aggregate_metrics {
         my %counters;
         my %gauges;
         my $metrics;
         my $count = 0;
         my $port = $start_port;
         while (1) {
                 $port++;
                 my $farmstat = get("http://127.0.0.1:$port/metrics");
                 if ( $farmstat ) {
                         $count++;
                         for my $line (split /^/, $farmstat ) {
                                 my ($label,$value) = split(/ /,$line);
                                 if ( $label =~ /responseTime/ ) {
                                         push(@{$gauges{$label}},$value);
                                         next;
                                 }
                                 $counters{$label} += $value;
                         }
                 } else {
                         last;
                 }
         }

         for my $label (keys %gauges) {
                 $metrics .= $label . " " . ( 
sum(@{$gauges{$label}})/@{$gauges{$label}} ) . "\n";
         }
         for my $label (keys %counters) {
                 $metrics .= "$label $counters{$label}\n";
         }

         $metrics .="radiator_FarmActive $count\n";

         eval {
                 openlog($0,'cons,pid','user');
                 syslog('info',"aggregated stats from $count farm workers");
                 closelog();
         };

         return $metrics;
}
*****

*****
# cat aggreagte-metrics.service
[Unit]
Description=Radiator Prometheus Metrics Aggregator Hack

[Service]
ExecStart=/etc/radiator/aggreagte-metrics.pl

[Install]
WantedBy=multi-user.target
*****

-- 
kelsey.cummings at sonic.com                 sonic.net, inc.
System Architect                          2260 Apollo Way
707.522.1000                              Santa Rosa, CA