From 5d0ad5f64916696987b2e19c7b3d60db7e068f2b Mon Sep 17 00:00:00 2001
From: Your Name <graham@grahamc.com>
Date: Wed, 18 Aug 2021 14:08:38 -0400
Subject: [PATCH] hydra-notify: initial scratch take of prometheus events

---
 flake.nix               |  1 +
 src/script/hydra-notify | 31 ++++++++++++++++++++++++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/flake.nix b/flake.nix
index 616fa81c..44e5898c 100644
--- a/flake.nix
+++ b/flake.nix
@@ -470,6 +470,7 @@
                 NetPrometheus
                 NetStatsd
                 PadWalker
+                ParallelForkManager
                 PerlCriticCommunity
                 PrometheusTinyShared
                 Readonly
diff --git a/src/script/hydra-notify b/src/script/hydra-notify
index 181fba36..f64b5d72 100755
--- a/src/script/hydra-notify
+++ b/src/script/hydra-notify
@@ -3,12 +3,29 @@
 use strict;
 use utf8;
 use Getopt::Long;
+use HTTP::Server::PSGI;
 use Hydra::Event;
 use Hydra::Event::BuildFinished;
 use Hydra::Helper::AddBuilds;
 use Hydra::Helper::Nix;
 use Hydra::Plugin;
 use Hydra::PostgresListener;
+use Parallel::ForkManager;
+use Prometheus::Tiny::Shared;
+use Time::HiRes qw( gettimeofday tv_interval );
+
+my $prom = Prometheus::Tiny::Shared->new;
+
+my $fork_manager = Parallel::ForkManager->new(1 );
+$fork_manager->start_child("metrics_exporter", sub {
+    my $server = HTTP::Server::PSGI->new(
+        host => "127.0.0.1",
+        port => 9091,
+        timeout => 5,
+    );
+
+    $server->run($prom->psgi);
+});
 
 STDERR->autoflush(1);
 STDOUT->autoflush(1);
@@ -36,11 +53,19 @@ $listener->subscribe("step_finished");
 sub runPluginsForEvent {
     my ($event) = @_;
 
+    my $channelName = $event->{'channel_name'};
+
     foreach my $plugin (@plugins) {
+        $prom->inc("notify_plugin_executions", { channel => $channelName, plugin => ref $plugin });
         eval {
+            my $startTime = [gettimeofday()];
             $event->execute($db, $plugin);
+
+            $prom->histogram_observe("notify_plugin_runtime", tv_interval($startTime), { channel => $channelName, plugin => ref $plugin });
+            $prom->inc("notify_plugin_success", { channel => $channelName, plugin => ref $plugin });
             1;
         } or do {
+            $prom->inc("notify_plugin_error", { channel => $channelName, plugin => ref $plugin });
             print STDERR "error running $event->{'channel_name'} hooks: $@\n";
         }
     }
@@ -60,19 +85,23 @@ for my $build ($db->resultset('Builds')->search(
 
 # Process incoming notifications.
 while (!$queued_only) {
+    $prom->inc("event_loop_iterations");
     my $messages = $listener->block_for_messages();
     while (my $message = $messages->()) {
-
+        $prom->set("event_received", time());
         my $channelName = $message->{"channel"};
         my $pid = $message->{"pid"};
         my $payload = $message->{"payload"};
 
+        $prom->inc("notify_event", { channel => $channelName });
+
         eval {
             my $event = Hydra::Event->new_event($channelName, $message->{"payload"});
             runPluginsForEvent($event);
 
             1;
         } or do {
+            $prom->inc("notify_event_error", { channel => $channelName });
             print STDERR "error processing message '$payload' on channel '$channelName': $@\n";
         }
     }