darkkirb/nixos-config
1
0
Fork 0
Code Issues 1 Pull requests 2 Projects Releases Packages Wiki Activity Actions

Merge pull request #307 from DarkKirb/update-tracing-config

Browse source 
update observability
This commit is contained in:
Charlotte 🦝 Delenk 2023-12-10 12:01:56 +01:00 committed by GitHub
commit ef662ca61b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 325 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
config/default.nix
View file

@ -5,7 +5,6 @@
...
}: {
imports = [
./zfs.nix
./users/darkkirb.nix
./users/root.nix
./nix.nix
@ -14,8 +13,11 @@
./services/restic.nix
./specialization.nix
./services/promtail.nix
./services/loki.nix
./services/prometheus.nix
./env.nix
./tailscale.nix
./services/otel.nix
];
services.openssh.enable = true;
environment.systemPackages = with pkgs; [
@ -32,22 +34,6 @@
users.defaultUserShell = pkgs.zsh;
# Prometheus node exporter
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"interrupts"
"lnstat"
"mountstats"
"network_route"
"ntp"
"processes"
"systemd"
"tcpstat"
];
listenAddress = "0.0.0.0";
};
environment.pathsToLink = ["/share/zsh"];
console.keyMap = "neo";
@ -58,7 +44,6 @@
programs.zsh.enable = true;
users.mutableUsers = false;
boot.kernelParams = ["nohibernate"];
sops.secrets."root/aws/credentials" = {
sopsFile = ../secrets/shared.yaml;

1
config/instance-20221213-1915.nix
View file

@ -28,6 +28,7 @@
./services/uptime-kuma.nix
./services/matrix-sliding-sync.nix
./wireguard
./zfs.nix
];
boot.initrd.availableKernelModules = ["xhci_pci" "virtio_pci" "usbhid"];

1
config/nas.nix
View file

@ -28,6 +28,7 @@
./users/remote-build.nix
./services/kubo-local.nix
./services/nfs.nix
./services/tempo.nix
];
hardware.cpu.amd.updateMicrocode = true;

2
config/nixos-8gb-fsn1-1.nix
View file

@ -22,7 +22,6 @@
./services/postfixadmin.nix
./services/dovecot.nix
./services/postfix.nix
./services/loki.nix
./services/reverse-proxy.nix
./services/matrix-media-repo.nix
./bittorrent-blocker.nix
@ -34,6 +33,7 @@
./services/wordpress.nix
./services/initrd-ssh.nix
./wireguard
./zfs.nix
];
boot.initrd.availableKernelModules = ["ata_piix" "virtio_pci" "virtio_scsi" "xhci_pci" "sd_mod" "sr_mod"];

1
config/rainbow-resort.nix
View file

@ -101,4 +101,5 @@
services.tailscale.useRoutingFeatures = "client";
home-manager.users.darkkirb._module.args.withNSFW = lib.mkForce true;
system.autoUpgrade.allowReboot = true;
services.prometheus.exporters.node.enabledCollectors = ["drm"];
}

1
config/server.nix
View file

@ -3,6 +3,5 @@
imports = [
./services/caddy
./services/acme.nix
./services/fail2ban.nix
];
}

14
config/services/dovecot.nix
View file

@ -139,8 +139,20 @@ in {
};
services.prometheus.exporters.dovecot = {
enable = true;
listenAddress = "0.0.0.0";
port = 35496;
};
services.prometheus.scrapeConfigs = [
{
job_name = "dovecot";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.dovecot.port}"
];
}
];
}
];
sops.secrets."services/dovecot/rspamd_password" = {owner = "dovecot";};
sops.secrets."services/dovecot/dovecot-sql.conf.ext" = {owner = "dovecot";};
services.postgresql.ensureUsers = [

7
config/services/elasticsearch.nix
View file

@ -1,7 +0,0 @@
_: {
services.elasticsearch = {
enable = true;
cluster_name = "chir-rs";
};
# TODO: elasticsearch-exporter
}

19
config/services/fail2ban.nix
View file

@ -1,19 +0,0 @@
_: {
services.fail2ban = {
enable = true;
bantime-increment.enable = true;
bantime-increment.maxtime = "48h";
jails = {
dovecot = ''
enabled = true
filter = dovecot
action = iptables-multiport[name=dovecot, port="pop3,pop3s,imap,imaps", protocol=tcp]
'';
postfix = ''
enabled = true
filter = postfix
action = iptables-multiport[name=postfix, port="imap,imaps,submission", protocol=tcp]
'';
};
};
}

12
config/services/gitea.nix
View file

@ -95,4 +95,16 @@
port = 6379;
};
sops.secrets."services/gitea" = {owner = "gitea";};
services.prometheus.scrapeConfigs = [
{
job_name = "forgejo";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.gitea.httpPort}"
];
}
];
}
];
}

3
config/services/grafana.nix
View file

@ -4,9 +4,6 @@
pkgs,
...
}: {
imports = [
./prometheus.nix
];
services.grafana = {
enable = true;
domain = "grafana.int.chir.rs";

12
config/services/hydra.nix
View file

@ -178,4 +178,16 @@ in {
OnUnitActiveSec = 604800;
};
};
services.prometheus.scrapeConfigs = [
{
job_name = "hydra";
static_configs = [
{
targets = [
"127.0.0.1:9199"
];
}
];
}
];
}

13
config/services/kubo-common.nix
View file

@ -89,4 +89,17 @@
networking.firewall.allowedUDPPorts = [
4001
];
services.prometheus.scrapeConfigs = [
{
job_name = "kubo";
metrics_path = "/debug/metrics/prometheus";
static_configs = [
{
targets = [
"127.0.0.1:5001"
];
}
];
}
];
}

74
config/services/loki.nix
View file

@ -1,6 +1,78 @@
_: {
services.loki = {
enable = true;
configFile = ./loki.yaml;
configuration = {
server.http_listen_port = 24545;
auth_enabled = false;
ingester = {
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore = {
store = "inmemory";
};
replication_factor = 1;
};
};
chunk_idle_period = "1h";
max_chunk_age = "1h";
chunk_target_size = 999999;
chunk_retain_period = "30s";
max_transfer_retries = 0;
};
schema_config = {
configs = [
{
from = "2022-06-06";
store = "boltdb-shipper";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
storage_config = {
boltdb_shipper = {
active_index_directory = "/var/lib/loki/boltdb-shipper-active";
cache_location = "/var/lib/loki/boltdb-shipper-cache";
cache_ttl = "24h";
shared_store = "filesystem";
};
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
limits_config = {
reject_old_samples = true;
reject_old_samples_max_age = "168h";
};
chunk_store_config = {
max_look_back_period = "0s";
};
table_manager = {
retention_deletes_enabled = false;
retention_period = "0s";
};
compactor = {
working_directory = "/var/lib/loki";
shared_store = "filesystem";
compactor_ring = {
kvstore = {
store = "inmemory";
};
};
};
};
};
}

30
config/services/loki.yaml
View file

@ -1,30 +0,0 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
ruler:
alertmanager_url: http://localhost:9093

12
config/services/matrix-media-repo.nix
View file

@ -216,4 +216,16 @@ in {
}
'';
};
services.prometheus.scrapeConfigs = [
{
job_name = "matrix-media-repo";
static_configs = [
{
targets = [
"127.0.0.1:9000"
];
}
];
}
];
}

13
config/services/matrix-sliding-sync.nix
View file

@ -11,6 +11,7 @@
SYNCV3_SERVER = "https://matrix.chir.rs";
SYNCV3_DB = "postgresql:///sliding_sync?sslmode=disable&host=/run/postgresql";
SYNCV3_BINDADDR = "127.0.0.1:45587";
SYNCV3_PROM = "127.0.0.1:50372";
};
serviceConfig = {
Type = "simple";
@ -47,4 +48,16 @@
reverse_proxy http://127.0.0.1:45587
'';
};
services.prometheus.scrapeConfigs = [
{
job_name = "matrix-sliding-sync";
static_configs = [
{
targets = [
"127.0.0.1:50372"
];
}
];
}
];
}

14
config/services/named-submissive.nix
View file

@ -41,8 +41,20 @@ in {
enable = true;
bindGroups = ["server" "view" "tasks"];
bindURI = "http://127.0.0.1:8653/";
port = 1533;
};
services.prometheus.scrapeConfigs = [
{
job_name = "bind";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.bind.port}"
];
}
];
}
];
systemd.tmpfiles.rules = [
"d /var/lib/named 4700 named named - -"
];

13
config/services/named.nix
View file

@ -130,6 +130,19 @@ in {
enable = true;
bindGroups = ["server" "view" "tasks"];
bindURI = "http://127.0.0.1:8653/";
port = 1533;
};
services.prometheus.scrapeConfigs = [
{
job_name = "bind";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.bind.port}"
];
}
];
}
];
sops.secrets."services/dns/named-keys" = {owner = "named";};
}

23
config/services/otel.nix Normal file
View file

@ -0,0 +1,23 @@
{
services.opentelemetry-collector = {
enable = true;
settings = {
receivers = {
otlp.protocols = {
grpc = {};
http = {};
};
jaeger.protocols = {
thrift_http = {};
grpc = {};
thrift_binary = {};
thrift_compact = {};
};
};
processors.batch = {};
exporters = {
"otlp".endpoint = "nas.int.chir.rs:4317";
};
};
};
}

14
config/services/postgres.nix
View file

@ -19,6 +19,18 @@
services.prometheus.exporters.postgres = {
enable = true;
user = "postgres";
listenAddress = "0.0.0.0";
port = 1589;
};
services.prometheus.scrapeConfigs = [
{
job_name = "postgresql";
static_configs = [
{
targets = [
"127.0.0.1:1589"
];
}
];
}
];
}

129
config/services/prometheus.nix
View file

@ -1,128 +1,33 @@
{config, ...}: {
services.prometheus = {
port = 26678;
enable = true;
port = 9002;
globalConfig = {
scrape_interval = "10s";
scrape_timeout = "2s";
exporters = {
node = {
port = 31941;
enabledCollectors = [
"buddyinfo"
"cgroups"
"systemd"
"ethtool"
];
enable = true;
};
};
# ingest the published nodes
scrapeConfigs = [
{
job_name = "node_exporter";
job_name = "nodes";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"nutty-noon.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"nas.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"thinkrac.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"instance-20221213-1915.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"vf2.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"127.0.0.1:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
{
job_name = "bind_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.bind.port}"
];
}
];
}
{
job_name = "postgres_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.postgres.port}"
"nas.int.chir.rs:${toString config.services.prometheus.exporters.postgres.port}"
"instance-20221213-1915.int.chir.rs:${toString config.services.prometheus.exporters.postgres.port}"
];
}
];
}
{
job_name = "gitea_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.gitea.httpPort}"
];
}
];
}
{
job_name = "dovecot_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.dovecot.port}"
];
}
];
}
{
job_name = "hydra_exporter";
static_configs = [
{
targets = [
"nas.int.chir.rs:9199"
];
}
];
}
{
job_name = "statsd_exporter";
static_configs = [
{
targets = [
"nas.int.chir.rs:9102"
];
}
];
}
{
job_name = "matrix_media_repo";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:9000"
];
}
];
}
{
job_name = "rspamd_exporter";
static_configs = [
{
targets = [
"nas.int.chir.rs:7980"
];
}
];
}
{
job_name = "synapse";
scrape_interval = "30s";
scrape_timeout = "10s";
metrics_path = "/_synapse/metrics";
static_configs = [
{
targets = [
"matrix.int.chir.rs:80"
];
labels = {
instance = "matrix.chir.rs";
job = "master";
index = "1";
};
}
];
}
];
checkConfig = false;
};
}

79
config/services/promtail.nix
View file

@ -1,47 +1,38 @@
{
config,
pkgs,
lib,
...
}: let
promtail_config = {
server = {
http_listen_port = 28183;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
client = {
url = "http://nixos-8gb-fsn1-1.int.chir.rs:3100/loki/api/v1/push";
external_labels.host = config.networking.hostName;
};
scrape_configs = [
{
job_name = "journal";
journal = {
max_age = "12h";
labels.job = "systemd-journal";
};
relabel_configs = [
{
source_labels = ["__journal__systemd_unit"];
target_label = "unit";
}
];
}
];
};
promtail_yml = pkgs.writeText "promtail.yml" (lib.generators.toYAML {} promtail_config);
in {
systemd.services.promtail = {
description = "Promtail service for Loki";
wantedBy = ["multi-user.target"];
serviceConfig = {
ExecStart = ''
${pkgs.grafana-loki}/bin/promtail --config.file ${promtail_yml}
'';
{config, ...}: {
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 45871;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
clients = [
{
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
}
];
scrape_configs = [
{
job_name = "journal";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = "${config.networking.hostName}";
};
};
relabel_configs = [
{
source_labels = ["__journal__systemd_unit"];
target_label = "unit";
}
];
}
];
};
# extraFlags
};
}

17
config/services/rspamd.nix
View file

@ -194,5 +194,20 @@
sops.secrets."services/rspamd/dkim/darkkirb.de" = {owner = "rspamd";};
sops.secrets."services/rspamd/dkim/miifox.net" = {owner = "rspamd";};
sops.secrets."services/rspamd/dkim/chir.rs" = {owner = "rspamd";};
services.prometheus.exporters.rspamd.enable = true;
services.prometheus.exporters.rspamd = {
enable = true;
port = 58636;
};
services.prometheus.scrapeConfigs = [
{
job_name = "rspamd";
static_configs = [
{
targets = [
"127.0.0.1:58636"
];
}
];
}
];
}

13
config/services/synapse.nix
View file

@ -192,4 +192,17 @@
key = "services/mautrix/whatsapp.yaml";
owner = "matrix-synapse";
};
services.prometheus.scrapeConfigs = [
{
job_name = "matrix-synapse";
metrics_path = "/_synapse/metrics";
static_configs = [
{
targets = [
"127.0.0.1:8008"
];
}
];
}
];
}

26
config/services/tempo.nix Normal file
View file

@ -0,0 +1,26 @@
{lib, ...}: {
services.tempo = {
enable = true;
settings = {
server = {
http_listen_address = "0.0.0.0";
http_listen_port = 2144;
graceful_shutdown_timeout = "10s";
};
distrubtor.receiver = {
otlp.protocols = {
grpc = {};
http = {};
};
jaeger.protocols = {
thrift_http = {};
grpc = {};
thrift_binary = {};
thrift_compact = {};
};
};
storage.trace.backend = "local";
};
};
services.opentelemetry-collector.enable = lib.mkForce false;
}

1
config/thinkrac.nix
View file

@ -121,4 +121,5 @@
services.blueman.enable = true;
services.tailscale.useRoutingFeatures = "client";
system.autoUpgrade.allowReboot = true;
services.prometheus.exporters.node.enabledCollectors = ["wifi"];
}