darkkirb/nixos-config
1
0
Fork 0
Code Issues 1 Pull requests 2 Projects Releases Packages Wiki Activity Actions

make prometheus/loki system-local

Browse source
This commit is contained in:
Charlotte 🦝 Delenk 2023-12-10 10:36:28 +01:00
parent ac3b29674c
commit 232be57ee4
No known key found for this signature in database
24 changed files with 275 additions and 240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
config/default.nix
View file

@ -5,7 +5,6 @@
...
}: {
imports = [
./zfs.nix
./users/darkkirb.nix
./users/root.nix
./nix.nix
@ -14,6 +13,8 @@
./services/restic.nix
./specialization.nix
./services/promtail.nix
./services/loki.nix
./services/prometheus.nix
./env.nix
./tailscale.nix
];
@ -32,22 +33,6 @@
users.defaultUserShell = pkgs.zsh;
# Prometheus node exporter
services.prometheus.exporters.node = {
enable = true;
enabledCollectors = [
"interrupts"
"lnstat"
"mountstats"
"network_route"
"ntp"
"processes"
"systemd"
"tcpstat"
];
listenAddress = "0.0.0.0";
};
environment.pathsToLink = ["/share/zsh"];
console.keyMap = "neo";
@ -58,7 +43,6 @@
programs.zsh.enable = true;
users.mutableUsers = false;
boot.kernelParams = ["nohibernate"];
sops.secrets."root/aws/credentials" = {
sopsFile = ../secrets/shared.yaml;

1
config/instance-20221213-1915.nix
View file

@ -28,6 +28,7 @@
./services/uptime-kuma.nix
./services/matrix-sliding-sync.nix
./wireguard
./zfs.nix
];
boot.initrd.availableKernelModules = ["xhci_pci" "virtio_pci" "usbhid"];

2
config/nixos-8gb-fsn1-1.nix
View file

@ -22,7 +22,6 @@
./services/postfixadmin.nix
./services/dovecot.nix
./services/postfix.nix
./services/loki.nix
./services/reverse-proxy.nix
./services/matrix-media-repo.nix
./bittorrent-blocker.nix
@ -34,6 +33,7 @@
./services/wordpress.nix
./services/initrd-ssh.nix
./wireguard
./zfs.nix
];
boot.initrd.availableKernelModules = ["ata_piix" "virtio_pci" "virtio_scsi" "xhci_pci" "sd_mod" "sr_mod"];

1
config/rainbow-resort.nix
View file

@ -101,4 +101,5 @@
services.tailscale.useRoutingFeatures = "client";
home-manager.users.darkkirb._module.args.withNSFW = lib.mkForce true;
system.autoUpgrade.allowReboot = true;
services.prometheus.exporters.node.enabledCollectors = ["drm"];
}

1
config/server.nix
View file

@ -3,6 +3,5 @@
imports = [
./services/caddy
./services/acme.nix
./services/fail2ban.nix
];
}

14
config/services/dovecot.nix
View file

@ -139,8 +139,20 @@ in {
};
services.prometheus.exporters.dovecot = {
enable = true;
listenAddress = "0.0.0.0";
port = 35496;
};
services.prometheus.scrapeConfigs = [
{
job_name = "dovecot";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.dovecot.port}"
];
}
];
}
];
sops.secrets."services/dovecot/rspamd_password" = {owner = "dovecot";};
sops.secrets."services/dovecot/dovecot-sql.conf.ext" = {owner = "dovecot";};
services.postgresql.ensureUsers = [

7
config/services/elasticsearch.nix
View file

@ -1,7 +0,0 @@
_: {
services.elasticsearch = {
enable = true;
cluster_name = "chir-rs";
};
# TODO: elasticsearch-exporter
}

19
config/services/fail2ban.nix
View file

@ -1,19 +0,0 @@
_: {
services.fail2ban = {
enable = true;
bantime-increment.enable = true;
bantime-increment.maxtime = "48h";
jails = {
dovecot = ''
enabled = true
filter = dovecot
action = iptables-multiport[name=dovecot, port="pop3,pop3s,imap,imaps", protocol=tcp]
'';
postfix = ''
enabled = true
filter = postfix
action = iptables-multiport[name=postfix, port="imap,imaps,submission", protocol=tcp]
'';
};
};
}

12
config/services/gitea.nix
View file

@ -95,4 +95,16 @@
port = 6379;
};
sops.secrets."services/gitea" = {owner = "gitea";};
services.prometheus.scrapeConfigs = [
{
job_name = "forgejo";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.config.services.gitea.httpPort}"
];
}
];
}
];
}

3
config/services/grafana.nix
View file

@ -4,9 +4,6 @@
pkgs,
...
}: {
imports = [
./prometheus.nix
];
services.grafana = {
enable = true;
domain = "grafana.int.chir.rs";

12
config/services/hydra.nix
View file

@ -178,4 +178,16 @@ in {
OnUnitActiveSec = 604800;
};
};
services.prometheus.scrapeConfigs = [
{
job_name = "hydra";
static_configs = [
{
targets = [
"127.0.0.1:9199"
];
}
];
}
];
}

13
config/services/kubo-common.nix
View file

@ -89,4 +89,17 @@
networking.firewall.allowedUDPPorts = [
4001
];
services.prometheus.scrapeConfigs = [
{
job_name = "kubo";
metrics_path = "/debug/metrics/prometheus";
static_configs = [
{
targets = [
"127.0.0.1:5001"
];
}
];
}
];
}

74
config/services/loki.nix
View file

@ -1,6 +1,78 @@
_: {
services.loki = {
enable = true;
configFile = ./loki.yaml;
configuration = {
server.http_listen_port = 24545;
auth_enabled = false;
ingester = {
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore = {
store = "inmemory";
};
replication_factor = 1;
};
};
chunk_idle_period = "1h";
max_chunk_age = "1h";
chunk_target_size = 999999;
chunk_retain_period = "30s";
max_transfer_retries = 0;
};
schema_config = {
configs = [
{
from = "2022-06-06";
store = "boltdb-shipper";
object_store = "filesystem";
schema = "v11";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
storage_config = {
boltdb_shipper = {
active_index_directory = "/var/lib/loki/boltdb-shipper-active";
cache_location = "/var/lib/loki/boltdb-shipper-cache";
cache_ttl = "24h";
shared_store = "filesystem";
};
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
limits_config = {
reject_old_samples = true;
reject_old_samples_max_age = "168h";
};
chunk_store_config = {
max_look_back_period = "0s";
};
table_manager = {
retention_deletes_enabled = false;
retention_period = "0s";
};
compactor = {
working_directory = "/var/lib/loki";
shared_store = "filesystem";
compactor_ring = {
kvstore = {
store = "inmemory";
};
};
};
};
};
}

30
config/services/loki.yaml
View file

@ -1,30 +0,0 @@
auth_enabled: false
server:
http_listen_port: 3100
grpc_listen_port: 9096
common:
path_prefix: /tmp/loki
storage:
filesystem:
chunks_directory: /tmp/loki/chunks
rules_directory: /tmp/loki/rules
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
schema_config:
configs:
- from: 2020-10-24
store: boltdb-shipper
object_store: filesystem
schema: v11
index:
prefix: index_
period: 24h
ruler:
alertmanager_url: http://localhost:9093

12
config/services/matrix-media-repo.nix
View file

@ -216,4 +216,16 @@ in {
}
'';
};
services.prometheus.scrapeConfigs = [
{
job_name = "matrix-media-repo";
static_configs = [
{
targets = [
"127.0.0.1:9000"
];
}
];
}
];
}

13
config/services/matrix-sliding-sync.nix
View file

@ -11,6 +11,7 @@
SYNCV3_SERVER = "https://matrix.chir.rs";
SYNCV3_DB = "postgresql:///sliding_sync?sslmode=disable&host=/run/postgresql";
SYNCV3_BINDADDR = "127.0.0.1:45587";
SYNCV3_PROM = "127.0.0.1:50372";
};
serviceConfig = {
Type = "simple";
@ -47,4 +48,16 @@
reverse_proxy http://127.0.0.1:45587
'';
};
services.prometheus.scrapeConfigs = [
{
job_name = "matrix-sliding-sync";
static_configs = [
{
targets = [
"127.0.0.1:50372"
];
}
];
}
];
}

14
config/services/named-submissive.nix
View file

@ -41,8 +41,20 @@ in {
enable = true;
bindGroups = ["server" "view" "tasks"];
bindURI = "http://127.0.0.1:8653/";
port = 1533;
};
services.prometheus.scrapeConfigs = [
{
job_name = "bind";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.bind}"
];
}
];
}
];
systemd.tmpfiles.rules = [
"d /var/lib/named 4700 named named - -"
];

13
config/services/named.nix
View file

@ -130,6 +130,19 @@ in {
enable = true;
bindGroups = ["server" "view" "tasks"];
bindURI = "http://127.0.0.1:8653/";
port = 1533;
};
services.prometheus.scrapeConfigs = [
{
job_name = "bind";
static_configs = [
{
targets = [
"127.0.0.1:${toString config.services.prometheus.exporters.bind}"
];
}
];
}
];
sops.secrets."services/dns/named-keys" = {owner = "named";};
}

15
config/services/postgres.nix
View file

@ -19,6 +19,19 @@
services.prometheus.exporters.postgres = {
enable = true;
user = "postgres";
listenAddress = "0.0.0.0";
port = "1589"
};
services.prometheus.scrapeConfigs = [
{
job_name = "postgresql";
static_configs = [
{
targets = [
"127.0.0.1:1589"
];
}
];
}
];
}

129
config/services/prometheus.nix
View file

@ -1,128 +1,33 @@
{config, ...}: {
services.prometheus = {
port = 26678;
enable = true;
port = 9002;
globalConfig = {
scrape_interval = "10s";
scrape_timeout = "2s";
exporters = {
node = {
port = 31941;
enabledCollectors = [
"buddyinfo"
"cgroups"
"systemd"
"ethtool"
];
enable = true;
};
};
# ingest the published nodes
scrapeConfigs = [
{
job_name = "node_exporter";
job_name = "nodes";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"nutty-noon.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"nas.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"thinkrac.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"instance-20221213-1915.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"vf2.int.chir.rs:${toString config.services.prometheus.exporters.node.port}"
"127.0.0.1:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
{
job_name = "bind_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.bind.port}"
];
}
];
}
{
job_name = "postgres_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.postgres.port}"
"nas.int.chir.rs:${toString config.services.prometheus.exporters.postgres.port}"
"instance-20221213-1915.int.chir.rs:${toString config.services.prometheus.exporters.postgres.port}"
];
}
];
}
{
job_name = "gitea_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.gitea.httpPort}"
];
}
];
}
{
job_name = "dovecot_exporter";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:${toString config.services.prometheus.exporters.dovecot.port}"
];
}
];
}
{
job_name = "hydra_exporter";
static_configs = [
{
targets = [
"nas.int.chir.rs:9199"
];
}
];
}
{
job_name = "statsd_exporter";
static_configs = [
{
targets = [
"nas.int.chir.rs:9102"
];
}
];
}
{
job_name = "matrix_media_repo";
static_configs = [
{
targets = [
"nixos-8gb-fsn1-1.int.chir.rs:9000"
];
}
];
}
{
job_name = "rspamd_exporter";
static_configs = [
{
targets = [
"nas.int.chir.rs:7980"
];
}
];
}
{
job_name = "synapse";
scrape_interval = "30s";
scrape_timeout = "10s";
metrics_path = "/_synapse/metrics";
static_configs = [
{
targets = [
"matrix.int.chir.rs:80"
];
labels = {
instance = "matrix.chir.rs";
job = "master";
index = "1";
};
}
];
}
];
checkConfig = false;
};
}

79
config/services/promtail.nix
View file

@ -1,47 +1,38 @@
{
config,
pkgs,
lib,
...
}: let
promtail_config = {
server = {
http_listen_port = 28183;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
client = {
url = "http://nixos-8gb-fsn1-1.int.chir.rs:3100/loki/api/v1/push";
external_labels.host = config.networking.hostName;
};
scrape_configs = [
{
job_name = "journal";
journal = {
max_age = "12h";
labels.job = "systemd-journal";
};
relabel_configs = [
{
source_labels = ["__journal__systemd_unit"];
target_label = "unit";
}
];
}
];
};
promtail_yml = pkgs.writeText "promtail.yml" (lib.generators.toYAML {} promtail_config);
in {
systemd.services.promtail = {
description = "Promtail service for Loki";
wantedBy = ["multi-user.target"];
serviceConfig = {
ExecStart = ''
${pkgs.grafana-loki}/bin/promtail --config.file ${promtail_yml}
'';
{config, ...}: {
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 45871;
grpc_listen_port = 0;
};
positions = {
filename = "/tmp/positions.yaml";
};
clients = [
{
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
}
];
scrape_configs = [
{
job_name = "journal";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
host = "${config.networking.hostName}";
};
};
relabel_configs = [
{
source_labels = ["__journal__systemd_unit"];
target_label = "unit";
}
];
}
];
};
# extraFlags
};
}

17
config/services/rspamd.nix
View file

@ -194,5 +194,20 @@
sops.secrets."services/rspamd/dkim/darkkirb.de" = {owner = "rspamd";};
sops.secrets."services/rspamd/dkim/miifox.net" = {owner = "rspamd";};
sops.secrets."services/rspamd/dkim/chir.rs" = {owner = "rspamd";};
services.prometheus.exporters.rspamd.enable = true;
services.prometheus.exporters.rspamd = {
enable = true;
port = 58636;
};
services.prometheus.scrapeConfigs = [
{
job_name = "rspamd";
static_configs = [
{
targets = [
"127.0.0.1:58636"
];
}
];
}
];
}

13
config/services/synapse.nix
View file

@ -192,4 +192,17 @@
key = "services/mautrix/whatsapp.yaml";
owner = "matrix-synapse";
};
services.prometheus.scrapeConfigs = [
{
job_name = "matrix-synapse";
metrics_path = "/_synapse/metrics";
static_configs = [
{
targets = [
"127.0.0.1:8008"
];
}
];
}
];
}

1
config/thinkrac.nix
View file

@ -121,4 +121,5 @@
services.blueman.enable = true;
services.tailscale.useRoutingFeatures = "client";
system.autoUpgrade.allowReboot = true;
services.prometheus.exporters.node.enabledCollectors = ["wifi"];
}