add grafana loki promtail for monitoring

hosts/controller/default.nix

@@ -10,6 +10,9 @@
     ../../modules/nixos/vaultwarden.nix
     ../../modules/nixos/uptime-kuma.nix
     ../../modules/nixos/gitea.nix
+    ../../modules/nixos/monitoring/grafana.nix
+    ../../modules/nixos/monitoring/loki.nix
+    ../../modules/nixos/monitoring/promtail.nix
   ];
 
   nixpkgs = {

modules/nixos/monitoring/grafana.nix

@@ -0,0 +1,46 @@
+{
+  lib,
+  username,
+  ...
+}: {
+  services = {
+    grafana = {
+      enable = true;
+      settings = {
+        server.http_port = 8888;
+        security = {
+          admin_user = "${username}";
+          admin_email = "0xtux@pm.me";
+        };
+      };
+    };
+
+    nginx = {
+      enable = lib.mkForce true;
+      virtualHosts = {
+        "grafana.0xtux.com" = {
+          forceSSL = true;
+          enableACME = true;
+          locations = {
+            "/" = {
+              proxyPass = "http://localhost:8888";
+              extraConfig = ''
+                proxy_set_header Host $host;
+              '';
+            };
+
+            "/api/live/" = {
+              proxyPass = "http://localhost:8888";
+              extraConfig = ''
+                proxy_http_version 1.1;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection $connection_upgrade;
+                proxy_set_header Host $host;
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}

modules/nixos/monitoring/loki.nix

@@ -0,0 +1,77 @@
+{lib, ...}: {
+  services = {
+    loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+        server = {
+          http_listen_port = 3100;
+        };
+        common = {
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore = {
+              store = "inmemory";
+            };
+          };
+          replication_factor = 1;
+          path_prefix = "/tmp/loki";
+        };
+        schema_config = {
+          configs = [
+            {
+              from = "2020-05-15";
+              store = "tsdb";
+              object_store = "filesystem";
+              schema = "v13";
+              index = {
+                prefix = "index_";
+                period = "24h";
+              };
+            }
+          ];
+        };
+        storage_config = {
+          filesystem = {
+            directory = "/tmp/loki/chunks";
+          };
+        };
+      };
+    };
+
+    nginx = {
+      enable = lib.mkForce true;
+      virtualHosts = {
+        "loki.0xtux.com" = {
+          forceSSL = true;
+          enableACME = true;
+          locations = {
+            "/" = {
+              proxyPass = "http://localhost:3100";
+              extraConfig = ''
+                proxy_http_version 1.1;
+                proxy_read_timeout 1800s;
+                proxy_connect_timeout 1600s;
+                proxy_set_header Upgrade $http_upgrade;
+                proxy_set_header Connection $connection_upgrade;
+                proxy_set_header Connection "Keep-Alive";
+                proxy_set_header Proxy-Connection "Keep-Alive";
+                proxy_redirect off;
+              '';
+            };
+
+            "/ready" = {
+              proxyPass = "http://localhost:3100";
+              extraConfig = ''
+                proxy_http_version 1.1;
+                proxy_set_header Connection "Keep-Alive";
+                proxy_set_header Proxy-Connection "Keep-Alive";
+                proxy_redirect off;
+              '';
+            };
+          };
+        };
+      };
+    };
+  };
+}

modules/nixos/monitoring/promtail.nix

@@ -0,0 +1,40 @@
+{...}: {
+  services = {
+    promtail = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_port = 9080;
+          grpc_listen_port = 0;
+        };
+        positions = {
+          filename = "/var/lib/promtail/positions.yaml";
+        };
+        clients = [
+          {
+            url = "https://loki.0xtux.com/loki/api/v1/push";
+          }
+        ];
+        scrape_configs = [
+          {
+            job_name = "journal";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+              };
+            };
+            relabel_configs = [
+              {
+                source_labels = [
+                  "__journal__systemd_unit"
+                ];
+                target_label = "unit";
+              }
+            ];
+          }
+        ];
+      };
+    };
+  };
+}