Add the monitoring stack

monitoring/files/loki-config.yaml

@@ -0,0 +1,64 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9096
+  log_level: debug
+  grpc_server_max_concurrent_streams: 1000
+
+common:
+  instance_addr: 127.0.0.1
+  path_prefix: /tmp/loki
+  storage:
+    filesystem:
+      chunks_directory: /tmp/loki/chunks
+      rules_directory: /tmp/loki/rules
+  replication_factor: 1
+  ring:
+    kvstore:
+      store: inmemory
+
+query_range:
+  results_cache:
+    cache:
+      embedded_cache:
+        enabled: true
+        max_size_mb: 100
+
+# limits_config:
+#   metric_aggregation_enabled: truez
+
+schema_config:
+  configs:
+    - from: 2020-10-24
+      store: tsdb
+      object_store: filesystem
+      schema: v13
+      index:
+        prefix: index_
+        period: 24h
+
+pattern_ingester:
+  enabled: true
+  # metric_aggregation:
+  #   enabled: true
+  #   loki_address: localhost:3100
+
+ruler:
+  alertmanager_url: http://localhost:9093
+
+frontend:
+  encoding: protobuf
+
+# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
+# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
+#
+# Statistics help us better understand how Loki is used, and they show us performance
+# levels for most users. This helps us prioritize features and documentation.
+# For more information on what's sent, look at
+# https://github.com/grafana/loki/blob/main/pkg/analytics/stats.go
+# Refer to the buildReport method to see what goes into a report.
+#
+# If you would like to disable reporting, uncomment the following lines:
+#analytics:
+#  reporting_enabled: false

monitoring/files/nginx.conf.j2

@@ -0,0 +1,20 @@
+server {
+
+  listen 80;
+  server_name {{ http_host }};
+
+  location / {
+    proxy_set_header Host $http_host;
+    proxy_pass http://localhost:3005/;
+  }
+
+  location /api/live/ {
+    proxy_http_version 1.1;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection "upgrade";
+    proxy_set_header Host $http_host;
+    proxy_pass http://localhost:3005/;
+  }
+
+}
+

monitoring/files/passwd

@@ -0,0 +1,2 @@
+esp8266:esp8266
+nodered:nodered

monitoring/files/promtail-config.yaml

@@ -0,0 +1,25 @@
+server:
+  http_listen_port: "{{ promtail_port }}"
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: "http://loki:{{ loki_port }}/loki/api/v1/push"
+
+scrape_configs:
+- job_name: system
+  static_configs:
+  - targets:
+      - localhost
+    labels:
+      job: varlogs
+      __path__: /var/log/*log
+- job_name: docker
+  static_configs:
+    - targets:
+        - localhost
+      labels:
+        job: docker
+        __path__: /var/lib/docker/containers/*/*.log

monitoring/playbook.yml

@@ -0,0 +1,14 @@
+---
+- name: Tanshu Monitoring Playbook
+  hosts: all
+  become: true
+  vars_files:
+    - vars/default.yml
+
+  roles:
+    - cadvisor
+    - node-exporter
+    - loki
+    - promtail
+    # - grafana
+    # - nginx

monitoring/readme.md

@@ -0,0 +1,8 @@
+# Mosquitto
+Permissions are all fucked up.
+Need to check the user and set the files permission to it
+
+
+Make the tunnel using either of the following to access node-red or influxdb web interfaces
+ssh -NT -L 8086:localhost:8086 -L 1880:localhost:1880 beacon
+ssh -L 8086:localhost:8086 -L 1880:localhost:1880 beacon

monitoring/roles/cadvisor/tasks/main.yaml

@@ -0,0 +1,26 @@
+---
+# Tasks for cAdvisor (Container Advisor) role
+- name: Pull Container Advisor image
+  docker_image:
+    name: "{{ cadvisor_image }}"
+    source: pull
+    force_source: yes
+
+- name: Create Container Advisor container
+  docker_container:
+    name: "{{ cadvisor_container }}"
+    image: "{{ cadvisor_image }}"
+    state: started
+    restart_policy: "unless-stopped"
+    env:
+      TZ: "Asia/Kolkata"
+    published_ports:
+      - "127.0.0.1:{{ cadvisor_port }}:8080"
+    volumes:
+      - /:/rootfs:ro
+      - /var/run:/var/run:ro
+      - /sys:/sys:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    privileged: yes
+    devices:
+      - "/dev/kmsg"

monitoring/roles/grafana/defaults/main.yaml

@@ -0,0 +1,6 @@
+---
+# Default variables for Grafana role
+grafana_user: "{{ ansible_user }}"
+grafana_image: "grafana/grafana:latest"
+grafana_directory: "/opt/grafana"
+grafana_container: "grafana"

monitoring/roles/grafana/tasks/main.yaml

@@ -0,0 +1,42 @@
+---
+# Tasks for Grafana role
+- name: Look up the Grafana user
+  getent:
+    database: passwd
+    key: "{{ grafana_user }}"
+    split: ":"
+
+- name: Pull Grafana image
+  docker_image:
+    name: "{{ grafana_image }}"
+    source: pull
+    force_source: yes
+
+- name: Ensure Grafana Directory exists
+  file:
+      path: "{{ grafana_directory }}"
+      state: directory
+      group: "{{ grafana_user }}"
+      owner: "{{ grafana_user }}"
+      mode: 0755
+
+- name: Create Grafana container
+  docker_container:
+    name: "{{ grafana_container }}"
+    image: "{{ grafana_image }}"
+    state: started
+    restart_policy: "unless-stopped"
+    user: "{{ getent_passwd[grafana_user][1] }}:{{ getent_passwd[grafana_user][2] }}"
+    env:
+      GF_SERVER_ROOT_URL: "https://{{ http_host }}"
+      GF_INSTALL_PLUGINS: "grafana-clock-panel, grafana-simple-json-datasource, natel-discrete-panel, briangann-gauge-panel, vonage-status-panel, neocat-cal-heatmap-panel, natel-plotly-panel"
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_AUTH_ANONYMOUS_ORG_NAME: "Public"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
+      GF_AUTH_ANONYMOUS_HIDE_VERSION: "true"
+    links:
+      - "{{ influx_container }}:influx"
+    published_ports:
+      - 127.0.0.1:3005:3000
+    volumes:
+      - "{{ grafana_directory }}:/var/lib/grafana"

monitoring/roles/loki/tasks/main.yaml

@@ -0,0 +1,48 @@
+---
+# The image is configured to run by default as user loki with UID 10001 and GID 10001.
+# You can use a different user, specially if you are using bind mounts, by specifying
+# the UID with a docker run command and using --user=UID with a numeric UID suited to your needs.
+
+# Tasks for Loki role
+- name: Pull Loki image
+  docker_image:
+    name: "{{ loki_image }}"
+    source: pull
+    force_source: yes
+
+- name: Ensure Loki Directory exists
+  file:
+      path: "{{ loki_directory }}"
+      state: directory
+      group: "{{ loki_user }}"
+      owner: "{{ loki_user }}"
+      mode: 0755
+
+# - name: Ensure Loki Config Directory exists
+#   file:
+#       path: "{{ loki_directory }}/config"
+#       state: directory
+#       group: "{{ loki_user }}"
+#       owner: "{{ loki_user }}"
+#       mode: 0755
+
+- name: Copy the Loki conf file
+  template:
+    src: "files/loki-config.yaml"
+    dest: "{{ loki_directory }}/loki-config.yaml"
+    group: "{{ loki_user }}"
+    owner: "{{ loki_user }}"
+    mode: 0644
+
+- name: Create Loki container
+  docker_container:
+    name: "{{ loki_container }}"
+    image: "{{ loki_image }}"
+    state: started
+    restart_policy: "unless-stopped"
+    published_ports:
+      - "127.0.0.1:{{ loki_port }}:3100"
+    volumes:
+      - "{{ loki_directory }}:/mnt/config"
+    command:
+      - "-config.file=/mnt/config/loki-config.yaml"

monitoring/roles/node-exporter/tasks/main.yaml

@@ -0,0 +1,25 @@
+---
+# Tasks for Node exporter role
+- name: Pull Node exporter image
+  docker_image:
+    name: "{{ node_exporter_image }}"
+    source: pull
+    force_source: yes
+
+- name: Create Node exporter container
+  docker_container:
+    name: "{{ node_exporter_container }}"
+    image: "{{ node_exporter_image }}"
+    state: started
+    restart_policy: "unless-stopped"
+    published_ports:
+      - "127.0.0.1:{{ node_exporter_port }}:9100"
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro,rslave
+    command:
+      - '--path.procfs=/host/proc'
+      - '--path.rootfs=/rootfs'
+      - '--path.sysfs=/host/sys'
+      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'

monitoring/roles/promtail/defaults/main.yaml

@@ -0,0 +1,6 @@
+---
+# Default variables for Mosquitto role
+mosquitto_user: "{{ ansible_user }}"
+mosquitto_image: "mosquitto:latest"
+mosquitto_directory: "/opt/mosquitto"
+mosquitto_container: "mosquitto"

monitoring/roles/promtail/tasks/main.yaml

@@ -0,0 +1,44 @@
+---
+# The image is configured to run by default as user promtail with UID 10001 and GID 10001.
+# You can use a different user, specially if you are using bind mounts, by specifying
+# the UID with a docker run command and using --user=UID with a numeric UID suited to your needs.
+
+# Tasks for Promtail role
+- name: Pull Promtail image
+  docker_image:
+    name: "{{ promtail_image }}"
+    source: pull
+    force_source: yes
+
+- name: Ensure Promtail Directory exists
+  file:
+      path: "{{ promtail_directory }}"
+      state: directory
+      group: "{{ promtail_user }}"
+      owner: "{{ promtail_user }}"
+      mode: 0755
+
+- name: Copy the Promtail conf file
+  template:
+    src: "files/promtail-config.yaml"
+    dest: "{{ promtail_directory }}/promtail-config.yaml"
+    group: "{{ promtail_user }}"
+    owner: "{{ promtail_user }}"
+    mode: 0644
+
+- name: Create Promtail container
+  docker_container:
+    name: "{{ promtail_container }}"
+    image: "{{ promtail_image }}"
+    state: started
+    restart_policy: "unless-stopped"
+    links:
+      - "{{ loki_container }}:loki"
+    volumes:
+      - "{{ promtail_directory }}:/mnt/config"
+      - "/var/log:/var/log:ro"
+      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
+    # published_ports:
+    #   - "127.0.0.1:{{ promtail_port }}:9080"
+    command:
+      - "-config.file=/mnt/config/promtail-config.yaml"

monitoring/vars/default.yml

@@ -0,0 +1,46 @@
+---
+host_directory: /var/lib/monitoring
+user: 'www-data'
+
+cadvisor_container: cadvisor
+cadvisor_image: gcr.io/cadvisor/cadvisor:latest
+cadvisor_port: 8085
+
+node_exporter_container: node-exporter
+node_exporter_image: prom/node-exporter:latest
+node_exporter_port: 9100
+
+loki_container: loki
+loki_image: grafana/loki:latest
+loki_directory: "{{ host_directory }}/loki"
+loki_user: 10001
+loki_port: 3100
+
+promtail_container: promtail
+promtail_image: grafana/promtail:latest
+promtail_directory: "{{ host_directory }}/promtail"
+promtail_user: "{{ loki_user }}"
+promtail_port: 9080
+
+# mosquitto_container: mosquitto
+# mosquitto_image: eclipse-mosquitto:latest
+# mosquitto_directory: "{{ host_directory }}/mosquitto"
+# mosquitto_user: 'root'
+
+# nodered_container: nodered
+# nodered_image: nodered/node-red:latest
+# nodered_directory: "{{ host_directory }}/nodered"
+# nodered_user: "{{ user }}"
+
+# influx_container: influxtestplay
+# influx_image: influxdb:latest
+# influx_directory: "{{ host_directory }}/influxtestplaydir"
+# influx_user: "{{ user }}"
+
+# grafana_container: grafana
+# grafana_image: grafana/grafana-enterprise:latest
+# grafana_directory: "{{ host_directory }}/grafana"
+# grafana_user: "{{ user }}"
+
+# http_host: "iot.tanshu.com"
+# http_conf: "iot.tanshu.com.conf"