En esta guia se instalarán los siguientes software:

1. Loki: Una base de datos para documtos y Logs
2. promtail: Agente local que envia logs locales a Loki
3. Prometheus: Una base de datos de series de tiempo para almencenar metricas
4. node_exporter: Agente que recaba las metricas locales para enviarlas al servidor Promtheus
5. Grafana: Visualizador de metricas, logs y trazas de fuentes multiples.

DOMINIO=prueba.com
mkdir /opt/loki/
cd /opt/loki/
mkdir certs
cd certs/
openssl genrsa -out ca.key 2048
openssl req -new -x509 -days 365 -key ca.key -subj "/C=CN/ST=GD/L=SZ/O=Acme, Inc./CN=Acme Root CA" -out ca.crt
ls -alh
openssl req -newkey rsa:2048 -nodes -keyout server.key -subj "/C=CN/ST=GD/L=SZ/O=Acme, Inc./CN=logs.$DOMINIO" -out loki.server.csr
openssl x509 -req -extfile <(printf "subjectAltName=DNS:$DOMINIO,DNS:logs.$DOMINIO") -days 1365 -in loki.server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out loki.server.crt
ls

NAME=nextcloud1
cd /opt/loki/certs
mkdir $NAME
cd $NAME
openssl req -newkey rsa:2048 -nodes -keyout client.key -subj "/C=CN/ST=GD/L=SZ/O=Acme, Inc./CN=*.$NAME.com" -out $NAME.client.csr
openssl x509 -req -extfile <(printf "subjectAltName=DNS:$NAME.com,DNS:www.$NAME.com") -days 1365 -in $NAME.client.csr -CA /opt/loki/certs/ca.crt -CAkey /opt/loki/certs/ca.key -CAcreateserial -out $NAME.client.crt
cp /opt/loki/certs/ca.crt .
ls -alh
cd /opt/loki/certs
tar -zcvvf $NAME.tar.gz $NAME
ls -alh

Ver: Prometheus

LOKI_VERSION=$(curl -s "https://api.github.com/repos/grafana/loki/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+')
mkdir /opt/loki
wget -qO /opt/loki/loki.gz "https://github.com/grafana/loki/releases/download/v${LOKI_VERSION}/loki-linux-amd64.zip"
gunzip /opt/loki/loki.gz
ls -alh /opt/loki/loki 
file /opt/loki/loki 
chmod a+x /opt/loki/loki
ln -s /opt/loki/loki /usr/local/bin/loki

wget -qO /opt/loki/loki-local-config.yaml "https://raw.githubusercontent.com/grafana/loki/v${LOKI_VERSION}/cmd/loki/loki-local-config.yaml"
cat /opt/loki/loki-local-config.yaml

auth_enabled: false

server:
  http_listen_port: 3100
  grpc_listen_port: 9096
  log_level: error
  http_tls_config:
    cert_file: /opt/loki/certs/loki.server.crt
    key_file: /opt/loki/certs/server.key
    client_auth_type: RequireAndVerifyClientCert
    client_ca_file: /opt/loki/certs/ca.crt

common:
  path_prefix: /tmp/loki
  storage:
    filesystem:
      chunks_directory: /tmp/loki/chunks
      rules_directory: /tmp/loki/rules
  replication_factor: 1
  ring:
    instance_addr: 127.0.0.1
    kvstore:
      store: inmemory

query_range:
  results_cache:
    cache:
      embedded_cache:
        enabled: true
        max_size_mb: 100
 
#Optimizacion para evitar los mensajes " too many aoutstanding requests"   
query_scheduler:
  max_outstanding_requests_per_tenant: 1024

limits_config:
  split_queries_by_interval: 4h

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

ruler:
  alertmanager_url: http://localhost:9093
 
# By default, Loki will send anonymous, but uniquely-identifiable usage and configuration
# analytics to Grafana Labs. These statistics are sent to https://stats.grafana.org/
#
# Statistics help us better understand how Loki is used, and they show us performance
# levels for most users. This helps us prioritize features and documentation.
# For more information on what's sent, look at
# https://github.com/grafana/loki/blob/main/pkg/usagestats/stats.go
# Refer to the buildReport method to see what goes into a report.
#
# If you would like to disable reporting, uncomment the following lines:
#analytics:
#  reporting_enabled: false

cat /etc/systemd/system/loki.service

[Unit]
Description=Loki log aggregation system
After=network.target
 
[Service]
ExecStart=/opt/loki/loki -config.file=/opt/loki/loki-local-config.yaml
Restart=always
 
[Install]
WantedBy=multi-user.target

loki -version
nano /etc/systemd/system/loki.service
systemctl  enable loki
systemctl  start loki
systemctl  status loki

PROMTAIL_VERSION=$(curl -s "https://api.github.com/repos/grafana/loki/releases/latest" | grep -Po '"tag_name": "v\K[0-9.]+')
mkdir /opt/promtail
wget -qO /opt/promtail/promtail.gz "https://github.com/grafana/loki/releases/download/v${PROMTAIL_VERSION}/promtail-linux-amd64.zip"
#Parar ARM64 (raspbian 64)
#wget -qO /opt/promtail/promtail.gz "https://github.com/grafana/loki/releases/download/v${PROMTAIL_VERSION}/promtail-linux-arm64.zip"
gunzip /opt/promtail/promtail.gz
ls -alh /opt/promtail/promtail 
file /opt/promtail/promtail 
chmod a+x /opt/promtail/promtail
ln -s /opt/promtail/promtail /usr/local/bin/promtail

cat /opt/promtail/promtail-local-config.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: https://logs.$DOMINIO:3100/loki/api/v1/push
    tls_config:
      ca_file: /opt/promtail/certs/ca.crt
      cert_file: /opt/promtail/certs/$NAME.client.crt
      key_file: /opt/promtail/certs/client.key
      server_name: logs.$DOMINIO
      insecure_skip_verify: false

scrape_configs:
  - job_name: system
    static_configs:
    - targets:
        - localhost
      labels:
        job: varlogs
        __path__: /var/log/*log

cat /etc/systemd/system/promtail.service

[Unit]
Description=Promtail log aggregation system
After=network.target
 
[Service]
ExecStart=/opt/promtail/promtail -config.expand-env=true \ 
                                 --client.external-labels=hostname=%H  \
                                 -config.file=/opt/promtail/promtail-local-config.yaml
 
Restart=always
 
[Install]
WantedBy=multi-user.target

nano /etc/systemd/system/promtail.service
 
promtail -version
systemctl  enable promtail
systemctl  start promtail
systemctl  status promtail

echo "*.*@@127.0.0.1:1514;RSYSLOG_SyslogProtocol23Format" >> /etc/rsyslog.conf
systemctl  restart rsyslog

Agregar esto al final del archivo /opt/promtail/promtail-local-config.yaml

  - job_name: syslog
    syslog:
      listen_address: 127.0.0.1:1514
      listen_protocol: tcp
      idle_timeout: 60s
      label_structured_data: yes
      labels:
        job: "syslog"
    relabel_configs: 
      - source_labels: [__syslog_message_hostname] 
        target_label: host 
      - source_labels: [__syslog_message_hostname] 
        target_label: hostname 
      - source_labels: [__syslog_message_severity] 
        target_label: level 
      - source_labels: [__syslog_message_app_name] 
        target_label: application 
      - source_labels: [__syslog_message_facility] 
        target_label: facility 
      - source_labels: [__syslog_connection_hostname] 
        target_label: connection_hostname

Agregar esto al final del archivo /opt/promtail/promtail-local-config.yaml

  - job_name: nginx
    static_configs:
      - targets:
          - localhost
        labels:
          job: nginx
          __path__: /var/log/nginx/*log

Agregar esto al final del archivo /opt/promtail/promtail-local-config.yaml

  - job_name: system
    static_configs:
    - targets:
        - localhost #Promtail target is localhost
      labels:
        instance: nubeades
        env: home-lab #Environment label
        job: nextcloud #Job label
        __path__: /srv/www/nextcloud/nextcloud/data/{nextcloud,audit}.log

apt-get install -y apt-transport-https
apt-get install -y software-properties-common wget
wget -q -O /usr/share/keyrings/grafana.key https://apt.grafana.com/gpg.key
echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com stable main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
echo "deb [signed-by=/usr/share/keyrings/grafana.key] https://apt.grafana.com beta main" | sudo tee -a /etc/apt/sources.list.d/grafana.list
apt-get update
apt-get install grafana
systemctl daemon-reload
systemctl enable grafana-server
systemctl start grafana-server
systemctl status grafana-server

Ahora a ingresar por primera vez al grafana en la direccion http://localhost:3000 Para poder cambiarle la clave al usuario admin.

Usuario: admin
Clave: admin

Dentro de grafana ir a Configuracion–>Data Sources–>Add new data souce –>Prometheus Luego en los datos de prometheus ingresar:

• URL: http://localhost:9090
• Basic auth: activado
• User: promadmin
• Password: la clave que se creo durante la instalacion

Luego probamos en opcion de Explore y seleccionamos las siguientes opciones

1. Data Source: Prometheus
2. Metric: node_memory_MemTotal_bytes
3. Botón: Run Query

Dentro de grafana ir a Configuracion–>Data Sources–>Add new data souce –>Loki Luego en los datos de loki ingresar:

1. URL: https://localhost:3100 (Ojo, es httpS lo que indica que usa certificados )
2. TLS Client Auth: Activado
3. ServerName: prueba.com
4. Client Cert: El contenido del archivo /opt/promtail/certs/grafana.client.crt
5. Client Key: El contenido del archivo /opt/promtail/certs/client.key

Luego de guardar, probamos la fuente de datos de la sección de Explore

1. Data Source: Loki
2. Label Filter: host=server (o el que aparezca)
3. Botón: Run Query

• Nextcloud
• Syslog

Ahora vamos a definir un subdirectorio (grafana)con reverse proxy en nginx, para ello modificamos las siguientes lineas cat /etc/grafana/grafana.ini

domain = logs.midominio.com
root_url = %(protocol)s://%(domain)s/grafana/

max_idle_connections = 1000

Agregar en el archivo /etc/nginx/sites-enabled/default antes de la sección server {…}

map $http_upgrade $connection_upgrade {
  default upgrade;
  '' close;
}
upstream grafana {
  server localhost:3000;
}

Y esta parte en el mismo archivo, dentro de la sección server {…}

    location /grafana/ {
      proxy_set_header Host $http_host;
      rewrite  ^/grafana/(.*)  /$1 break;
      #proxy_pass       http://127.0.0.1:3000; 
      proxy_pass http://grafana;

    }
    # Proxy Grafana Live WebSocket connections.
   location /grafana/api/live {
    rewrite  ^/grafana/api/live(.*)  /$1 break;
    proxy_http_version 1.1;
    proxy_set_header Host $http_host;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection $connection_upgrade;
    proxy_set_header Host $http_host;
    #proxy_pass http://127.0.0.1:3000;
     proxy_pass http://grafana;

  }

Reiniciamos los servicios

systemctl restart nginx
systemctl restart grafana

Y luego abrir el navegador usando la direccion http://127.0.0.1/grafana/

Agregar las siguientes lineas al archivo /etc/grafana/grafana.ini, en la sección de [smtp]

enabled = true
host = smtp.gmail.com:587
user = micorreodegmail@gmail.com
password = LASUPERCLAVE
from_address = micorreodegmail@gmail.com
ehlo_identity = logs.midominio.com

Para configurar el envio de correo de Gmail, se debe tener activada la autenticacion de dos factores y luego crear una clave para la aplicacion. Pueden leer mas acá https://support.google.com/accounts/answer/185833?hl=en/

En el nginx agregar

    location /prometheus/ {
      auth_basic "Prometheus";
      auth_basic_user_file "/etc/prometheus/.htpasswd";
      proxy_pass       http://127.0.0.1:9090;
    }

en el archivo /etc/systemd/system/prometheus.service modificar la sección a

 
ExecStart=/usr/local/bin/prometheus \
--config.file /etc/prometheus/prometheus.yml \
--storage.tsdb.path /var/lib/prometheus/ \
--web.console.templates=/etc/prometheus/consoles \
--web.console.libraries=/etc/prometheus/console_libraries \
--web.external-url=/prometheus/ \
--web.listen-address="127.0.0.1:9090"

En el grafana cambiar la URL del Data Source de Prometheus

• URL: http://logs.prueba.com/prometheus/

En la configuración de prometheus /etc/prometheus/prometheus.yml, cambiar el job local a

  - job_name: "prometheus"

    # metrics_path defaults to '/metrics'
    metrics_path: '/prometheus/metrics' #Que los sirve nginx en subdir

    static_configs:
      - targets: ["127.0.0.1:9090"]

1. No se han utilizado certificados para el sitio nginx
2. No se ha puesto el prometheus en una subcarpeta para mejor servicio

• https://lindevs.com/install-loki-on-ubuntu
• https://unixcop.com/grafana-loki-installation/
• https://grafana.com/docs/grafana/latest/setup-grafana/installation/debian/
• https://community.grafana.com/t/client-certificate-authentication-between-promtail-and-loki/78431
• https://www.virtualizationhowto.com/2022/11/grafana-loki-configuration-syslog-server-for-home-labs/
• https://sbcode.net/grafana/nginx-promtail/
• https://community.grafana.com/t/how-to-add-variable-hostname-label-to-static-config-in-promtail/68352/12
• https://voidquark.com/parsing-nextcloud-audit-logs-with-grafana-loki/
• https://community.grafana.com/t/client-certificate-authentication-between-promtail-and-loki/78431/2
• https://github.com/grafana/loki/issues/5123
• https://sbcode.net/grafana/nginx-promtail/
• https://geekflare.com/open-source-centralized-logging/
• https://grafana.com/blog/2020/04/21/how-labels-in-loki-can-make-log-queries-faster-and-easier/
• https://sbcode.net/grafana/nginx-promtail/
• https://grafana.com/blog/2021/03/23/how-i-fell-in-love-with-logs-thanks-to-grafana-loki/
• https://geekflare.com/prometheus-grafana-setup-for-linux/
• https://linux.how2shout.com/how-to-install-prometheus-in-debian-11-or-ubuntu-20-04/