Falco’s metrics are often scraped by Prometheus and visualized in Grafana to gain deep insights into system security events.
Here’s Falco running locally, exposing metrics on port 8080:
{
"output_fields": {
"@timestamp": "2023-10-27T10:00:00.123Z",
"container.id": "a1b2c3d4e5f6",
"container.name": "my-app-container",
"fd.name": "/var/log/syslog",
"fd.path": "/var/log/syslog",
"proc.cmdline": "/usr/sbin/syslogd -n",
"proc.exe": "/usr/sbin/syslogd",
"rule.name": "Write to sensitive file",
"severity": "error",
"user.name": "root",
"container.image": "ubuntu:latest",
"k8s.pod.name": "my-app-pod-xyz",
"k8s.namespace": "default",
"evt.type": "openat",
"evt.code": "openat",
"file.name": "syslog",
"file.path": "/var/log/syslog",
"file.dir": "/var/log",
"file.basename": "syslog",
"file.extension": "",
"file.mode": "0644",
"file.owner": "syslog",
"file.group": "syslog",
"net.protocol": "tcp",
"net.src.ip": "192.168.1.100",
"net.dst.ip": "192.168.1.1",
"net.src.port": 54321,
"net.dst.port": 80,
"proc.pid": 1234,
"proc.ppid": 1,
"proc.ppid": 1,
"proc.cwd": "/",
"proc.loginuid": 1000,
"proc.sid": 1234,
"proc.threads": 1,
"proc.uid": 0,
"proc.gid": 0,
"proc.euid": 0,
"proc.egid": 0,
"proc.suid": 0,
"proc.sgid": 0,
"proc.fsuid": 0,
"proc.fsgid": 0,
"proc.mntns": 123,
"proc.pidns": 456,
"proc.user": "root",
"proc.group": "root",
"proc.exe_path": "/usr/sbin/syslogd",
"proc.cmdline_path": "/usr/sbin/syslogd -n",
"proc.args": ["/usr/sbin/syslogd", "-n"],
"proc.comm": "syslogd",
"proc.tty": "pts/0",
"proc.nlwp": 1,
"proc.start_time": "2023-10-27T09:00:00.000Z",
"proc.parent.pid": 1,
"proc.parent.exe": "/sbin/init",
"proc.parent.cmdline": "/sbin/init",
"proc.parent.name": "init",
"proc.parent.uid": 0,
"proc.parent.gid": 0,
"proc.parent.euid": 0,
"proc.parent.egid": 0,
"proc.parent.suid": 0,
"proc.parent.sgid": 0,
"proc.parent.fsuid": 0,
"proc.parent.fsgid": 0,
"proc.parent.mntns": 123,
"proc.parent.pidns": 456,
"proc.parent.user": "root",
"proc.parent.group": "root",
"proc.parent.exe_path": "/sbin/init",
"proc.parent.cmdline_path": "/sbin/init",
"proc.parent.args": ["/sbin/init"],
"proc.parent.comm": "init",
"proc.parent.tty": "unknown",
"proc.parent.nlwp": 1,
"proc.parent.start_time": "2023-10-27T08:00:00.000Z",
"container.runtime": "docker",
"container.image.id": "sha256:abcdef123456",
"container.privileged": false,
"container.mounts": [
{"destination": "/dev", "source": "tmpfs", "type": "tmpfs"},
{"destination": "/proc", "source": "proc", "type": "proc"},
{"destination": "/sys", "source": "sysfs", "type": "sysfs"}
],
"k8s.pod.uid": "abcdef12-3456-7890-abcd-ef1234567890",
"k8s.node.name": "worker-node-1",
"k8s.pod.labels": {"app": "my-app", "version": "1.0"},
"k8s.pod.annotations": {"description": "My application"},
"k8s.pod.ip": "10.244.0.5",
"k8s.pod.host_ip": "192.168.1.100",
"k8s.pod.start_time": "2023-10-27T09:30:00.000Z",
"k8s.namespace.uid": "fedcba98-7654-3210-dcba-fedcba987654",
"k8s.cluster.name": "my-k8s-cluster",
"k8s.cluster.uid": "12345678-90ab-cdef-1234-567890abcdef",
"k8s.daemonset.name": "falco",
"k8s.daemonset.uid": "abcdef01-2345-6789-abcd-ef0123456789",
"k8s.deployment.name": "my-app-deployment",
"k8s.deployment.uid": "12345678-90ab-cdef-1234-567890abcdef",
"k8s.replicaset.name": "my-app-replicaset",
"k8s.replicaset.uid": "abcdef01-2345-6789-abcd-ef0123456789",
"k8s.statefulset.name": "my-database",
"k8s.statefulset.uid": "12345678-90ab-cdef-1234-567890abcdef",
"k8s.job.name": "my-batch-job",
"k8s.job.uid": "abcdef01-2345-6789-abcd-ef0123456789",
"k8s.cronjob.name": "my-cron-job",
"k8s.cronjob.uid": "12345678-90ab-cdef-1234-567890abcdef",
"k8s.node.uid": "fedcba98-7654-3210-dcba-fedcba987654",
"k8s.node.labels": {"node-role.kubernetes.io/worker": ""},
"k8s.node.annotations": {"kubelet.kubernetes.io/cpu-manager-policy": "none"},
"k8s.node.internal_ip": "192.168.1.100",
"k8s.node.external_ip": "1.2.3.4",
"k8s.node.os_image": "Ubuntu 20.04.3 LTS",
"k8s.node.kernel_version": "5.4.0-100-generic",
"k8s.node.container_runtime_version": "docker://20.10.7",
"k8s.node.kubelet_version": "v1.20.5",
"k8s.node.kube_proxy_version": "v1.20.5",
"k8s.node.architecture": "amd64",
"k8s.node.operating_system": "linux",
"k8s.node.hw_model": "Virtual Machine",
"k8s.node.num_cores": 4,
"k8s.node.memory_capacity": "8Gi",
"k8s.node.disk_total": "100Gi",
"k8s.node.disk_available": "50Gi",
"k8s.node.network_type": "cni",
"k8s.node.network_plugin": "calico",
"k8s.node.network_plugin_version": "v3.17.2",
"k8s.node.cloud_provider": "aws",
"k8s.node.region": "us-east-1",
"k8s.node.zone": "us-east-1a",
"k8s.node.instance_type": "t3.medium",
"k8s.node.instance_id": "i-0123456789abcdef0",
"k8s.node.ami_id": "ami-0abcdef1234567890",
"k8s.node.vpc_id": "vpc-0123456789abcdef0",
"k8s.node.subnet_id": "subnet-0123456789abcdef0",
"k8s.node.security_groups": ["sg-0123456789abcdef0"],
"k8s.node.private_ip_address": "192.168.1.100",
"k8s.node.public_ip_address": "1.2.3.4",
"k8s.node.tags": {"KubernetesCluster": "my-k8s-cluster", "Name": "worker-node-1"},
"k8s.node.conditions": [
{"type": "Ready", "status": "True"},
{"type": "MemoryPressure", "status": "False"},
{"type": "DiskPressure", "status": "False"},
{"type": "PIDPressure", "status": "False"},
{"type": "NetworkUnavailable", "status": "False"}
],
"k8s.node.allocatable.cpu": "3900m",
"k8s.node.allocatable.memory": "7872092Ki",
"k8s.node.allocatable.ephemeral_storage": "84060880Ki",
"k8s.node.allocatable.pods": 110,
"k8s.node.capacity.cpu": "4000m",
"k8s.node.capacity.memory": "8000000Ki",
"k8s.node.capacity.ephemeral_storage": "90000000Ki",
"k8s.node.capacity.pods": 110,
"k8s.node.non_terminated_pods": 50,
"k8s.node.creation_timestamp": "2023-10-26T15:00:00Z",
"k8s.node.uid": "fedcba98-7654-3210-dcba-fedcba987654",
"rule.id": "1",
"rule.tags": ["container", "filesystem", "mitre_execution"],
"rule.level": "error",
"rule.description": "Detects writes to sensitive system files.",
"rule.enabled": true,
"rule.source": "local",
"rule.time": "10m",
"rule.count": 1,
"rule.priority": 100,
"rule.syscall": "openat",
"rule.condition": "openat.filename.startswith('/etc/passwd') or openat.filename.startswith('/etc/shadow')",
"rule.output": "Write to sensitive file detected: {{.fd.name}} by {{.user.name}}",
"rule.threat:mitre:tactic": "Execution",
"rule.threat:mitre:technique": "T1059",
"rule.threat:mitre:technique_name": "Command and Scripting Interpreter",
"rule.threat:mitre:version": "v9",
"proc.cwd": "/",
"proc.exe": "/bin/bash",
"proc.cmdline": "/bin/bash -c 'echo hello > /root/test.txt'",
"proc.pid": 5678,
"proc.ppid": 1234,
"proc.uid": 0,
"proc.gid": 0,
"user.name": "root",
"container.id": "a1b2c3d4e5f6",
"container.name": "my-app-container",
"k8s.pod.name": "my-app-pod-xyz",
"k8s.namespace": "default",
"evt.type": "openat",
"evt.code": "openat",
"file.name": "test.txt",
"file.path": "/root/test.txt",
"file.dir": "/root",
"file.basename": "test.txt",
"file.extension": "",
"file.mode": "0644",
"file.owner": "root",
"file.group": "root",
"rule.name": "Write to sensitive file",
"rule.id": "1",
"rule.tags": ["container", "filesystem", "mitre_execution"],
"rule.level": "error",
"rule.description": "Detects writes to sensitive system files.",
"rule.enabled": true,
"rule.source": "local",
"rule.output": "Write to sensitive file detected: {{.fd.name}} by {{.user.name}}",
"rule.threat:mitre:tactic": "Execution",
"rule.threat:mitre:technique": "T1059",
"rule.threat:mitre:technique_name": "Command and Scripting Interpreter"
},
"output": "2023-10-27T10:00:00.123Z: Error: Write to sensitive file detected: /root/test.txt by root (container_id=a1b2c3d4e5f6, container_name=my-app-container, k8s_pod_name=my-app-pod-xyz, k8s_namespace=default, proc.pid=5678, proc.cmdline=/bin/bash -c 'echo hello > /root/test.txt')",
"priority": 2,
"rule": "Write to sensitive file",
"time": "10m",
"ver": "4.0.0",
"nodename": "falco-node-1",
"output_fields": {
"container.id": "a1b2c3d4e5f6",
"container.name": "my-app-container",
"evt.type": "openat",
"evt.code": "openat",
"file.name": "test.txt",
"file.path": "/root/test.txt",
"file.dir": "/root",
"file.basename": "test.txt",
"file.extension": "",
"file.mode": "0644",
"file.owner": "root",
"file.group": "root",
"proc.pid": 5678,
"proc.ppid": 1234,
"proc.uid": 0,
"proc.gid": 0,
"proc.exe": "/bin/bash",
"proc.cmdline": "/bin/bash -c 'echo hello > /root/test.txt'",
"proc.cwd": "/",
"user.name": "root",
"rule.name": "Write to sensitive file",
"rule.id": "1",
"rule.tags": ["container", "filesystem", "mitre_execution"],
"rule.level": "error",
"rule.description": "Detects writes to sensitive system files.",
"rule.enabled": true,
"rule.source": "local",
"rule.output": "Write to sensitive file detected: {{.fd.name}} by {{.user.name}}",
"rule.threat:mitre:tactic": "Execution",
"rule.threat:mitre:technique": "T1059",
"rule.threat:mitre:technique_name": "Command and Scripting Interpreter",
"k8s.pod.name": "my-app-pod-xyz",
"k8s.namespace": "default",
"k8s.node.name": "falco-node-1",
"container.image": "ubuntu:latest",
"container.runtime": "docker"
}
}
This data shows a Write to sensitive file event detected by Falco. The event details include the process (/bin/bash -c 'echo hello > /root/test.txt'), the user (root), the file being written to (/root/test.txt), and Kubernetes context (k8s_pod_name=my-app-pod-xyz, k8s_namespace=default).
Falco’s strength lies in its ability to generate rich, contextualized security events. When integrated with Prometheus and Grafana, these events transform into actionable security intelligence. Prometheus scrapes metrics exposed by Falco (typically via an HTTP endpoint), and Grafana visualizes these metrics, allowing you to build dashboards that provide real-time visibility into your system’s security posture.
To set this up, you’ll need to:
- Configure Falco to expose metrics: Edit your
falco.yamlto enable the gRPC server and expose metrics. - Configure Prometheus to scrape Falco: Add a scrape target for Falco in your
prometheus.yml. - Import a Falco dashboard into Grafana: Find or create a Grafana dashboard that can ingest and visualize Falco metrics.
Let’s dive into the configuration.
1. Configuring Falco for Metrics
Falco can expose its metrics via its gRPC output. You need to enable this in falco.yaml.
Find the grpc section and ensure it’s configured like this:
grpc:
enabled: true
listen_address: 0.0.0.0:8080 # Or your desired address and port
This tells Falco to start a gRPC server on 0.0.0.0:8080. Prometheus will connect to this address to scrape the metrics. The metrics exposed will include counts of rules triggered, event types, severities, and more.
2. Configuring Prometheus to Scrape Falco
In your Prometheus configuration (prometheus.yml), you need to add a scrape job for Falco. If you’re running Falco as a DaemonSet in Kubernetes, you can use Kubernetes service discovery. If running standalone, you’ll add a static configuration.
For Kubernetes (using kubernetes_sd_configs):
scrape_configs:
- job_name: 'falco'
kubernetes_sd_configs:
- role: pod
relabel_configs:
# Relabel to target Falco pods based on app label
- source_labels: [__meta_kubernetes_pod_label_app]
action: keep
regex: falco
# Relabel to scrape the metrics port (e.g., 8080)
- source_labels: [__meta_kubernetes_pod_container_port_number]
action: keep
regex: 8080
# Set the metric path
- source_labels: [__meta_kubernetes_pod_name]
target_label: instance
- target_label: __metrics_path__
replacement: /metrics # Falco exposes metrics at /metrics endpoint on its gRPC server
For Standalone (using static_configs):
scrape_configs:
- job_name: 'falco_standalone'
static_configs:
- targets: ['<falco-host-ip>:8080'] # Replace with your Falco host IP and port
Important: Falco’s gRPC server, when enabled, exposes metrics on a /metrics endpoint. Prometheus needs to be configured to hit this specific path. Ensure your prometheus.yml includes this.
3. Importing a Grafana Dashboard
Once Prometheus is scraping Falco, you can import a Grafana dashboard. Many community dashboards are available on Grafana.com. Search for "Falco" and find a dashboard that suits your needs. A popular one is the "Falco Metrics" dashboard.
When importing:
- Data Source: Select your Prometheus data source.
- Dashboard ID: Enter the ID of the dashboard you found on Grafana.com.
This dashboard will typically show panels for:
- Total Falco events.
- Events by rule name.
- Events by severity (Error, Warning, Info).
- Events per container or Kubernetes pod.
- Syscall counts.
The most surprising true thing about Falco’s metrics is that they are not just raw event counts. Falco exposes a rich set of Prometheus-compatible metrics that include detailed labeling, such as rule_name, severity, container_id, k8s_pod_name, and k8s_namespace. This allows for highly granular querying and visualization in Grafana. For instance, you can not only see the total number of "Write to sensitive file" alerts but also filter them down to a specific pod in a particular namespace, running on a specific node, all directly within your Grafana query.
The next step in enhancing your Falco observability is to integrate its audit logs with a SIEM for long-term storage and advanced threat hunting.