Cluster "production"
├── Node 1 (master, data)
├── Node 2 (data)
├── Node 3 (data)
└── Node 4 (ingest)

Index "logs-2024.01"
├── Shard 0 (Primary)
│   └── Replica 0
├── Shard 1 (Primary)
│   └── Replica 0
└── Shard 2 (Primary)
    └── Replica 0

{
  "_id": "1",
  "_index": "logs-2024.01",
  "_type": "_doc",
  "@timestamp": "2024-01-31T10:30:00Z",
  "message": "Application error occurred",
  "level": "ERROR",
  "service": "api-server"
}

Document → Ingest Pipeline → Analyzer → Lucene Index → Shards
         ↓              ↓           ↓          ↓         ↓
     Enrichment   Processors  Tokenization   Indexing  Distribution
     (optional)              (stemming)      (scoring)  (replication)

ES 7.10 (stable) → OpenSearch 1.x (compatible) → OpenSearch 2.x (enhanced)

┌─────────┐  ┌─────────┐  ┌─────────┐
│ Master  │  │ Master  │  │ Master  │
│ Data    │  │ Data    │  │ Data    │
│ Ingest  │  │ Ingest  │  │ Ingest  │
└─────────┘  └─────────┘  └─────────┘

┌──────────┐  ┌──────────┐  ┌──────────┐     ┌──────────┐  ┌──────────┐
│ Master   │  │ Master   │  │ Master   │     │ Ingest   │  │ Ingest   │
│ Voting   │  │ Voting   │  │ Voting   │     │ Pipeline │  │ Pipeline │
└──────────┘  └──────────┘  └──────────┘     └──────────┘  └──────────┘
       ↓             ↓             ↓                ↓             ↓
┌──────────┐  ┌──────────┐  ┌──────────┐     ┌──────────┐  ┌──────────┐
│ Data     │  │ Data     │  │ Data     │     │ Coordinate   │ Coordinate   │
│ Warm     │  │ Warm     │  │ Warm     │     │ Node     │  │ Node     │
└──────────┘  └──────────┘  └──────────┘     └──────────┘  └──────────┘

Zone 1            Zone 2            Zone 3
┌────────┐        ┌────────┐        ┌────────┐
│Master  │        │Master  │        │Master  │
│Data    │        │Data    │        │Data    │
│Hot     │        │Warm    │        │Cold    │
└────────┘        └────────┘        └────────┘
     ↓                  ↓                ↓
  (Recent)         (Historical)      (Archive)

Total Shards = (Data Size / Target Shard Size) × Replication Factor

Example:
- Data size: 1TB
- Target shard size: 50GB (optimal: 20-50GB)
- Replication factor: 2 (1 primary + 1 replica)
- Primary shards: 1000GB / 50GB = 20
- Total shards: 20 × 2 = 40 shards

{
  "policy": "logs_policy",
  "phases": {
    "hot": {
      "min_age": "0d",
      "actions": {
        "rollover": {
          "max_primary_shard_size": "50GB",
          "max_age": "1d"
        }
      }
    }
  }
}

{
  "warm": {
    "min_age": "7d",
    "actions": {
      "set_priority": {
        "priority": 25
      },
      "forcemerge": {
        "max_num_segments": 1
      }
    }
  }
}

{
  "cold": {
    "min_age": "30d",
    "actions": {
      "searchable_snapshot": {}
    }
  }
}

{
  "delete": {
    "min_age": "90d",
    "actions": {
      "delete": {}
    }
  }
}

{
  "mappings": {
    "properties": {
      "@timestamp": {
        "type": "date",
        "format": "epoch_millis"
      },
      "message": {
        "type": "text",
        "analyzer": "standard"
      },
      "level": {
        "type": "keyword"
      },
      "service": {
        "type": "keyword"
      },
      "response_time_ms": {
        "type": "integer"
      },
      "tags": {
        "type": "keyword"
      }
    }
  }
}

curl -X POST "localhost:9200/_bulk" -H 'Content-Type: application/json' -d'
{ "index" : { "_index" : "logs", "_id" : "1" } }
{ "timestamp": "2024-01-31T10:30:00Z", "level": "INFO", "message": "Started" }
{ "index" : { "_index" : "logs", "_id" : "2" } }
{ "timestamp": "2024-01-31T10:30:01Z", "level": "ERROR", "message": "Failed" }
'

Kafka → Logstash → Elasticsearch/OpenSearch

{
  "query": {
    "match": {
      "message": "application error"
    }
  }
}

{
  "query": {
    "term": {
      "level": "ERROR"
    }
  }
}

{
  "query": {
    "bool": {
      "must": [
        { "match": { "message": "error" } }
      ],
      "filter": [
        { "term": { "level": "ERROR" } },
        { "range": { "@timestamp": { "gte": "now-1h" } } }
      ],
      "should": [
        { "term": { "service": "api-server" } }
      ],
      "minimum_should_match": 1
    }
  }
}

{
  "query": {
    "range": {
      "response_time_ms": {
        "gte": 100,
        "lte": 1000
      }
    }
  }
}

{
  "aggs": {
    "errors_by_service": {
      "terms": {
        "field": "service",
        "size": 10
      }
    }
  }
}

{
  "aggs": {
    "errors_over_time": {
      "date_histogram": {
        "field": "@timestamp",
        "calendar_interval": "1h"
      }
    }
  }
}

{
  "aggs": {
    "avg_response_time": {
      "avg": {
        "field": "response_time_ms"
      }
    },
    "p99_response_time": {
      "percentiles": {
        "field": "response_time_ms",
        "percents": [99]
      }
    }
  }
}

# Set Xmx and Xms to same value (avoid heap resizing)
# Never exceed 50% of available RAM
# Typical: 16GB-32GB for data nodes
-Xmx31g
-Xms31g

thread_pool:
  search:
    size: <auto>  # 3 × num_cores
    queue_size: 1000
  index:
    size: <auto>
    queue_size: 200
  bulk:
    size: <auto>
    queue_size: 300

# Increase TCP buffer sizes
net.core.rmem_max: 134217728
net.core.wmem_max: 134217728
net.ipv4.tcp_rmem: 67108864 134217728 268435456
net.ipv4.tcp_wmem: 67108864 134217728 268435456

{
  "settings": {
    "index.refresh_interval": "30s"  # Default 1s
  }
}

{
  "settings": {
    "index.merge.policy.segments_per_tier": 10
  }
}

index.codec: best_compression  # Default: default

discovery.seed_hosts:
  - es-node-1:9300
  - es-node-2:9300
  - es-node-3:9300

cluster.initial_master_nodes:
  - es-node-1
  - es-node-2
  - es-node-3

node.attr.zone: us-east-1a
cluster.routing.allocation.awareness.attributes: zone

{
  "index.routing.allocation.require._name": "data-node-*"
}

# Check cluster health
curl -s localhost:9200/_cluster/health | jq .

# Response:
{
  "cluster_name": "elasticsearch",
  "status": "green",  # green = healthy, yellow = missing replicas, red = missing primary
  "timed_out": false,
  "number_of_nodes": 5,
  "number_of_data_nodes": 3,
  "active_primary_shards": 100,
  "active_shards": 200,
  "relocating_shards": 0,
  "initializing_shards": 0,
  "unassigned_shards": 0,
  "delayed_unassigned_shards": 0,
  "number_of_pending_tasks": 0,
  "number_of_in_flight_fetch": 0,
  "task_max_waiting_in_queue_millis": 0,
  "active_shards_percent_as_number": 100.0
}

Node 1: Shard-0P, Shard-1P, Shard-2R
Node 2: Shard-0R, Shard-1R, Shard-2P
Node 3: Shard-1P, Shard-2P, Shard-0P (if 3 replicas)

{
  "type": "s3",
  "settings": {
    "bucket": "my-elasticsearch-backups",
    "region": "us-east-1",
    "base_path": "backups"
  }
}

curl -X PUT "localhost:9200/_snapshot/my-repo/snapshot-2024-01-31"

# List snapshots
curl "localhost:9200/_snapshot/my-repo/_all"

# Restore snapshot
curl -X POST "localhost:9200/_snapshot/my-repo/snapshot-2024-01-31/_restore"

cluster.remote:
  leader:
    seeds:
      - remote-es-node:9300
    skip_unavailable: false

opendistro_security:
  authcz:
    admin_dn:
      - "cn=admin,dc=example,dc=com"
  auth_domain:
    ldap:
      description: "LDAP authenticator"
      http_authenticator:
        type: basic
        challenge: false
      authentication_backend:
        type: ldap
        config:
          ldap_host: ldap.example.com
          ldap_port: 389
          ldap_bind_dn: "cn=admin,dc=example,dc=com"
          ldap_bind_password: "password"
          ldap_basedn: "dc=example,dc=com"
          ldap_userbase: "ou=users,dc=example,dc=com"
          ldap_usersearch: "(uid={0})"

xpack.security.transport.ssl.enabled: true
xpack.security.transport.ssl.verification_mode: certificate
xpack.security.transport.ssl.keystore.path: "certs/elastic-stack-ca.p12"
xpack.security.transport.ssl.keystore.password: "password"

xpack.security.http.ssl.enabled: true
xpack.security.http.ssl.keystore.path: "certs/elastic-stack-ca.p12"
xpack.security.http.ssl.keystore.password: "password"

xpack.security.audit.enabled: true
xpack.security.audit.outputs: [logfile]
xpack.security.audit.logfile.events.include:
  - access_denied
  - access_granted
  - authentication_failed
  - authentication_success
  - privilege_check_failure
  - privilege_check_success

GET /_cluster/stats
GET /_nodes/stats
GET /_cat/indices
GET /_cat/nodes

{
  "trigger": {
    "schedule": {
      "interval": "5m"
    }
  },
  "input": {
    "search": {
      "indices": ["logs"],
      "body": {
        "query": {
          "bool": {
            "filter": {
              "term": {
                "level": "ERROR"
              }
            }
          }
        }
      }
    }
  },
  "condition": {
    "script": {
      "source": "ctx.payload.hits.total > 10"
    }
  },
  "actions": {
    "send_email": {
      "email": {
        "to": "ops@example.com",
        "subject": "High error rate detected"
      }
    }
  }
}

Applications
     ↓
  Logs (JSON)
     ↓
  Filebeat
     ↓
Elasticsearch
     ↓
  Kibana (visualization)

Servers/K8s
     ↓
System Metrics
     ↓
Metricbeat
     ↓
Elasticsearch (time-series data)
     ↓
Dashboards & Alerting

Firewalls, IDS, WAF, Endpoints
          ↓
      Raw Events
          ↓
      Logstash (parse, enrich, correlate)
          ↓
   Elasticsearch/OpenSearch
          ↓
   Threat Detection & Response

{
  "query": {
    "multi_match": {
      "query": "wireless headphones",
      "fields": ["title^2", "description", "tags"]
    }
  }
}

# Check node status
curl -s localhost:9200/_nodes | jq '.nodes'

# Check shard allocation
curl -s localhost:9200/_cat/shards

# Check index settings
curl -s localhost:9200/logs/_settings

# Check node info
curl -s localhost:9200/_nodes/stats/jvm | jq '.nodes[] | {name, heap_percent}'

# Enable debug logging
curl -X PUT "localhost:9200/_cluster/settings" -d '{
  "transient": {
    "logger.org.elasticsearch": "DEBUG"
  }
}'

Central Cluster (Analytics)
        ↑
        ├── Regional Cluster 1 → Cross-Cluster Search
        ├── Regional Cluster 2 → Cross-Cluster Search
        └── Regional Cluster 3 → Cross-Cluster Search

Cluster A ←→ Cross-Cluster Replication ←→ Cluster B
   (DR)           (Bidirectional)          (DR)

Day 1-7:   Hot nodes (SSD, high performance)
Day 8-30:  Warm nodes (HDD, slower, lower cost)
Day 31+:   Cold nodes (Archive, very slow, cheapest)