🔎 Elasticsearch 1.x 2.x 5.x 6.x 7.x 8.x Cheatsheet 🔍

Links

First thing, forget about your curl calls and install Sense please!

First thing, forget about your curl calls and install Kibana please!

• Elasticsearch: The Definitive Guide, official book;
• Elasticsearch: The Definitive Guide, official book;
• Elasticsearch: The Definitive Guide, official book (not up to date);
• Download Elasticsearch, official download page;
• Download Elasticsearch, official download page;
• Download pure OSS Elasticsearch (<= 7.10.2 only), Apache licensed download page;
• Elasticsearch Reference, official documentation;
• Elasticsearch Reference, official documentation;
• Elasticsearch Reference, official documentation;
• Elasticsearch Reference, official documentation;
• Elasticsearch Reference, official documentation;
• Elasticsearch Reference, official documentation;
• Docker repository, quick start any Elastic software;
• Dev.to Elasticsearch, great source of content about Elastic;
• Official forum and StackOverflow for support;
• Official online migration tool to help upgrading the stack to 6.x;
• Comrade: a Python based Elasticsearch web admin and monitoring panel;
• Cerebro: a Java based Elasticsearch web admin;

Queries

There are two syntaxes for the basic queries: a simple one on the left, where you can't use any option, and an extended one on the right. Most of the beginner headache with the DSL come from this:

GET _search
{
  "query": {
    "match": {
      "FIELD": "TEXT"
    }
  }
}

GET _search
{
  "query": {
    "match": {
      "FIELD": {
        "query": "TEXT",
        "OPTION": "VALUE"
      }
    }
  }
}

Full search example with aggregation, highlight, filter...

GET /_search
{
  "query": {
    "filtered": {
      "query": {
        "bool": {
          "must": [
            {
              "match": {
                "title": "smith"
              }
            }
          ],
          "must_not": [
            {
              "match_phrase": {
                "title": "granny smith"
              }
            }
          ]
        }
      },
      "filter": {
        "bool": {
          "must_not": [
            {
              "missing": {
                "field": "title"
              }
            }
          ]
        }
      }
    }
  },
  "aggs": {
    "my_agg": {
      "terms": {
        "field": "user",
        "size": 10
      }
    }
  },
  "highlight": {
    "pre_tags" : ["<em>"],
    "post_tags" : ["</em>"],
    "fields": {
      "body" : {
        "number_of_fragments": 1,
        "fragment_size": 20
      },
      "title": {}
    }
  },
  "size": 20,
  "from": 100,
  "_source": ["title", "id"],
  "sort": [
    { "_id" : {"order" : "desc"}}
  ]
}

GET /_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "title": "smith"
          }
        }
      ],
      "must_not": [
        {
          "match_phrase": {
            "title": "granny smith"
          }
        }
      ],
      "filter": [
        {
          "exists": {
            "field": "title"
          }
        }
      ]
    }
  },
  "aggs": {
    "my_agg": {
      "terms": {
        "field": "user",
        "size": 10
      }
    }
  },
  "highlight": {
    "pre_tags": [
      "<em>"
    ],
    "post_tags": [
      "</em>"
    ],
    "fields": {
      "body": {
        "number_of_fragments": 1,
        "fragment_size": 20
      },
      "title": {}
    }
  },
  "size": 20,
  "from": 100,
  "_source": [
    "title",
    "id"
  ],
  "sort": [
    {
      "_id": {
        "order": "desc"
      }
    }
  ]
}

Control total hit count

Accept true, false or a fixed number, default to 10000.

GET /_search
{
  "track_total_hits": true,
  "query": {}
}

Common queries

"multi_match": {
  "query": "Elastic",
  "fields": ["user.*", "title^3"],
  "type": "best_fields"
}

"bool": {
  "must": [],
  "must_not": [],
  "filter": [],
  "should": [],
  "minimum_should_match" : 1
}

"bool": {
  "must": [],
  "must_not": [],
  "should": [],
  "minimum_should_match" : 1
}

"filtered": {
  "query": {
    "match_all": {}
  },
  "filter": {
    "range": {...}
  }
}

"range": {
  "age": {
    "gte": 10,
    "lte": 20,
    "boost": 2
  }
}

QueryString syntax

Search in the default _all field:

GET /_search?q=pony

Complex search with operator and exact phrase search with boost:

GET /_search?q=title:(joli OR code) AND author:"Damien Alexandre"^2

Search with wildcard and special queries:

GET /_search?q=_exists_:title OR title:singl? noneOrAnyChar*cter

Search with fuzzyness and range:

GET /_search?q=title:elastichurch~3 AND date:[2016-01-01 TO 2018-12-31]

Use in Query DSL (not recommended for user search):

GET /_search
{
  "query": {
    "query_string": {
      "default_field": "content",
      "query": "elastic AND (title:lucene OR title:solr)"
    }
  }
}

Search After - Pagination cursor

Search with a custom sort:

GET products/_search
{
    "size": 10,
    "sort": [
        {"date": "asc"},
        {"_id": "desc"}
    ]
}

On the next "page", pass the sort values from the last result:

GET product/_search
{
    "size": 10,
    "search_after": [1463538857, "654323"],
    "sort": [
        {"date": "asc"},
        {"_id": "desc"}
    ]
}

Indexes and mapping

Create an index with settings and mapping

PUT /my_index_name
{
  "settings": {
    "number_of_replicas": 1,
    "number_of_shards": 3,
    "analysis": {},
    "refresh_interval": "1s"
  },
  "mappings": {
    "my_type_name": {
      "dynamic": false,
      "properties": {
        "title": {
          "type": "string",
          "analyzer": "english"
        }
      }
    }
  }
}

PUT /my_index_name
{
  "settings": {
    "number_of_replicas": 1,
    "number_of_shards": 3,
    "analysis": {},
    "refresh_interval": "1s"
  },
  "mappings": {
    "my_type_name": {
      "dynamic": false,
      "properties": {
        "title": {
          "type": "text",
          "analyzer": "english"
        }
      }
    }
  }
}

PUT /my_index_name
{
  "settings": {
    "number_of_replicas": 1,
    "number_of_shards": 3,
    "analysis": {},
    "refresh_interval": "1s"
  },
  "mappings": {
    "dynamic": false,
    "properties": {
      "title": {
        "type": "text",
        "analyzer": "english"
      }
    }
  }
}

Update index settings dynamically

PUT /my_index_name/_settings
{
  "index": {
    "refresh_interval": "-1",
    "number_of_replicas": 0
  }
}

Update an index by adding a field to a type

PUT /my_index_name/_mapping/my_type_name
{
  "my_type_name": {
    "properties": {
      "tag": {
        "type": "string",
        "index": "not_analyzed"
      }
    }
  }
}

PUT /my_index_name/_mapping/my_type_name
{
  "my_type_name": {
    "properties": {
      "tag": {
        "type": "keyword"
      }
    }
  }
}

PUT /my_index_name/_mapping
{
  "properties": {
    "tag": {
      "type": "keyword"
    }
  }
}

Get the mapping and the settings

GET /my_index_name

GET /my_index_name/_mapping

GET /my_index_name/_settings

Create a document (auto-generated ID)

POST /my_index_name/my_type_name
{
  "title": "Elastic is funny",
  "tag": [
    "lucene"
  ]
}

POST /my_index_name/_doc
{
  "title": "Elastic is funny",
  "tag": [
    "lucene"
  ]
}

Get a document

GET /my_index_name/my_type_name/12abc

GET /my_index_name/_doc/12abc

Create or update a document

PUT /my_index_name/my_type_name/12abc
{
  "title": "Elastic is funny",
  "tag": [
    "lucene"
  ]
}

PUT /my_index_name/_doc/12abc
{
  "title": "Elastic is funny",
  "tag": [
    "lucene"
  ]
}

Delete a document

DELETE /my_index_name/my_type_name/12abc

DELETE /my_index_name/_doc/12abc

Open and close indexes to save memory and CPU

POST /my_index_name/_close

POST /my_index_name/_open

Remove and create aliases

POST /_aliases
{
  "actions": [
    {
      "remove": {
        "index": "my_index_name",
        "alias": "foo"
      }
    },
    {
      "add": {
        "index": "my_index_name",
        "alias": "bar",
        "filter" : { "term" : { "user" : "damien" } }
      }
    }
  ]
}

List aliases

GET /_aliases

GET /my_index_name/_alias/*

GET /*/_alias/*

GET /*/_alias/foo

Full custom analyzer declaration

PUT /english_example
{
  "settings": {
    "analysis": {
      "filter": {
        "english_stop": {
          "type":       "stop",
          "stopwords":  "_english_"
        },
        "english_stemmer": {
          "type":       "stemmer",
          "language":   "english"
        }
      },
      "analyzer": {
        "my_english": {
          "char_filter":  ["html_strip"],
          "tokenizer":  "standard",
          "filter": [
            "lowercase",
            "english_stop",
            "english_stemmer"
          ]
        }
      }
    }
  }
}

Create and list index warmer

Create a query that will be played on every new document:

PUT /my_index_name/_warmer/my_warmer
{
  "query": {
    "match_all": {}
  },
  "aggs": {
    "costly_aggs": {
      "terms": {
        "field": "title"
      }
    }
  }
}

GET /my_index_name/_warmer

GET /my_index_name/_warmer/*

Indices monitoring and information

GET /my_index_name/_stats

GET /my_index_name/_segments

GET /my_index_name/_recovery?pretty&human

GET /my_index_name/_warmer/*

Indices status and management

POST /my_index_name/_cache/clear

POST /my_index_name/_refresh

POST /my_index_name/_flush

POST /my_index_name/_optimize

POST /my_index_name/_forcemerge

POST /_reindex
{
  "source": {
    "index": "test-index"
  },
  "dest": {
    "index": "test-index-new"
  }
}

POST /_reindex
{
  "source": {
    "index": "test-index",
    "query": {
      "match": {
        "gender": "female"
      }
    }
  },
  "dest": {
    "index": "test-index-new",
    "type": "female"
  }
}

Debug and development

Queries

Get a detailed view of what a query do:

GET /blog/_validate/query?explain=true
{
  "query": {
    "match": {
      "title": "Smith"
    }
  }
}

Get an explanation about a document matching or not:

GET /blog/post/1/_explain
{
  "query": {
    "match": {
      "title": "Smith"
    }
  }
}

GET /blog/_explain/1
{
  "query": {
    "match": {
      "title": "Smith"
    }
  }
}

Analysis

Test how a content is tokenized in a field:

GET /blog/_analyze
{
  "field": "title",
  "text": "powerful"
}

Test analyzer token output by analyzer:

GET /blog/_analyze
{
  "analyzer": "english",
  "text": "powerful"
}

Slowlog

Lower the slowlog threshold to see all the search queries in the logs:

PUT /blog/_settings
{
  "index.search.slowlog.threshold.query.trace": "0s",
  "index.search.slowlog.level": "trace"
}

Go back to the default configuration:

PUT /blog/_settings
{
  "index.search.slowlog.threshold.query.trace": "500ms",
  "index.search.slowlog.level": "info"
}

Cluster management and plugins

docker network create elastic
docker run --rm -it --name "elastically_es" -e "discovery.type=single-node" --net elastic docker.elastic.co/elasticsearch/elasticsearch:8.1.2

docker run --rm -it --name "elastically_es" -p 9200:9200 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.17.2

docker run -d --name "elastically_es" -p 9200:9200 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:6.8.6

docker run -d --name "elastically_es" -p 9200:9200 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:5.6.16

docker run --rm -it --name "elastically_kibana" --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:8.1.2

docker run -e "ELASTICSEARCH_HOSTS=http://127.0.0.1:9200/" --network host docker.elastic.co/kibana/kibana:7.17.2

docker run -e "ELASTICSEARCH_HOSTS=http://127.0.0.1:9200/" --network host docker.elastic.co/kibana/kibana:6.8.6

docker run -e "ELASTICSEARCH_HOSTS=http://127.0.0.1:9200/" --network host docker.elastic.co/kibana/kibana:5.6.16

Cluster and node information

GET /_cluster/health?pretty

GET /_cluster/health?wait_for_status=yellow&timeout=50s

GET /_cluster/state

GET /_cluster/stats?human&pretty

GET /_cluster/pending_tasks

GET /_nodes

GET /_nodes/stats

GET /_nodes/nodeId1,nodeId2/stats

GET /_shutdown 😱

Get the full reference of all the settings:

GET /_cluster/settings?include_defaults=true&flat_settings=true

Moving shards manually

Ask the index my_index_name shard 0 of node1 to go to node2:

POST /_cluster/reroute
{
  "commands": [
    {
      "move": {
        "index": "my_index_name",
        "shard": 0,
        "from_node": "node1",
        "to_node": "node2"
      }
    },
    {
      "allocate": {
        "index": "my_index_name",
        "shard": 1,
        "node": "node3"
      }
    }
  ]
}

Updating settings

PUT /_cluster/settings
{
  "persistent": {
    "discovery.zen.minimum_master_nodes": 3
  }
}

PUT /_cluster/settings
{
  "transient": {
    "discovery.zen.minimum_master_nodes": 2
  }
}

Disable shard allocation, useful before a rolling restart:

PUT /_cluster/settings
{
    "transient" : {
        "cluster.routing.allocation.enable" : "none"
    }
}

PUT /_cluster/settings
{
    "transient" : {
        "cluster.routing.allocation.enable" : "all"
    }
}

Snapshots and Restore

PUT /_snapshot/my_backup
{
  "type": "fs",
  "settings": {
    "location": "my_backup_location"
  }
}

PUT /_snapshot/my_backup/snapshot_a
{
  "indices": "index_1,index_2",
  "ignore_unavailable": "true",
  "include_global_state": false
}

POST /_snapshot/my_backup/snapshot_a/_restore
{
  "indices": "index_1,index_2",
  "ignore_unavailable": "true",
  "include_global_state": false,
  "rename_pattern": "index_(.+)",
  "rename_replacement": "restored_index_$1"
}

Most useful plugins

Plugins management

bin/plugin --url file:///path/to/plugin --install plugin-name

bin/plugin --remove [pluginname]

bin/plugin install file:///path/to/plugin

bin/plugin list

bin/plugin remove [pluginname]

bin/elasticsearch-plugin install file:///path/to/plugin

bin/elasticsearch-plugin list

bin/elasticsearch-plugin remove [pluginname]

Other information

Where to find the plugin binary?

RPM: /usr/share/elasticsearch/bin

Debian: /usr/share/elasticsearch/bin

What are the default ports?

Kibana: http://localhost:5601/.

Elasticsearch: http://localhost:9200/.

How to set the correct HEAP SIZE value?

The best value for a single purpose Elasticsearch server is about 50% of available RAM but under 32g.
Assuming Ubuntu / Debian server, you can change those files:

/etc/security/limits.conf

elasticsearch - nofile 65535
elasticsearch - memlock unlimited

/etc/default/elasticsearch (on CentOS/RH: /etc/sysconfig/elasticsearch)

ES_HEAP_SIZE=20g
MAX_OPEN_FILES=65535
MAX_LOCKED_MEMORY=unlimited

Useful settings to change in elasticsearch.yml

cluster.name: jolicluster
node.name: ${HOSTNAME}
plugin.mandatory: marvel-agent,analysis-icu,license
node.data: true
node.master: true
bootstrap.mlockall: true
action.auto_create_index: +aaa*,-bbb*,+ccc*,-*

cluster.name: jolicluster
node.name: ${HOSTNAME}
discovery.zen.ping.unicast.hosts: ["front01", "front02"]
discovery.zen.minimum_master_nodes: 2
network.host: _site_
network.bind_host: [_site_, _local_]
plugin.mandatory: analysis-icu, marvel-agent
node.data: true
node.master: true
bootstrap.mlockall: true
action.auto_create_index: +aaa*,-bbb*,+ccc*,-*

cluster.name: jolicluster
node.name: ${HOSTNAME}
discovery.zen.ping.unicast.hosts: ["front01", "front02"]
discovery.zen.minimum_master_nodes: 2
network.host: _site_
network.bind_host: [_site_, _local_]
plugin.mandatory: analysis-icu
node.data: true
node.master: true
bootstrap.memory_lock: true
action.auto_create_index: +aaa*,-bbb*,+ccc*,-*

# Disable X-Pack features
xpack.security.enabled: false
xpack.monitoring.enabled: false
xpack.ml.enabled: false
xpack.watcher.enabled: false
xpack.sql.enabled: false
xpack.graph.enabled: false

# Only for XPack plugin for ES <= 6.2
#xpack.reporting.enabled: false

cluster.name: jolicluster
node.name: ${HOSTNAME} # by default

network.host: [_local_, _site_]
plugin.mandatory: analysis-icu
node.data: true
node.master: true
node.ingest: true
bootstrap.memory_lock: true
action.auto_create_index: +aaa*,-bbb*,+ccc*,-*

discovery.seed_hosts:
  - 192.168.1.10:9300
  - 192.168.1.11
  - seeds.mydomain.com

# Needed for first cluster boot
cluster.initial_master_nodes:
  - 10.0.10.101
  - 10.0.10.102:9300
  - 10.0.10.102:9301
  - master-node-name

# Disable X-Pack features, choose wisely
xpack.ccr.enabled: false
xpack.data_frame.enabled: false
xpack.enrich.enabled: false
xpack.flattened.enabled: false
xpack.graph.enabled: false
xpack.ilm.enabled: false
xpack.logstash.enabled: false
xpack.ml.enabled: false
xpack.monitoring.enabled: false
xpack.rollup.enabled: false
xpack.security.enabled: false
xpack.slm.enabled: false
xpack.sql.enabled: false
xpack.transform.enabled: false
xpack.vectors.enabled: false
xpack.watcher.enabled: false