tools/mappings.py

#!/usr/bin/env python3

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# utility to check mappings, report differences and create missing mappings

#  ** INITIAL VERSION, liable to change **

import argparse
import sys
import yaml
from plugins.elastic import Elastic

# Needs 3.4 or higher to work
if sys.version_info <= (3, 3):
    print("This script requires Python 3.4 or higher in order to work!")
    sys.exit(-1)

# the desired mappings
mapping_file = yaml.safe_load(open("mappings.yaml", "r"))

elastic = Elastic()
major = elastic.engineMajor()
if major != 7:
    print("This script requires ElasticSearch 7 API in order to work!")
    sys.exit(-1)

parser = argparse.ArgumentParser(description="Command line options.")
parser.add_argument(
    "--create",
    dest="create",
    action="store_true",
    help="Create the missing mapping(s)",
)
parser.add_argument(
    "--shards",
    dest="shards",
    type=int,
    help="Create the missing indices",
)
parser.add_argument(
    "--replicas",
    dest="replicas",
    type=int,
    help="Create the missing indices",
)
parser.add_argument('names', nargs='*')
args = parser.parse_args()

def descend(hsh, mapping, parent):
  for k,v in mapping.items():
    keys=parent.copy()
    keys.append(k)
    if 'properties' in v:
      descend(hsh, v['properties'], keys)
    else:
      key = ".".join(keys)
      hsh[key] = v

def check_mapping(index):
  # expected mappings
  mappings_expected = mapping_file[index]['properties']

  index_name = elastic.index_name(index)

  # Check that index exists
  if not elastic.indices.exists(index_name):
    if args.shards and args.replicas is not None:
      print("Creating index")
      settings = {"number_of_shards": args.shards, "number_of_replicas": args.replicas}
      elastic.indices.create(
        index=index_name, body={"mappings": mapping_file[index], "settings": settings}
      )
      print("Created index!")
      return
    else:
      print("Index not found!")
      print("Specify --shards and --replicas to create the index")
      return

  # actual mappings
  mappings = elastic.indices.get_mapping(index=index_name)[index_name]['mappings']['properties']

  expected = yaml.dump(mappings_expected, sort_keys = True)
  actual = yaml.dump(mappings, sort_keys = True).replace("'true'","true")

  if actual == expected:
    print("Mappings are as expected, hoorah!")
  else:
    print("Mappings differ:")

    exp = dict()
    descend(exp, mappings_expected,[])
    act = dict()
    descend(act, mappings,[])

    for k,v in exp.items():
      if not k in act:
        if v == {'dynamic': True, 'type': 'object'}:
          print(f' Key {k} is dynamic')
        else:
          print(f"Expected {k} {v}; not found")
      else:
        if not v == act[k]:
          print(f"Expected {k} {v},  found {k} {act[k]}")
    

for type in args.names if len(args.names) > 0 else mapping_file.keys():
  print("Checking " + type)
  check_mapping(type)