setup/setup.py - kibble-1 - Git at Google

 #!/usr/bin/env python3
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
 # this work for additional information regarding copyright ownership.
 # The ASF licenses this file to You under the Apache License, Version 2.0
 # (the "License"); you may not use this file except in compliance with
 # the License.  You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.

 KIBBLE_VERSION = '0.1.0' # ABI/API compat demarcation.
 KIBBLE_DB_VERSION = 2 # Second database revision

 ELASTIC_SEARCH_MIN_VERSION = 8

 import sys

 if sys.version_info <= (3, 3):
     print("This script requires Python 3.4 or higher")
     sys.exit(-1)

 import os
 import getpass
 import subprocess
 import argparse
 import shutil
 import yaml
 import bcrypt
 import json

 mappings = json.load(open("mappings.json"))
 myyaml = yaml.load(open("kibble.yaml.sample"), Loader=yaml.Loader)

 dopip = False
 try:
     from elasticsearch import Elasticsearch
     from elasticsearch import VERSION as ES_VERSION
     ES_MAJOR = ES_VERSION[0]
 except:
     dopip = True

 if dopip and (getpass.getuser() != "root"):
     print("It looks like you need to install some python modules first")
     print("Either run this as root to do so, or run: ")
     print("pip3 install elasticsearch certifi bcrypt")
     sys.exit(-1)

 elif dopip:
     print("Before we get started, we need to install some modules")
     print("Hang on!")
     try:
         subprocess.check_call(('pip3','install','elasticsearch', 'certifi', 'bcrypt'))
         from elasticsearch import Elasticsearch
     except:
         print("Oh dear, looks like this failed :(")
         print("Please install elasticsearch and certifi before you try again:")
         print("pip install elasticsearch certifi")
         sys.exit(-1)


 # Arguments for non-interactive setups like docker
 arg_parser = argparse.ArgumentParser()
 arg_parser.add_argument("-e", "--hostname", help="Pre-defined hostname for ElasticSearch (docker setups)")
 arg_parser.add_argument("-p", "--port", help="Pre-defined port for ES (docker setups)")
 arg_parser.add_argument("-d", "--dbname", help="Pre-defined Database prefix (docker setups)")
 arg_parser.add_argument("-s", "--shards", help="Predefined number of ES shards (docker setups)")
 arg_parser.add_argument("-r", "--replicas", help="Predefined number of replicas for ES (docker setups)")
 arg_parser.add_argument("-m", "--mailhost", help="Pre-defined mail server host (docker setups)")
 arg_parser.add_argument("-a", "--autoadmin", action='store_true', help="Generate generic admin account (docker setups)")
 arg_parser.add_argument("-k", "--skiponexist", action='store_true', help="Skip DB creation if DBs exist (docker setups)")
 args = arg_parser.parse_args()

 print("Welcome to the Apache Kibble setup script!")
 print("Let's start by determining some settings...")
 print("")


 hostname = args.hostname or ""
 port = int(args.port) if args.port else 0
 dbname = args.dbname or ""
 mlserver = args.mailhost or ""
 mldom = ""
 wc = ""
 genname = ""
 wce = False
 shards = int(args.shards) if args.shards else 0
 replicas = int(args.replicas) if args.replicas else -1

 while hostname == "":
     hostname = input("What is the hostname of the ElasticSearch server? [localhost]: ")
     if hostname == "":
         print("Using default; localhost")
         hostname = "localhost"
 while port < 1:
     try:
         port = input("What port is ElasticSearch listening on? [9200]: ")
         if port == "":
             print("Using default; 9200")
             port = 9200
         port = int(port)
     except ValueError:
         pass

 while dbname == "":
     dbname = input("What would you like to call (lowercase) the DB index [kibble]: ")
     if dbname == "":
         print("Using default; kibble")
         dbname = "kibble"

 while mlserver == "":
     mlserver = input("What is the hostname of the outgoing mailserver? [localhost:25]: ")
     if mlserver == "":
         print("Using default; localhost:25")
         mlserver = "localhost:25"

 while shards < 1:
     try:
         shards = input("How many shards for the ElasticSearch index? [5]:")
         if shards == "":
             print("Using default; 5")
             shards = 5
         shards = int(shards)
     except ValueError:
         pass

 while replicas < 0:
     try:
         replicas = input("How many replicas for each shard? [1]: ")
         if replicas == "":
             print("Using default; 1")
             replicas = 1
         replicas = int(replicas)
     except ValueError:
         pass

 adminName = ""
 adminPass = ""
 if args.autoadmin:
     adminName = "admin@kibble"
     adminPass = "kibbleAdmin"
 while adminName == "":
     adminName = input("Enter an email address for the adminstrator account: ")
 while adminPass == "":
     adminPass = input("Enter a password for the adminstrator account: ")

 print("Okay, I got all I need, setting up Kibble...")

 def createIndex():
     global mappings
     global es

     defaultELConfig = {
         'host': hostname,
         'port': port
     }

     versionHint = ELASTIC_SEARCH_MIN_VERSION

     if versionHint >= 7:
         defaultELConfig['scheme'] = 'http'
         defaultELConfig['path_prefix'] = ''
     else:
         defaultELConfig['use_ssl'] = False
         defaultELConfig['url_prefix'] = ''

     es = Elasticsearch([ defaultELConfig],
         max_retries=5,
         retry_on_timeout=True
     )

     es6 = True if int(es.info()['version']['number'].split('.')[0]) >= 6 else False
     es7 = True if int(es.info()['version']['number'].split('.')[0]) >= 7 else False
     es8 = True if int(es.info()['version']['number'].split('.')[0]) >= 8 else False
     if not es6:
         print("New Kibble installations require ElasticSearch 6.x or newer! You appear to be running %s!" % es.info()['version']['number'])
         sys.exit(-1)
     # If ES >= 7, _doc is invalid and mapping should be rooted
     if es7:
         mappings['mappings'] = mappings['mappings']['_doc']
     # Check if index already exists
     if es.indices.exists(index=dbname+"_api"):
         if args.skiponexist: # Skip this is DB exists and -k added
             print("DB prefix exists, but --skiponexist used, skipping this step.")
             return
         print("Error: ElasticSearch DB prefix '%s' already exists!" % dbname)
         sys.exit(-1)

     types = [
         'api',
         # ci_*: CI service stats
         'ci_build',
         'ci_queue',
         # code_* + evolution + file_history: git repo stats
         'code_commit',
         'code_commit_unique',
         'code_modification',
         'evolution',
         'file_history',
         # forum_*: forum stats (SO, Discourse, Askbot etc)
         'forum_post',
         'forum_topic',
         'ghstats',
         # GitHub stats
         # im_*: Instant messaging stats
         'im_stats',
         'im_ops',
         'im_msg',
         'issue',
         'logstats',
         # email, mail*: Email statitics
         'email',
         'mailstats',
         'mailtop',
         # organisation, view, source, publish: UI Org DB
         'organisation',
         'view',
         'publish',
         'source',
         # stats: Miscellaneous stats
         'stats',
         # social_*: Twitter, Mastodon, Facebook etc
         'social_follow',
         'social_followers',
         'social_follower',
         'social_person',
         # uisession, useraccount, message: UI user DB
         'uisession',
         'useraccount',
         'message',
         # person: contributor DB
         'person',
     ]

     for t in types:
         iname = "%s_%s" % (dbname, t)
         print("Creating index " + iname)

         settings = {
             "number_of_shards" :   shards,
             "number_of_replicas" : replicas
         }


         res = es.indices.create(index = iname, body = {
                     "mappings" : mappings['mappings'],
                     "settings": settings
                 }
             )

     print("Indices created! %s " % res)


 def createAccount():
     global es

     salt = bcrypt.gensalt()
     pwd = bcrypt.hashpw(adminPass.encode('utf-8'), salt)   # .decode('ascii') ->  'str' object has no attribute 'decode'
     print("Creating administrator account")
     doc = {
             'email': adminName,                 # Username (email)
             'password': pwd,              # Hashed password
             'displayName': "Administrator",     # Display Name
             'organisations': [],                # Orgs user belongs to (default is none)
             'ownerships': [],                   # Orgs user owns (default is none)
             'defaultOrganisation': None,        # Default org for user
             #'defaultOrganisation':  'apache',        # Default org for user
             'verified': True,                   # Account verified via email?
             'userlevel': "admin"                # User level (user/admin)
         }
     dbdoc = {
         'apiversion': KIBBLE_VERSION,           # Log current API version
         'dbversion': KIBBLE_DB_VERSION          # Log the database revision we accept (might change!)
     }

     es7 = True if int(es.info()['version']['number'].split('.')[0]) >= 7 else False

     if es7:
         es.index(index=dbname+'_useraccount', id = adminName, body = doc)
         es.index(index=dbname+'_api', id = 'current', body = dbdoc)
     else:
         es.index(index=dbname+'_useraccount', doc_type = '_doc', id = adminName, body = doc)
         es.index(index=dbname+'_api', doc_type = '_doc', id = 'current', body = dbdoc)
     print("Account created!")


 try:
     import logging
     # elasticsearch logs lots of warnings on retries/connection failure
     logging.getLogger("elasticsearch").setLevel(logging.ERROR)
     createIndex()
     createAccount()


 except Exception as e:
     print("Index creation failed: %s" % e)
     sys.exit(1)

 kibble_yaml = '../api/yaml/kibble.yaml'

 if os.path.exists(kibble_yaml):
     print("%s already exists! Writing to %s.tmp instead" % (kibble_yaml, kibble_yaml))
     kibble_yaml = kibble_yaml + ".tmp"


 print("Writing Kibble config (%s)" % kibble_yaml)

 m = mlserver.split(':')
 if len(m) == 1:
     m.append(25)

 myconfig = {
     'api': {
         'version': KIBBLE_VERSION,
         'database': KIBBLE_DB_VERSION
     },
     'elasticsearch': {
         'host': hostname,
         'port': port,
         'ssl': False,
         'dbname': dbname,
         'versionHint': ELASTIC_SEARCH_MIN_VERSION
     },
     'mail': {
         'mailhost': m[0],
         'mailport': m[1],
         'sender': 'Kibble <noreply@kibble.kibble>'
     },
     'accounts': {
         'allowSignup': True,
         'verify': True
     }
 }

 with open(kibble_yaml, "w") as f:
     f.write(yaml.dump(myconfig, default_flow_style = False))
     f.close()


 print("All done, Kibble should...work now :)")
	#!/usr/bin/env python3
	# Licensed to the Apache Software Foundation (ASF) under one or more
	# contributor license agreements. See the NOTICE file distributed with
	# this work for additional information regarding copyright ownership.
	# The ASF licenses this file to You under the Apache License, Version 2.0
	# (the "License"); you may not use this file except in compliance with
	# the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	KIBBLE_VERSION = '0.1.0' # ABI/API compat demarcation.
	KIBBLE_DB_VERSION = 2 # Second database revision

	ELASTIC_SEARCH_MIN_VERSION = 8

	import sys

	if sys.version_info <= (3, 3):
	print("This script requires Python 3.4 or higher")
	sys.exit(-1)

	import os
	import getpass
	import subprocess
	import argparse
	import shutil
	import yaml
	import bcrypt
	import json

	mappings = json.load(open("mappings.json"))
	myyaml = yaml.load(open("kibble.yaml.sample"), Loader=yaml.Loader)

	dopip = False
	try:
	from elasticsearch import Elasticsearch
	from elasticsearch import VERSION as ES_VERSION
	ES_MAJOR = ES_VERSION[0]
	except:
	dopip = True

	if dopip and (getpass.getuser() != "root"):
	print("It looks like you need to install some python modules first")
	print("Either run this as root to do so, or run: ")
	print("pip3 install elasticsearch certifi bcrypt")
	sys.exit(-1)

	elif dopip:
	print("Before we get started, we need to install some modules")
	print("Hang on!")
	try:
	subprocess.check_call(('pip3','install','elasticsearch', 'certifi', 'bcrypt'))
	from elasticsearch import Elasticsearch
	except:
	print("Oh dear, looks like this failed :(")
	print("Please install elasticsearch and certifi before you try again:")
	print("pip install elasticsearch certifi")
	sys.exit(-1)


	# Arguments for non-interactive setups like docker
	arg_parser = argparse.ArgumentParser()
	arg_parser.add_argument("-e", "--hostname", help="Pre-defined hostname for ElasticSearch (docker setups)")
	arg_parser.add_argument("-p", "--port", help="Pre-defined port for ES (docker setups)")
	arg_parser.add_argument("-d", "--dbname", help="Pre-defined Database prefix (docker setups)")
	arg_parser.add_argument("-s", "--shards", help="Predefined number of ES shards (docker setups)")
	arg_parser.add_argument("-r", "--replicas", help="Predefined number of replicas for ES (docker setups)")
	arg_parser.add_argument("-m", "--mailhost", help="Pre-defined mail server host (docker setups)")
	arg_parser.add_argument("-a", "--autoadmin", action='store_true', help="Generate generic admin account (docker setups)")
	arg_parser.add_argument("-k", "--skiponexist", action='store_true', help="Skip DB creation if DBs exist (docker setups)")
	args = arg_parser.parse_args()

	print("Welcome to the Apache Kibble setup script!")
	print("Let's start by determining some settings...")
	print("")


	hostname = args.hostname or ""
	port = int(args.port) if args.port else 0
	dbname = args.dbname or ""
	mlserver = args.mailhost or ""
	mldom = ""
	wc = ""
	genname = ""
	wce = False
	shards = int(args.shards) if args.shards else 0
	replicas = int(args.replicas) if args.replicas else -1

	while hostname == "":
	hostname = input("What is the hostname of the ElasticSearch server? [localhost]: ")
	if hostname == "":
	print("Using default; localhost")
	hostname = "localhost"
	while port < 1:
	try:
	port = input("What port is ElasticSearch listening on? [9200]: ")
	if port == "":
	print("Using default; 9200")
	port = 9200
	port = int(port)
	except ValueError:
	pass

	while dbname == "":
	dbname = input("What would you like to call (lowercase) the DB index [kibble]: ")
	if dbname == "":
	print("Using default; kibble")
	dbname = "kibble"

	while mlserver == "":
	mlserver = input("What is the hostname of the outgoing mailserver? [localhost:25]: ")
	if mlserver == "":
	print("Using default; localhost:25")
	mlserver = "localhost:25"

	while shards < 1:
	try:
	shards = input("How many shards for the ElasticSearch index? [5]:")
	if shards == "":
	print("Using default; 5")
	shards = 5
	shards = int(shards)
	except ValueError:
	pass

	while replicas < 0:
	try:
	replicas = input("How many replicas for each shard? [1]: ")
	if replicas == "":
	print("Using default; 1")
	replicas = 1
	replicas = int(replicas)
	except ValueError:
	pass

	adminName = ""
	adminPass = ""
	if args.autoadmin:
	adminName = "admin@kibble"
	adminPass = "kibbleAdmin"
	while adminName == "":
	adminName = input("Enter an email address for the adminstrator account: ")
	while adminPass == "":
	adminPass = input("Enter a password for the adminstrator account: ")

	print("Okay, I got all I need, setting up Kibble...")

	def createIndex():
	global mappings
	global es

	defaultELConfig = {
	'host': hostname,
	'port': port
	}

	versionHint = ELASTIC_SEARCH_MIN_VERSION

	if versionHint >= 7:
	defaultELConfig['scheme'] = 'http'
	defaultELConfig['path_prefix'] = ''
	else:
	defaultELConfig['use_ssl'] = False
	defaultELConfig['url_prefix'] = ''

	es = Elasticsearch([ defaultELConfig],
	max_retries=5,
	retry_on_timeout=True
	)

	es6 = True if int(es.info()['version']['number'].split('.')[0]) >= 6 else False
	es7 = True if int(es.info()['version']['number'].split('.')[0]) >= 7 else False
	es8 = True if int(es.info()['version']['number'].split('.')[0]) >= 8 else False
	if not es6:
	print("New Kibble installations require ElasticSearch 6.x or newer! You appear to be running %s!" % es.info()['version']['number'])
	sys.exit(-1)
	# If ES >= 7, _doc is invalid and mapping should be rooted
	if es7:
	mappings['mappings'] = mappings['mappings']['_doc']
	# Check if index already exists
	if es.indices.exists(index=dbname+"_api"):
	if args.skiponexist: # Skip this is DB exists and -k added
	print("DB prefix exists, but --skiponexist used, skipping this step.")
	return
	print("Error: ElasticSearch DB prefix '%s' already exists!" % dbname)
	sys.exit(-1)

	types = [
	'api',
	# ci_*: CI service stats
	'ci_build',
	'ci_queue',
	# code_* + evolution + file_history: git repo stats
	'code_commit',
	'code_commit_unique',
	'code_modification',
	'evolution',
	'file_history',
	# forum_*: forum stats (SO, Discourse, Askbot etc)
	'forum_post',
	'forum_topic',
	'ghstats',
	# GitHub stats
	# im_*: Instant messaging stats
	'im_stats',
	'im_ops',
	'im_msg',
	'issue',
	'logstats',
	# email, mail*: Email statitics
	'email',
	'mailstats',
	'mailtop',
	# organisation, view, source, publish: UI Org DB
	'organisation',
	'view',
	'publish',
	'source',
	# stats: Miscellaneous stats
	'stats',
	# social_*: Twitter, Mastodon, Facebook etc
	'social_follow',
	'social_followers',
	'social_follower',
	'social_person',
	# uisession, useraccount, message: UI user DB
	'uisession',
	'useraccount',
	'message',
	# person: contributor DB
	'person',
	]

	for t in types:
	iname = "%s_%s" % (dbname, t)
	print("Creating index " + iname)

	settings = {
	"number_of_shards" : shards,
	"number_of_replicas" : replicas
	}


	res = es.indices.create(index = iname, body = {
	"mappings" : mappings['mappings'],
	"settings": settings
	}
	)

	print("Indices created! %s " % res)


	def createAccount():
	global es

	salt = bcrypt.gensalt()
	pwd = bcrypt.hashpw(adminPass.encode('utf-8'), salt) # .decode('ascii') -> 'str' object has no attribute 'decode'
	print("Creating administrator account")
	doc = {
	'email': adminName, # Username (email)
	'password': pwd, # Hashed password
	'displayName': "Administrator", # Display Name
	'organisations': [], # Orgs user belongs to (default is none)
	'ownerships': [], # Orgs user owns (default is none)
	'defaultOrganisation': None, # Default org for user
	#'defaultOrganisation': 'apache', # Default org for user
	'verified': True, # Account verified via email?
	'userlevel': "admin" # User level (user/admin)
	}
	dbdoc = {
	'apiversion': KIBBLE_VERSION, # Log current API version
	'dbversion': KIBBLE_DB_VERSION # Log the database revision we accept (might change!)
	}

	es7 = True if int(es.info()['version']['number'].split('.')[0]) >= 7 else False

	if es7:
	es.index(index=dbname+'_useraccount', id = adminName, body = doc)
	es.index(index=dbname+'_api', id = 'current', body = dbdoc)
	else:
	es.index(index=dbname+'_useraccount', doc_type = '_doc', id = adminName, body = doc)
	es.index(index=dbname+'_api', doc_type = '_doc', id = 'current', body = dbdoc)
	print("Account created!")


	try:
	import logging
	# elasticsearch logs lots of warnings on retries/connection failure
	logging.getLogger("elasticsearch").setLevel(logging.ERROR)
	createIndex()
	createAccount()


	except Exception as e:
	print("Index creation failed: %s" % e)
	sys.exit(1)

	kibble_yaml = '../api/yaml/kibble.yaml'

	if os.path.exists(kibble_yaml):
	print("%s already exists! Writing to %s.tmp instead" % (kibble_yaml, kibble_yaml))
	kibble_yaml = kibble_yaml + ".tmp"


	print("Writing Kibble config (%s)" % kibble_yaml)

	m = mlserver.split(':')
	if len(m) == 1:
	m.append(25)

	myconfig = {
	'api': {
	'version': KIBBLE_VERSION,
	'database': KIBBLE_DB_VERSION
	},
	'elasticsearch': {
	'host': hostname,
	'port': port,
	'ssl': False,
	'dbname': dbname,
	'versionHint': ELASTIC_SEARCH_MIN_VERSION
	},
	'mail': {
	'mailhost': m[0],
	'mailport': m[1],
	'sender': 'Kibble <noreply@kibble.kibble>'
	},
	'accounts': {
	'allowSignup': True,
	'verify': True
	}
	}

	with open(kibble_yaml, "w") as f:
	f.write(yaml.dump(myconfig, default_flow_style = False))
	f.close()


	print("All done, Kibble should...work now :)")