blob: 1c12c61291246c9e95f6b08510004960e808da4d [file] [log] [blame]
#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""builds the entire stack"""
#For every matching build change here, that produces new installable
#files, the cloud.spec file and the Debian control files must be
#revised and tested.
import shutil,os
import Utils,Node,Options,Logs,Scripting,Environment,Build,Configure
from os import unlink as _unlink, makedirs as _makedirs, getcwd as _getcwd, chdir as _chdir
from os.path import abspath as _abspath, basename as _basename, dirname as _dirname, exists as _exists, isdir as _isdir, split as _split, join as _join, sep, pathsep, pardir, curdir
from glob import glob as _glob
from subprocess import Popen as _Popen,PIPE
try: set([1,2,3])
except Exception: from Sets import set
import re
import zipfile,tarfile
import time
try:
from os import chmod as _chmod,chown as _chown
import pwd,stat,grp
except ImportError:
_chmod,_chown,pwd,stat,grp = (None,None,None,None,None)
# Global variables setup
sourcedir = bld.srcnode.abspath()
builddir = bld.path.abspath(bld.env)
#buildpremium = _exists(_join(sourcedir,"cloudstack-proprietary"))
buildpremium = False
filelist = bld.path.ant_glob
distdir = Utils.relpath(_join(sourcedir,"dist"))
targetdir = Utils.relpath(_join(sourcedir,"target"))
def gitinfo(dir=None):
if dir and not _isdir(dir): return ''
try: p = _Popen(['git','remote','show','-n','origin'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir)
except OSError,e:
if e.errno == 2: return '' # svn command is not installed
raise
stdout,stderr = p.communicate('')
retcode = p.wait()
# If the guess fails, just return nothing.
if retcode: return
stdout = [ s.strip() for s in stdout.splitlines() ]
try: url = [ s[11:] for s in stdout if s.startswith("Fetch URL") ][0]
except IndexError: url = [ s[5:] for s in stdout if s.startswith("URL") ][0]
assert url
p = _Popen(['git','log','-1'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir)
stdout,stderr = p.communicate('')
retcode = p.wait()
if retcode: return
# If the guess fails, just return nothing.
stdout = [ s.strip() for s in stdout.splitlines() ]
commitid = [ s.split()[1] for s in stdout if s.startswith("commit") ][0]
assert commitid
return "Git Revision: %s"%commitid + "\n" + "Git URL: %s"%url + "\n"
def build_utils_docs ():
stdout = gitinfo()
if stdout:
f = file("sccs-info","w")
f.write(stdout)
f.flush()
f.close()
else:
if _exists("sccs-info"):
# If the file already existed, we preserve it
return
else:
f = file("sccs-info","w")
f.write("No revision control information could be detected when the source distribution was built.")
f.flush()
f.close()
sccsinfo = _join(sourcedir,"sccs-info")
if _exists(sccsinfo): bld.install_files("${DOCDIR}","sccs-info")
tgen = bld(features='subst', name='configure-info', source="configure-info.in", target="configure-info")
tgen.dict = {"CONFIGUREVARS":bld.getconfig()}
bld.install_files("${DOCDIR}","configure-info")
# compile jar files using ant
# ant only needs to be reinvoked if the version with build number changes
# we here trim all the depended targets from the target list:
def build_jars ():
Implementation_Version = bld.env.VERSION
# this is to trigger recompilation / cache avoidance if the relevant environment for ant changes
ant_args = [
"build-all",
"-Dimpl.version=%s" % Implementation_Version,
"-Dtarget.dir=%s" % targetdir,
"-Ddist.dir=%s" % distdir,
"-Dbase.dir=%s" % sourcedir,
"-f %s" % Utils.relpath (_join(sourcedir, "build.xml")),
]
if buildpremium:
ant_args.append("-Dbuild.premium=true")
tgen = bld(features='subst', name='version-info', source="version-info.in", target="version-info")
tgen.dict = { "Implementation_Version":Implementation_Version,"ant_args":ant_args }
bld.install_files("${DOCDIR}","version-info")
bld.srcnode.ensure_dir_node_from_path("target/jar")
bld.srcnode.ensure_dir_node_from_path("dist")
tgen = bld.new_task_gen (rule = Utils.runant,
name = "runant",
antargs = ant_args)
jarnode = bld.srcnode.find_dir ('target/jar')
jars_str = jarnode.ant_glob ('*.jar').split ()
ant_jars = []
excludes = ["cloud-xstream-1.3.1.jar", "cloud-commons-dbcp-1.2.2.jar",
"cloud-commons-httpclient-3.1.jar", "cloud-commons-pool-1.4.jar",
"cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar",
"cloud-ws-commons-util-1.0.2.jar",
"cloud-commons-collections-3.2.1.jar", "vmware*.jar", "cloud-secstorage-extras.jar",
"cloud-agent-simulator.jar", "cloud-awsapi.jar", "cloud-test.jar", "cloud-wsdl4j.jar", "cloud-console-proxy.jar"]
for a in jars_str:
if _basename (a).startswith ("cloud-") \
and not _basename (a) in excludes:
a = jarnode.abspath () + os.sep + a
ant_jars.append (a)
bld.install_files ('${JAVADIR}', ant_jars)
def build_premium ():
if buildpremium: bld.recurse(["cloudstack-proprietary/"],'build')
def build_thirdparty_dir ():
start_path = bld.path.find_dir ("thirdparty")
bld.install_files('${PREMIUMJAVADIR}','*.jar', cwd = start_path)
Utils.pprint ("GREEN", "Installed files of thirdparty/")
def build_dependences ():
excludes = ["cloud-xstream-1.3.1.jar", "cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar",
"cloud-ws-commons-util-1.0.2.jar",
"cloud-commons-collections-3.2.1.jar", "cloud-wsdl4j.jar"]
start_path = bld.path.find_dir ("deps")
bld.install_files('${JAVADIR}',start_path.ant_glob(["CAStorSDK-*.jar", "javax.persistence-2.0.0.jar", "apache-log4j-extras-1.1.jar", "libvirt-0.4.9.jar", "axis2-1.5.1.jar", "jstl-1.2.jar", "commons-discovery-0.5.jar", "commons-codec-1.6.jar", "ejb-api-3.0.jar", "xmlrpc-client-3.1.3.jar", "commons-dbcp-1.4.jar", "commons-pool-1.6.jar", "gson-1.7.1.jar",
"netscaler-1.0.jar", "netscaler-sdx-1.0.jar", "backport-util-concurrent-3.1.jar", "ehcache-1.5.0.jar", "httpcore-4.0.jar", "log4j-1.2.16.jar", "trilead-ssh2-build213-svnkit-1.3-patch.jar", "cglib-nodep-2.2.2.jar", "xmlrpc-common-3.*.jar",
"axiom*.jar", "axis2*.jar", "antlr*.jar", "XmlSchema*.jar", "json-simple*.jar", "neethi*.jar", "woden*.jar", "xercesImpl*.jar", "xml-apis*.jar", "dom4j*.jar", "javassist*.jar", "commons-fileupload*.jar",
"xmlrpc-client-3.*.jar", "wsdl4j-1.6.2.jar", "bcprov-jdk16-1.45.jar", "jsch-0.1.42.jar", "jasypt-1.9.0.jar", "commons-configuration-1.8.jar", "mail-1.4.jar", "activation-1.1.jar", "xapi-5.6.100-1-SNAPSHOT.jar"], excl = excludes), cwd=start_path)
#def build_console_proxy ():
# binary unsubstitutable files:
# start_path = bld.path.find_dir ("console-proxy")
# bld.install_files("${CPLIBDIR}",start_path.ant_glob("images/**",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True)
# text substitutable files (substitute with tokens from the environment bld.env):
# bld.substitute('css/** js/** ui/** scripts/**',install_to="${CPLIBDIR}", cwd=start_path)
# config files (do not replace them if preserve config option is true)
# if not Options.options.PRESERVECONFIG: bld.install_files_filtered("${CPSYSCONFDIR}","conf.dom0/*", cwd=start_path)
def build_patches ():
# done here because the patches require substituted files
start_path = bld.path.find_dir ("patches")
bld.substitute("*/**",name="patchsubst", cwd = start_path)
for virttech in Utils.to_list(start_path.ant_glob("*",dir=True)):
if virttech in ["shared"]: continue
patchfiles = start_path.ant_glob('shared/** %s/debian/config/**'%virttech,src=False,bld=True,dir=False,flat=True)
tmp = bld.path
bld.path = start_path
tgen = bld(
features = 'tar',#Utils.tar_up,
source = patchfiles,
target = 'cloud-scripts.tgz',
name = 'cloud-scripts_tgz',
root = os.path.join("patches", virttech + "/debian/config"),
rename = lambda x: re.sub(".subst$","",x),
)
bld.path = tmp
def build_systemvm_patch ():
if bld.env.DISTRO not in ["Windows","Mac"]:
# patch creation
bld.install_files ("${COMMONLIBDIR}/vms", "%s/systemvm.zip" % distdir)
# ISO creation
bld.install_as("${COMMONLIBDIR}/vms/systemvm.iso", "%s/systemvm.iso" % distdir)
def build_systemvm_iso ():
if buildpremium:
bld.install_as("${AGENTLIBDIR}/vms/systemvm-premium.iso", "%s/systemvm-premium.iso" % distdir)
# =================== Empty directory / symlink creation on install target ====================
def build_dirs_symlinks ():
bld.createuser(bld.env.MSUSER,bld.env.MSENVIRON,'/bin/sh')
# 7. make log and cache dirs (this actually runs first)
if bld.env.DISTRO in 'Windows Mac': pass
else:
x = ("root",bld.env.MSUSER)
directories = [
("${MSLOGDIR}",0770,x),
("${AGENTLOGDIR}",0770,x),
("${USAGELOGDIR}",0770,x),
# ("${CPLOGDIR}",0770,x),
("${IPALLOCATORLOGDIR}",0770,x),
("${LOCALSTATEDIR}/cache/${MSPATH}",0770,x),
("${LOCALSTATEDIR}/cache/${MSPATH}/temp",0770,x),
("${LOCALSTATEDIR}/cache/${MSPATH}/work",0770,x),
("${SHAREDSTATEDIR}/${MSPATH}",0770,x),
("${MSMNTDIR}",0770,x),
("${MSCONF}/Catalina",0770,x),
("${MSCONF}/Catalina/localhost",0770,x),
("${MSCONF}/Catalina/localhost/client",0770,x),
("${PIDDIR}",0755,("root","root")),
("${LOCKDIR}",0755,("root","root")),
]
for a,mode,owner in directories:
s = bld.subst_add_destdir(a,bld)
if Options.is_install:
bld.install_dir(a)
bld.setownership(a,owner[0],owner[1],mode)
# 8. create environment symlinks
symlinks = [
('${MSENVIRON}/bin', '${TOMCATHOME}/bin'),
('${MSENVIRON}/lib', '${TOMCATHOME}/lib'),
('${MSENVIRON}/logs', "${MSLOGDIR}"),
('${MSENVIRON}/temp', '${LOCALSTATEDIR}/cache/${MSPATH}/temp'),
('${MSENVIRON}/work','${LOCALSTATEDIR}/cache/${MSPATH}/work'),
('${MSENVIRON}/conf', '${SYSCONFDIR}/${MSPATH}'),
# ("${AGENTLIBDIR}/css", '${CPLIBDIR}/css'),
# ("${AGENTLIBDIR}/images", '${CPLIBDIR}/images'),
# ("${AGENTLIBDIR}/js", '${CPLIBDIR}/js'),
# ("${AGENTLIBDIR}/ui", '${CPLIBDIR}/ui'),
("${MSCONF}/server.xml", '${MSCONF}/server-nonssl.xml'),
("${MSCONF}/tomcat6.conf", '${MSCONF}/tomcat6-nonssl.conf'),
]
for lnk,dst in symlinks: bld.symlink_as(lnk,Utils.subst_vars(dst,bld.env))
def build_scripts ():
start_path = bld.path.find_dir ("scripts")
bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path)
def build_bin_exec_dirs ():
#bld.install_files_filtered("${LIBEXECDIR}","*/libexec/* cloudstack-proprietary/*/libexec/*",chmod=0755)
#bld.install_files_filtered("${BINDIR}","*/bindir/* cloudstack-proprietary/*/bindir/*",chmod=0755)
#bld.install_files_filtered("${SBINDIR}","*/sbindir/* cloudstack-proprietary/*/sbindir/*",chmod=0755)
bld.install_files_filtered("${LIBEXECDIR}","*/libexec/*",chmod=0755)
bld.install_files_filtered("${BINDIR}","*/bindir/*",chmod=0755)
bld.install_files_filtered("${SBINDIR}","*/sbindir/*",chmod=0755)
def build_server_client ():
start_path = bld.path.find_dir("client/WEB-INF")
bld.install_files('${MSENVIRON}/webapps/client/WEB-INF',
start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True),
cwd=start_path,relative_trick=True)
start_path = bld.path.find_dir("client")
bld.install_files("${MSCONF}/resources",'WEB-INF/classes/resources/*.properties',chmod=0640, cwd=start_path)
if not Options.options.PRESERVECONFIG:
bld.install_files_filtered("${MSCONF}","tomcatconf/*", cwd=start_path)
bld.install_files("${MSCONF}",'tomcatconf/db.properties',chmod=0640, cwd=start_path)
bld.setownership("${MSCONF}/db.properties","root",bld.env.MSUSER)
start_path = bld.path.find_dir("server")
if not Options.options.PRESERVECONFIG:
bld.install_files_filtered("${SERVERSYSCONFDIR}","conf/*", cwd=start_path)
def build_agent ():
start_path = bld.path.find_dir ("agent")
bld.install_files("${AGENTLIBDIR}",
start_path.ant_glob("storagepatch/**",src=True,bld=False,dir=False,flat=True),
cwd=start_path,relative_trick=True)
if not Options.options.PRESERVECONFIG:
bld.install_files_filtered("${AGENTSYSCONFDIR}","conf/*", cwd = start_path)
def build_ui ():
# binary unsubstitutable files:
start_path = bld.path.find_dir ("ui")
bld.install_files ("${MSENVIRON}/webapps/client",start_path.ant_glob("*.ico **/*png **/*jpg **/*gif",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True)
# text substitutable files (substitute with tokens from the environment bld.env):
bld.substitute ("*html **/*html **/*js **/*css **/*properties **/*jsp *jsp",install_to="${MSENVIRON}/webapps/client", cwd=start_path)
def build_conf_files ():
# apply distro-specific config on top of the 'all' generic cloud-management config
globspec = _join("*","distro",bld.env.DISTRO.lower(),"*") # matches premium/distro/centos/SYSCONFDIR
#distrospecificdirs=_glob(globspec) + _glob(_join("cloudstack-proprietary",globspec))
distrospecificdirs=_glob(globspec)
for dsdir in distrospecificdirs:
start_path = bld.srcnode.find_dir(dsdir)
subpath,varname = _split(dsdir)
dsdirwithvar = _join("${%s}"%varname)
files = filelist('%s/**'%dsdir,src=True,bld=False,dir=False,flat=True)
mode = 0644
if "SYSCONFDIR" in dsdir:
mode = 0755
if Options.options.PRESERVECONFIG: continue
bld.install_files_filtered(dsdirwithvar, files, cwd=start_path, relative_trick=True,chmod=mode)
# cloudstack-proprietary still has db files for usage
def build_db_files ():
#bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/*/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge"))
start_path = bld.path.find_dir ("setup/db")
bld.substitute('**',"${SETUPDATADIR}", cwd=start_path)
if buildpremium:
start_path = bld.path.find_dir ("cloudstack-proprietary/premium/db")
bld.substitute('**',"${SETUPDATADIR}", cwd=start_path)
def build_plugins ():
# ====================== Feature-specific plugins ========================
for plugin in _glob(_join("plugins","*")) + _glob(_join("cloudstack-proprietary","plugins","*")):
if not _exists(_join(plugin,"build.xml")): continue
pluginname = _basename(plugin)
target = 'target/jar/cloud-%s.jar' % pluginname
sources = filelist( '%s/**/*.java' % plugin.replace(sep,"/") , src=True, bld=False, dir=False )
tgen = bld(rule=lambda x: runant("compile-%s"%pluginname), name='compile_%s'%pluginname, source=sources, target=target, after='runant')
bld.install_files('${PLUGINJAVADIR}',target)
# ====================== End feature-specific plugins ====================
# ====================== Vendor-specific plugins ========================
for vendor in _glob(_join("vendor","*")) + _glob(_join("cloudstack-proprietary","vendor","*")):
if not Options.options.PRESERVECONFIG:
bld.install_files_filtered("${MSCONF}/%s"%vendor,filelist("%s/tomcatconf/*"%vendor))
# ====================== End vendor-specific plugins ====================
def build_xml_api_description ():
def generate_xml_api_description(task):
relationship = Utils.relpath(sourcedir,os.getcwd())
cp = [ _join(relationship,x) for x in task.generator.env.CLASSPATH.split(pathsep) ]
jarnames = ['utils','server','core', 'api', 'server-extras']
props = ["client/tomcatconf/commands.properties.in"]
sources = []
for i in jarnames:
str = 'target/jar/cloud-%s.jar' % i
sources.append (str)
sources.append ("client/tomcatconf/commands.properties.in")
if buildpremium:
sources.append("client/tomcatconf/commands-ext.properties.in")
buildproducts = []
for i in sources:
path = bld.path.abspath() + os.sep + i
buildproducts.append (path)
jars = [ x for x in buildproducts if x.endswith("jar") ]
properties = [ x for x in buildproducts if x.endswith("properties.in") ]
cp += jars
cp = pathsep.join(cp)
arguments = ["-f",",".join(properties),"-d",builddir]
ret = Utils.exec_command(["java","-cp",cp,"com.cloud.api.doc.ApiXmlDocWriter"]+arguments,log=True)
return ret
#TODO: We can't use 'source' token here because task_gen check if sources
# exist before all task_gen. This bring a problem that 'runant' task doesn't
# run when the check happen, which results in no source found at target/jar.
# Ask waf community to fix that
tgen = bld.new_task_gen (
rule = generate_xml_api_description,
target = 'commands.xml',
name = 'xmlapi',
after = 'runant',
install_path="${CLIDIR}"
)
bld.install_as("${PYTHONDIR}/cloud_utils.py", 'python/lib/cloud_utils.py')
bld.install_files("${PYTHONDIR}/cloudtool", 'cloud-cli/cloudtool/*')
bld.install_files("${PYTHONDIR}/cloudutils", 'python/lib/cloudutils/*')
bld.install_as("${PYTHONDIR}/cloudapis.py", 'cloud-cli/cloudapis/cloud.py')
def build_ovm():
start_path = bld.path.find_dir("plugins/hypervisors/ovm/scripts")
bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path)
def build_test():
start_path = bld.path.find_dir("test/scripts")
bld.install_files('${LIBDIR}/${PACKAGE}/test', \
start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \
cwd=start_path,relative_trick=True)
start_path = bld.path.find_dir("test/metadata")
bld.install_files('${SHAREDSTATEDIR}/${PACKAGE}/test', \
start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \
cwd=start_path,relative_trick=True)
if not Options.options.PRESERVECONFIG:
start_path = bld.path.find_dir("test")
bld.install_files('${SYSCONFDIR}/${PACKAGE}/test','conf/*', cwd = start_path)
Utils.pprint ("GREEN", "Installed files of test/")
def build_usage_dir ():
start_path = bld.path.find_dir ("usage")
bld.install_files_filtered("${USAGESYSCONFDIR}","conf/*", cwd = start_path)
bld.symlink_as("${USAGESYSCONFDIR}/db.properties", \
Utils.subst_vars("${MSCONF}/db.properties",bld.env))
Utils.pprint ("GREEN", "Installed files of usage/")
# Get started to execute here
#build_utils_docs ()
build_jars ()
build_premium ()
#build_thirdparty_dir()
build_dependences ()
#build_patches ()
build_systemvm_patch ()
build_dirs_symlinks ()
build_scripts ()
build_bin_exec_dirs ()
build_server_client ()
build_agent ()
build_ui ()
build_conf_files ()
build_db_files ()
build_plugins ()
build_xml_api_description ()
build_ovm ()
#build_test()
build_usage_dir()
# ====================== Magic! =========================================
bld.use_the_magic()