| #!/usr/bin/env python | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| """builds the entire stack""" | |
| #For every matching build change here, that produces new installable | |
| #files, the cloud.spec file and the Debian control files must be | |
| #revised and tested. | |
| import shutil,os | |
| import Utils,Node,Options,Logs,Scripting,Environment,Build,Configure | |
| from os import unlink as _unlink, makedirs as _makedirs, getcwd as _getcwd, chdir as _chdir | |
| from os.path import abspath as _abspath, basename as _basename, dirname as _dirname, exists as _exists, isdir as _isdir, split as _split, join as _join, sep, pathsep, pardir, curdir | |
| from glob import glob as _glob | |
| from subprocess import Popen as _Popen,PIPE | |
| try: set([1,2,3]) | |
| except Exception: from Sets import set | |
| import re | |
| import zipfile,tarfile | |
| import time | |
| try: | |
| from os import chmod as _chmod,chown as _chown | |
| import pwd,stat,grp | |
| except ImportError: | |
| _chmod,_chown,pwd,stat,grp = (None,None,None,None,None) | |
| # Global variables setup | |
| sourcedir = bld.srcnode.abspath() | |
| builddir = bld.path.abspath(bld.env) | |
| #buildpremium = _exists(_join(sourcedir,"cloudstack-proprietary")) | |
| buildpremium = False | |
| filelist = bld.path.ant_glob | |
| distdir = Utils.relpath(_join(sourcedir,"dist")) | |
| targetdir = Utils.relpath(_join(sourcedir,"target")) | |
| def gitinfo(dir=None): | |
| if dir and not _isdir(dir): return '' | |
| try: p = _Popen(['git','remote','show','-n','origin'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir) | |
| except OSError,e: | |
| if e.errno == 2: return '' # svn command is not installed | |
| raise | |
| stdout,stderr = p.communicate('') | |
| retcode = p.wait() | |
| # If the guess fails, just return nothing. | |
| if retcode: return | |
| stdout = [ s.strip() for s in stdout.splitlines() ] | |
| try: url = [ s[11:] for s in stdout if s.startswith("Fetch URL") ][0] | |
| except IndexError: url = [ s[5:] for s in stdout if s.startswith("URL") ][0] | |
| assert url | |
| p = _Popen(['git','log','-1'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir) | |
| stdout,stderr = p.communicate('') | |
| retcode = p.wait() | |
| if retcode: return | |
| # If the guess fails, just return nothing. | |
| stdout = [ s.strip() for s in stdout.splitlines() ] | |
| commitid = [ s.split()[1] for s in stdout if s.startswith("commit") ][0] | |
| assert commitid | |
| return "Git Revision: %s"%commitid + "\n" + "Git URL: %s"%url + "\n" | |
| def build_utils_docs (): | |
| stdout = gitinfo() | |
| if stdout: | |
| f = file("sccs-info","w") | |
| f.write(stdout) | |
| f.flush() | |
| f.close() | |
| else: | |
| if _exists("sccs-info"): | |
| # If the file already existed, we preserve it | |
| return | |
| else: | |
| f = file("sccs-info","w") | |
| f.write("No revision control information could be detected when the source distribution was built.") | |
| f.flush() | |
| f.close() | |
| sccsinfo = _join(sourcedir,"sccs-info") | |
| if _exists(sccsinfo): bld.install_files("${DOCDIR}","sccs-info") | |
| tgen = bld(features='subst', name='configure-info', source="configure-info.in", target="configure-info") | |
| tgen.dict = {"CONFIGUREVARS":bld.getconfig()} | |
| bld.install_files("${DOCDIR}","configure-info") | |
| # compile jar files using ant | |
| # ant only needs to be reinvoked if the version with build number changes | |
| # we here trim all the depended targets from the target list: | |
| def build_jars (): | |
| Implementation_Version = bld.env.VERSION | |
| # this is to trigger recompilation / cache avoidance if the relevant environment for ant changes | |
| ant_args = [ | |
| "build-all", | |
| "-Dimpl.version=%s" % Implementation_Version, | |
| "-Dtarget.dir=%s" % targetdir, | |
| "-Ddist.dir=%s" % distdir, | |
| "-Dbase.dir=%s" % sourcedir, | |
| "-f %s" % Utils.relpath (_join(sourcedir, "build.xml")), | |
| ] | |
| if buildpremium: | |
| ant_args.append("-Dbuild.premium=true") | |
| tgen = bld(features='subst', name='version-info', source="version-info.in", target="version-info") | |
| tgen.dict = { "Implementation_Version":Implementation_Version,"ant_args":ant_args } | |
| bld.install_files("${DOCDIR}","version-info") | |
| bld.srcnode.ensure_dir_node_from_path("target/jar") | |
| bld.srcnode.ensure_dir_node_from_path("dist") | |
| tgen = bld.new_task_gen (rule = Utils.runant, | |
| name = "runant", | |
| antargs = ant_args) | |
| jarnode = bld.srcnode.find_dir ('target/jar') | |
| jars_str = jarnode.ant_glob ('*.jar').split () | |
| ant_jars = [] | |
| excludes = ["cloud-xstream-1.3.1.jar", "cloud-commons-dbcp-1.2.2.jar", | |
| "cloud-commons-httpclient-3.1.jar", "cloud-commons-pool-1.4.jar", | |
| "cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar", | |
| "cloud-ws-commons-util-1.0.2.jar", | |
| "cloud-commons-collections-3.2.1.jar", "vmware*.jar", "cloud-secstorage-extras.jar", | |
| "cloud-agent-simulator.jar", "cloud-awsapi.jar", "cloud-test.jar", "cloud-wsdl4j.jar", "cloud-console-proxy.jar"] | |
| for a in jars_str: | |
| if _basename (a).startswith ("cloud-") \ | |
| and not _basename (a) in excludes: | |
| a = jarnode.abspath () + os.sep + a | |
| ant_jars.append (a) | |
| bld.install_files ('${JAVADIR}', ant_jars) | |
| def build_premium (): | |
| if buildpremium: bld.recurse(["cloudstack-proprietary/"],'build') | |
| def build_thirdparty_dir (): | |
| start_path = bld.path.find_dir ("thirdparty") | |
| bld.install_files('${PREMIUMJAVADIR}','*.jar', cwd = start_path) | |
| Utils.pprint ("GREEN", "Installed files of thirdparty/") | |
| def build_dependences (): | |
| excludes = ["cloud-xstream-1.3.1.jar", "cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar", | |
| "cloud-ws-commons-util-1.0.2.jar", | |
| "cloud-commons-collections-3.2.1.jar", "cloud-wsdl4j.jar"] | |
| start_path = bld.path.find_dir ("deps") | |
| bld.install_files('${JAVADIR}',start_path.ant_glob(["CAStorSDK-*.jar", "javax.persistence-2.0.0.jar", "apache-log4j-extras-1.1.jar", "libvirt-0.4.9.jar", "axis2-1.5.1.jar", "jstl-1.2.jar", "commons-discovery-0.5.jar", "commons-codec-1.6.jar", "ejb-api-3.0.jar", "xmlrpc-client-3.1.3.jar", "commons-dbcp-1.4.jar", "commons-pool-1.6.jar", "gson-1.7.1.jar", | |
| "netscaler-1.0.jar", "netscaler-sdx-1.0.jar", "backport-util-concurrent-3.1.jar", "ehcache-1.5.0.jar", "httpcore-4.0.jar", "log4j-1.2.16.jar", "trilead-ssh2-build213-svnkit-1.3-patch.jar", "cglib-nodep-2.2.2.jar", "xmlrpc-common-3.*.jar", | |
| "axiom*.jar", "axis2*.jar", "antlr*.jar", "XmlSchema*.jar", "json-simple*.jar", "neethi*.jar", "woden*.jar", "xercesImpl*.jar", "xml-apis*.jar", "dom4j*.jar", "javassist*.jar", "commons-fileupload*.jar", | |
| "xmlrpc-client-3.*.jar", "wsdl4j-1.6.2.jar", "bcprov-jdk16-1.45.jar", "jsch-0.1.42.jar", "jasypt-1.9.0.jar", "commons-configuration-1.8.jar", "mail-1.4.jar", "activation-1.1.jar", "xapi-5.6.100-1-SNAPSHOT.jar"], excl = excludes), cwd=start_path) | |
| #def build_console_proxy (): | |
| # binary unsubstitutable files: | |
| # start_path = bld.path.find_dir ("console-proxy") | |
| # bld.install_files("${CPLIBDIR}",start_path.ant_glob("images/**",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True) | |
| # text substitutable files (substitute with tokens from the environment bld.env): | |
| # bld.substitute('css/** js/** ui/** scripts/**',install_to="${CPLIBDIR}", cwd=start_path) | |
| # config files (do not replace them if preserve config option is true) | |
| # if not Options.options.PRESERVECONFIG: bld.install_files_filtered("${CPSYSCONFDIR}","conf.dom0/*", cwd=start_path) | |
| def build_patches (): | |
| # done here because the patches require substituted files | |
| start_path = bld.path.find_dir ("patches") | |
| bld.substitute("*/**",name="patchsubst", cwd = start_path) | |
| for virttech in Utils.to_list(start_path.ant_glob("*",dir=True)): | |
| if virttech in ["shared"]: continue | |
| patchfiles = start_path.ant_glob('shared/** %s/debian/config/**'%virttech,src=False,bld=True,dir=False,flat=True) | |
| tmp = bld.path | |
| bld.path = start_path | |
| tgen = bld( | |
| features = 'tar',#Utils.tar_up, | |
| source = patchfiles, | |
| target = 'cloud-scripts.tgz', | |
| name = 'cloud-scripts_tgz', | |
| root = os.path.join("patches", virttech + "/debian/config"), | |
| rename = lambda x: re.sub(".subst$","",x), | |
| ) | |
| bld.path = tmp | |
| def build_systemvm_patch (): | |
| if bld.env.DISTRO not in ["Windows","Mac"]: | |
| # patch creation | |
| bld.install_files ("${COMMONLIBDIR}/vms", "%s/systemvm.zip" % distdir) | |
| # ISO creation | |
| bld.install_as("${COMMONLIBDIR}/vms/systemvm.iso", "%s/systemvm.iso" % distdir) | |
| def build_systemvm_iso (): | |
| if buildpremium: | |
| bld.install_as("${AGENTLIBDIR}/vms/systemvm-premium.iso", "%s/systemvm-premium.iso" % distdir) | |
| # =================== Empty directory / symlink creation on install target ==================== | |
| def build_dirs_symlinks (): | |
| bld.createuser(bld.env.MSUSER,bld.env.MSENVIRON,'/bin/sh') | |
| # 7. make log and cache dirs (this actually runs first) | |
| if bld.env.DISTRO in 'Windows Mac': pass | |
| else: | |
| x = ("root",bld.env.MSUSER) | |
| directories = [ | |
| ("${MSLOGDIR}",0770,x), | |
| ("${AGENTLOGDIR}",0770,x), | |
| ("${USAGELOGDIR}",0770,x), | |
| # ("${CPLOGDIR}",0770,x), | |
| ("${IPALLOCATORLOGDIR}",0770,x), | |
| ("${LOCALSTATEDIR}/cache/${MSPATH}",0770,x), | |
| ("${LOCALSTATEDIR}/cache/${MSPATH}/temp",0770,x), | |
| ("${LOCALSTATEDIR}/cache/${MSPATH}/work",0770,x), | |
| ("${SHAREDSTATEDIR}/${MSPATH}",0770,x), | |
| ("${MSMNTDIR}",0770,x), | |
| ("${MSCONF}/Catalina",0770,x), | |
| ("${MSCONF}/Catalina/localhost",0770,x), | |
| ("${MSCONF}/Catalina/localhost/client",0770,x), | |
| ("${PIDDIR}",0755,("root","root")), | |
| ("${LOCKDIR}",0755,("root","root")), | |
| ] | |
| for a,mode,owner in directories: | |
| s = bld.subst_add_destdir(a,bld) | |
| if Options.is_install: | |
| bld.install_dir(a) | |
| bld.setownership(a,owner[0],owner[1],mode) | |
| # 8. create environment symlinks | |
| symlinks = [ | |
| ('${MSENVIRON}/bin', '${TOMCATHOME}/bin'), | |
| ('${MSENVIRON}/lib', '${TOMCATHOME}/lib'), | |
| ('${MSENVIRON}/logs', "${MSLOGDIR}"), | |
| ('${MSENVIRON}/temp', '${LOCALSTATEDIR}/cache/${MSPATH}/temp'), | |
| ('${MSENVIRON}/work','${LOCALSTATEDIR}/cache/${MSPATH}/work'), | |
| ('${MSENVIRON}/conf', '${SYSCONFDIR}/${MSPATH}'), | |
| # ("${AGENTLIBDIR}/css", '${CPLIBDIR}/css'), | |
| # ("${AGENTLIBDIR}/images", '${CPLIBDIR}/images'), | |
| # ("${AGENTLIBDIR}/js", '${CPLIBDIR}/js'), | |
| # ("${AGENTLIBDIR}/ui", '${CPLIBDIR}/ui'), | |
| ("${MSCONF}/server.xml", '${MSCONF}/server-nonssl.xml'), | |
| ("${MSCONF}/tomcat6.conf", '${MSCONF}/tomcat6-nonssl.conf'), | |
| ] | |
| for lnk,dst in symlinks: bld.symlink_as(lnk,Utils.subst_vars(dst,bld.env)) | |
| def build_scripts (): | |
| start_path = bld.path.find_dir ("scripts") | |
| bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path) | |
| def build_bin_exec_dirs (): | |
| #bld.install_files_filtered("${LIBEXECDIR}","*/libexec/* cloudstack-proprietary/*/libexec/*",chmod=0755) | |
| #bld.install_files_filtered("${BINDIR}","*/bindir/* cloudstack-proprietary/*/bindir/*",chmod=0755) | |
| #bld.install_files_filtered("${SBINDIR}","*/sbindir/* cloudstack-proprietary/*/sbindir/*",chmod=0755) | |
| bld.install_files_filtered("${LIBEXECDIR}","*/libexec/*",chmod=0755) | |
| bld.install_files_filtered("${BINDIR}","*/bindir/*",chmod=0755) | |
| bld.install_files_filtered("${SBINDIR}","*/sbindir/*",chmod=0755) | |
| def build_server_client (): | |
| start_path = bld.path.find_dir("client/WEB-INF") | |
| bld.install_files('${MSENVIRON}/webapps/client/WEB-INF', | |
| start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), | |
| cwd=start_path,relative_trick=True) | |
| start_path = bld.path.find_dir("client") | |
| bld.install_files("${MSCONF}/resources",'WEB-INF/classes/resources/*.properties',chmod=0640, cwd=start_path) | |
| if not Options.options.PRESERVECONFIG: | |
| bld.install_files_filtered("${MSCONF}","tomcatconf/*", cwd=start_path) | |
| bld.install_files("${MSCONF}",'tomcatconf/db.properties',chmod=0640, cwd=start_path) | |
| bld.setownership("${MSCONF}/db.properties","root",bld.env.MSUSER) | |
| start_path = bld.path.find_dir("server") | |
| if not Options.options.PRESERVECONFIG: | |
| bld.install_files_filtered("${SERVERSYSCONFDIR}","conf/*", cwd=start_path) | |
| def build_agent (): | |
| start_path = bld.path.find_dir ("agent") | |
| bld.install_files("${AGENTLIBDIR}", | |
| start_path.ant_glob("storagepatch/**",src=True,bld=False,dir=False,flat=True), | |
| cwd=start_path,relative_trick=True) | |
| if not Options.options.PRESERVECONFIG: | |
| bld.install_files_filtered("${AGENTSYSCONFDIR}","conf/*", cwd = start_path) | |
| def build_ui (): | |
| # binary unsubstitutable files: | |
| start_path = bld.path.find_dir ("ui") | |
| bld.install_files ("${MSENVIRON}/webapps/client",start_path.ant_glob("*.ico **/*png **/*jpg **/*gif",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True) | |
| # text substitutable files (substitute with tokens from the environment bld.env): | |
| bld.substitute ("*html **/*html **/*js **/*css **/*properties **/*jsp *jsp",install_to="${MSENVIRON}/webapps/client", cwd=start_path) | |
| def build_conf_files (): | |
| # apply distro-specific config on top of the 'all' generic cloud-management config | |
| globspec = _join("*","distro",bld.env.DISTRO.lower(),"*") # matches premium/distro/centos/SYSCONFDIR | |
| #distrospecificdirs=_glob(globspec) + _glob(_join("cloudstack-proprietary",globspec)) | |
| distrospecificdirs=_glob(globspec) | |
| for dsdir in distrospecificdirs: | |
| start_path = bld.srcnode.find_dir(dsdir) | |
| subpath,varname = _split(dsdir) | |
| dsdirwithvar = _join("${%s}"%varname) | |
| files = filelist('%s/**'%dsdir,src=True,bld=False,dir=False,flat=True) | |
| mode = 0644 | |
| if "SYSCONFDIR" in dsdir: | |
| mode = 0755 | |
| if Options.options.PRESERVECONFIG: continue | |
| bld.install_files_filtered(dsdirwithvar, files, cwd=start_path, relative_trick=True,chmod=mode) | |
| # cloudstack-proprietary still has db files for usage | |
| def build_db_files (): | |
| #bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/*/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge")) | |
| start_path = bld.path.find_dir ("setup/db") | |
| bld.substitute('**',"${SETUPDATADIR}", cwd=start_path) | |
| if buildpremium: | |
| start_path = bld.path.find_dir ("cloudstack-proprietary/premium/db") | |
| bld.substitute('**',"${SETUPDATADIR}", cwd=start_path) | |
| def build_plugins (): | |
| # ====================== Feature-specific plugins ======================== | |
| for plugin in _glob(_join("plugins","*")) + _glob(_join("cloudstack-proprietary","plugins","*")): | |
| if not _exists(_join(plugin,"build.xml")): continue | |
| pluginname = _basename(plugin) | |
| target = 'target/jar/cloud-%s.jar' % pluginname | |
| sources = filelist( '%s/**/*.java' % plugin.replace(sep,"/") , src=True, bld=False, dir=False ) | |
| tgen = bld(rule=lambda x: runant("compile-%s"%pluginname), name='compile_%s'%pluginname, source=sources, target=target, after='runant') | |
| bld.install_files('${PLUGINJAVADIR}',target) | |
| # ====================== End feature-specific plugins ==================== | |
| # ====================== Vendor-specific plugins ======================== | |
| for vendor in _glob(_join("vendor","*")) + _glob(_join("cloudstack-proprietary","vendor","*")): | |
| if not Options.options.PRESERVECONFIG: | |
| bld.install_files_filtered("${MSCONF}/%s"%vendor,filelist("%s/tomcatconf/*"%vendor)) | |
| # ====================== End vendor-specific plugins ==================== | |
| def build_xml_api_description (): | |
| def generate_xml_api_description(task): | |
| relationship = Utils.relpath(sourcedir,os.getcwd()) | |
| cp = [ _join(relationship,x) for x in task.generator.env.CLASSPATH.split(pathsep) ] | |
| jarnames = ['utils','server','core', 'api', 'server-extras'] | |
| props = ["client/tomcatconf/commands.properties.in"] | |
| sources = [] | |
| for i in jarnames: | |
| str = 'target/jar/cloud-%s.jar' % i | |
| sources.append (str) | |
| sources.append ("client/tomcatconf/commands.properties.in") | |
| if buildpremium: | |
| sources.append("client/tomcatconf/commands-ext.properties.in") | |
| buildproducts = [] | |
| for i in sources: | |
| path = bld.path.abspath() + os.sep + i | |
| buildproducts.append (path) | |
| jars = [ x for x in buildproducts if x.endswith("jar") ] | |
| properties = [ x for x in buildproducts if x.endswith("properties.in") ] | |
| cp += jars | |
| cp = pathsep.join(cp) | |
| arguments = ["-f",",".join(properties),"-d",builddir] | |
| ret = Utils.exec_command(["java","-cp",cp,"com.cloud.api.doc.ApiXmlDocWriter"]+arguments,log=True) | |
| return ret | |
| #TODO: We can't use 'source' token here because task_gen check if sources | |
| # exist before all task_gen. This bring a problem that 'runant' task doesn't | |
| # run when the check happen, which results in no source found at target/jar. | |
| # Ask waf community to fix that | |
| tgen = bld.new_task_gen ( | |
| rule = generate_xml_api_description, | |
| target = 'commands.xml', | |
| name = 'xmlapi', | |
| after = 'runant', | |
| install_path="${CLIDIR}" | |
| ) | |
| bld.install_as("${PYTHONDIR}/cloud_utils.py", 'python/lib/cloud_utils.py') | |
| bld.install_files("${PYTHONDIR}/cloudtool", 'cloud-cli/cloudtool/*') | |
| bld.install_files("${PYTHONDIR}/cloudutils", 'python/lib/cloudutils/*') | |
| bld.install_as("${PYTHONDIR}/cloudapis.py", 'cloud-cli/cloudapis/cloud.py') | |
| def build_ovm(): | |
| start_path = bld.path.find_dir("plugins/hypervisors/ovm/scripts") | |
| bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path) | |
| def build_test(): | |
| start_path = bld.path.find_dir("test/scripts") | |
| bld.install_files('${LIBDIR}/${PACKAGE}/test', \ | |
| start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \ | |
| cwd=start_path,relative_trick=True) | |
| start_path = bld.path.find_dir("test/metadata") | |
| bld.install_files('${SHAREDSTATEDIR}/${PACKAGE}/test', \ | |
| start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \ | |
| cwd=start_path,relative_trick=True) | |
| if not Options.options.PRESERVECONFIG: | |
| start_path = bld.path.find_dir("test") | |
| bld.install_files('${SYSCONFDIR}/${PACKAGE}/test','conf/*', cwd = start_path) | |
| Utils.pprint ("GREEN", "Installed files of test/") | |
| def build_usage_dir (): | |
| start_path = bld.path.find_dir ("usage") | |
| bld.install_files_filtered("${USAGESYSCONFDIR}","conf/*", cwd = start_path) | |
| bld.symlink_as("${USAGESYSCONFDIR}/db.properties", \ | |
| Utils.subst_vars("${MSCONF}/db.properties",bld.env)) | |
| Utils.pprint ("GREEN", "Installed files of usage/") | |
| # Get started to execute here | |
| #build_utils_docs () | |
| build_jars () | |
| build_premium () | |
| #build_thirdparty_dir() | |
| build_dependences () | |
| #build_patches () | |
| build_systemvm_patch () | |
| build_dirs_symlinks () | |
| build_scripts () | |
| build_bin_exec_dirs () | |
| build_server_client () | |
| build_agent () | |
| build_ui () | |
| build_conf_files () | |
| build_db_files () | |
| build_plugins () | |
| build_xml_api_description () | |
| build_ovm () | |
| #build_test() | |
| build_usage_dir() | |
| # ====================== Magic! ========================================= | |
| bld.use_the_magic() |