| #!/usr/bin/env python |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| """builds the entire stack""" |
| #For every matching build change here, that produces new installable |
| #files, the cloud.spec file and the Debian control files must be |
| #revised and tested. |
| |
| import shutil,os |
| import Utils,Node,Options,Logs,Scripting,Environment,Build,Configure |
| from os import unlink as _unlink, makedirs as _makedirs, getcwd as _getcwd, chdir as _chdir |
| from os.path import abspath as _abspath, basename as _basename, dirname as _dirname, exists as _exists, isdir as _isdir, split as _split, join as _join, sep, pathsep, pardir, curdir |
| from glob import glob as _glob |
| from subprocess import Popen as _Popen,PIPE |
| try: set([1,2,3]) |
| except Exception: from Sets import set |
| import re |
| import zipfile,tarfile |
| import time |
| try: |
| from os import chmod as _chmod,chown as _chown |
| import pwd,stat,grp |
| except ImportError: |
| _chmod,_chown,pwd,stat,grp = (None,None,None,None,None) |
| |
| |
| # Global variables setup |
| sourcedir = bld.srcnode.abspath() |
| builddir = bld.path.abspath(bld.env) |
| #buildpremium = _exists(_join(sourcedir,"cloudstack-proprietary")) |
| buildpremium = False |
| filelist = bld.path.ant_glob |
| distdir = Utils.relpath(_join(sourcedir,"dist")) |
| targetdir = Utils.relpath(_join(sourcedir,"target")) |
| |
| def gitinfo(dir=None): |
| if dir and not _isdir(dir): return '' |
| try: p = _Popen(['git','remote','show','-n','origin'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir) |
| except OSError,e: |
| if e.errno == 2: return '' # svn command is not installed |
| raise |
| stdout,stderr = p.communicate('') |
| retcode = p.wait() |
| # If the guess fails, just return nothing. |
| if retcode: return |
| stdout = [ s.strip() for s in stdout.splitlines() ] |
| try: url = [ s[11:] for s in stdout if s.startswith("Fetch URL") ][0] |
| except IndexError: url = [ s[5:] for s in stdout if s.startswith("URL") ][0] |
| assert url |
| |
| p = _Popen(['git','log','-1'],stdin=PIPE,stdout=PIPE,stderr=PIPE,cwd=dir) |
| stdout,stderr = p.communicate('') |
| retcode = p.wait() |
| if retcode: return |
| # If the guess fails, just return nothing. |
| stdout = [ s.strip() for s in stdout.splitlines() ] |
| commitid = [ s.split()[1] for s in stdout if s.startswith("commit") ][0] |
| assert commitid |
| |
| return "Git Revision: %s"%commitid + "\n" + "Git URL: %s"%url + "\n" |
| |
| def build_utils_docs (): |
| stdout = gitinfo() |
| if stdout: |
| f = file("sccs-info","w") |
| f.write(stdout) |
| f.flush() |
| f.close() |
| else: |
| if _exists("sccs-info"): |
| # If the file already existed, we preserve it |
| return |
| else: |
| f = file("sccs-info","w") |
| f.write("No revision control information could be detected when the source distribution was built.") |
| f.flush() |
| f.close() |
| |
| sccsinfo = _join(sourcedir,"sccs-info") |
| if _exists(sccsinfo): bld.install_files("${DOCDIR}","sccs-info") |
| |
| tgen = bld(features='subst', name='configure-info', source="configure-info.in", target="configure-info") |
| tgen.dict = {"CONFIGUREVARS":bld.getconfig()} |
| bld.install_files("${DOCDIR}","configure-info") |
| |
| # compile jar files using ant |
| # ant only needs to be reinvoked if the version with build number changes |
| # we here trim all the depended targets from the target list: |
| def build_jars (): |
| Implementation_Version = bld.env.VERSION |
| |
| # this is to trigger recompilation / cache avoidance if the relevant environment for ant changes |
| ant_args = [ |
| "build-all", |
| "-Dimpl.version=%s" % Implementation_Version, |
| "-Dtarget.dir=%s" % targetdir, |
| "-Ddist.dir=%s" % distdir, |
| "-Dbase.dir=%s" % sourcedir, |
| "-f %s" % Utils.relpath (_join(sourcedir, "build.xml")), |
| ] |
| |
| if buildpremium: |
| ant_args.append("-Dbuild.premium=true") |
| |
| tgen = bld(features='subst', name='version-info', source="version-info.in", target="version-info") |
| tgen.dict = { "Implementation_Version":Implementation_Version,"ant_args":ant_args } |
| bld.install_files("${DOCDIR}","version-info") |
| |
| bld.srcnode.ensure_dir_node_from_path("target/jar") |
| bld.srcnode.ensure_dir_node_from_path("dist") |
| |
| tgen = bld.new_task_gen (rule = Utils.runant, |
| name = "runant", |
| antargs = ant_args) |
| |
| jarnode = bld.srcnode.find_dir ('target/jar') |
| jars_str = jarnode.ant_glob ('*.jar').split () |
| ant_jars = [] |
| excludes = ["cloud-xstream-1.3.1.jar", "cloud-commons-dbcp-1.2.2.jar", |
| "cloud-commons-httpclient-3.1.jar", "cloud-commons-pool-1.4.jar", |
| "cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar", |
| "cloud-ws-commons-util-1.0.2.jar", |
| "cloud-commons-collections-3.2.1.jar", "vmware*.jar", "cloud-secstorage-extras.jar", |
| "cloud-agent-simulator.jar", "cloud-awsapi.jar", "cloud-test.jar", "cloud-wsdl4j.jar", "cloud-console-proxy.jar"] |
| |
| for a in jars_str: |
| if _basename (a).startswith ("cloud-") \ |
| and not _basename (a) in excludes: |
| a = jarnode.abspath () + os.sep + a |
| ant_jars.append (a) |
| |
| bld.install_files ('${JAVADIR}', ant_jars) |
| |
| |
| def build_premium (): |
| if buildpremium: bld.recurse(["cloudstack-proprietary/"],'build') |
| |
| def build_thirdparty_dir (): |
| start_path = bld.path.find_dir ("thirdparty") |
| bld.install_files('${PREMIUMJAVADIR}','*.jar', cwd = start_path) |
| Utils.pprint ("GREEN", "Installed files of thirdparty/") |
| |
| def build_dependences (): |
| excludes = ["cloud-xstream-1.3.1.jar", "cloud-servlet-api.jar", "cloud-commons-logging-1.1.1.jar", |
| "cloud-ws-commons-util-1.0.2.jar", |
| "cloud-commons-collections-3.2.1.jar", "cloud-wsdl4j.jar"] |
| |
| start_path = bld.path.find_dir ("deps") |
| |
| bld.install_files('${JAVADIR}',start_path.ant_glob(["CAStorSDK-*.jar", "javax.persistence-2.0.0.jar", "apache-log4j-extras-1.1.jar", "libvirt-0.4.9.jar", "axis2-1.5.1.jar", "jstl-1.2.jar", "commons-discovery-0.5.jar", "commons-codec-1.6.jar", "ejb-api-3.0.jar", "xmlrpc-client-3.1.3.jar", "commons-dbcp-1.4.jar", "commons-pool-1.6.jar", "gson-1.7.1.jar", |
| "netscaler-1.0.jar", "netscaler-sdx-1.0.jar", "backport-util-concurrent-3.1.jar", "ehcache-1.5.0.jar", "httpcore-4.0.jar", "log4j-1.2.16.jar", "trilead-ssh2-build213-svnkit-1.3-patch.jar", "cglib-nodep-2.2.2.jar", "xmlrpc-common-3.*.jar", |
| "axiom*.jar", "axis2*.jar", "antlr*.jar", "XmlSchema*.jar", "json-simple*.jar", "neethi*.jar", "woden*.jar", "xercesImpl*.jar", "xml-apis*.jar", "dom4j*.jar", "javassist*.jar", "commons-fileupload*.jar", |
| "xmlrpc-client-3.*.jar", "wsdl4j-1.6.2.jar", "bcprov-jdk16-1.45.jar", "jsch-0.1.42.jar", "jasypt-1.9.0.jar", "commons-configuration-1.8.jar", "mail-1.4.jar", "activation-1.1.jar", "xapi-5.6.100-1-SNAPSHOT.jar"], excl = excludes), cwd=start_path) |
| |
| #def build_console_proxy (): |
| # binary unsubstitutable files: |
| # start_path = bld.path.find_dir ("console-proxy") |
| # bld.install_files("${CPLIBDIR}",start_path.ant_glob("images/**",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True) |
| |
| # text substitutable files (substitute with tokens from the environment bld.env): |
| # bld.substitute('css/** js/** ui/** scripts/**',install_to="${CPLIBDIR}", cwd=start_path) |
| |
| # config files (do not replace them if preserve config option is true) |
| # if not Options.options.PRESERVECONFIG: bld.install_files_filtered("${CPSYSCONFDIR}","conf.dom0/*", cwd=start_path) |
| |
| def build_patches (): |
| # done here because the patches require substituted files |
| start_path = bld.path.find_dir ("patches") |
| bld.substitute("*/**",name="patchsubst", cwd = start_path) |
| |
| for virttech in Utils.to_list(start_path.ant_glob("*",dir=True)): |
| if virttech in ["shared"]: continue |
| patchfiles = start_path.ant_glob('shared/** %s/debian/config/**'%virttech,src=False,bld=True,dir=False,flat=True) |
| tmp = bld.path |
| bld.path = start_path |
| tgen = bld( |
| features = 'tar',#Utils.tar_up, |
| source = patchfiles, |
| target = 'cloud-scripts.tgz', |
| name = 'cloud-scripts_tgz', |
| root = os.path.join("patches", virttech + "/debian/config"), |
| rename = lambda x: re.sub(".subst$","",x), |
| ) |
| bld.path = tmp |
| |
| |
| def build_systemvm_patch (): |
| if bld.env.DISTRO not in ["Windows","Mac"]: |
| # patch creation |
| bld.install_files ("${COMMONLIBDIR}/vms", "%s/systemvm.zip" % distdir) |
| # ISO creation |
| bld.install_as("${COMMONLIBDIR}/vms/systemvm.iso", "%s/systemvm.iso" % distdir) |
| |
| def build_systemvm_iso (): |
| if buildpremium: |
| bld.install_as("${AGENTLIBDIR}/vms/systemvm-premium.iso", "%s/systemvm-premium.iso" % distdir) |
| |
| # =================== Empty directory / symlink creation on install target ==================== |
| |
| def build_dirs_symlinks (): |
| bld.createuser(bld.env.MSUSER,bld.env.MSENVIRON,'/bin/sh') |
| |
| # 7. make log and cache dirs (this actually runs first) |
| if bld.env.DISTRO in 'Windows Mac': pass |
| else: |
| x = ("root",bld.env.MSUSER) |
| directories = [ |
| ("${MSLOGDIR}",0770,x), |
| ("${AGENTLOGDIR}",0770,x), |
| ("${USAGELOGDIR}",0770,x), |
| # ("${CPLOGDIR}",0770,x), |
| ("${IPALLOCATORLOGDIR}",0770,x), |
| ("${LOCALSTATEDIR}/cache/${MSPATH}",0770,x), |
| ("${LOCALSTATEDIR}/cache/${MSPATH}/temp",0770,x), |
| ("${LOCALSTATEDIR}/cache/${MSPATH}/work",0770,x), |
| ("${SHAREDSTATEDIR}/${MSPATH}",0770,x), |
| ("${MSMNTDIR}",0770,x), |
| ("${MSCONF}/Catalina",0770,x), |
| ("${MSCONF}/Catalina/localhost",0770,x), |
| ("${MSCONF}/Catalina/localhost/client",0770,x), |
| ("${PIDDIR}",0755,("root","root")), |
| ("${LOCKDIR}",0755,("root","root")), |
| ] |
| |
| for a,mode,owner in directories: |
| s = bld.subst_add_destdir(a,bld) |
| if Options.is_install: |
| bld.install_dir(a) |
| bld.setownership(a,owner[0],owner[1],mode) |
| |
| # 8. create environment symlinks |
| symlinks = [ |
| ('${MSENVIRON}/bin', '${TOMCATHOME}/bin'), |
| ('${MSENVIRON}/lib', '${TOMCATHOME}/lib'), |
| ('${MSENVIRON}/logs', "${MSLOGDIR}"), |
| ('${MSENVIRON}/temp', '${LOCALSTATEDIR}/cache/${MSPATH}/temp'), |
| ('${MSENVIRON}/work','${LOCALSTATEDIR}/cache/${MSPATH}/work'), |
| ('${MSENVIRON}/conf', '${SYSCONFDIR}/${MSPATH}'), |
| # ("${AGENTLIBDIR}/css", '${CPLIBDIR}/css'), |
| # ("${AGENTLIBDIR}/images", '${CPLIBDIR}/images'), |
| # ("${AGENTLIBDIR}/js", '${CPLIBDIR}/js'), |
| # ("${AGENTLIBDIR}/ui", '${CPLIBDIR}/ui'), |
| ("${MSCONF}/server.xml", '${MSCONF}/server-nonssl.xml'), |
| ("${MSCONF}/tomcat6.conf", '${MSCONF}/tomcat6-nonssl.conf'), |
| ] |
| |
| for lnk,dst in symlinks: bld.symlink_as(lnk,Utils.subst_vars(dst,bld.env)) |
| |
| |
| def build_scripts (): |
| start_path = bld.path.find_dir ("scripts") |
| bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path) |
| |
| def build_bin_exec_dirs (): |
| #bld.install_files_filtered("${LIBEXECDIR}","*/libexec/* cloudstack-proprietary/*/libexec/*",chmod=0755) |
| #bld.install_files_filtered("${BINDIR}","*/bindir/* cloudstack-proprietary/*/bindir/*",chmod=0755) |
| #bld.install_files_filtered("${SBINDIR}","*/sbindir/* cloudstack-proprietary/*/sbindir/*",chmod=0755) |
| |
| bld.install_files_filtered("${LIBEXECDIR}","*/libexec/*",chmod=0755) |
| bld.install_files_filtered("${BINDIR}","*/bindir/*",chmod=0755) |
| bld.install_files_filtered("${SBINDIR}","*/sbindir/*",chmod=0755) |
| |
| def build_server_client (): |
| start_path = bld.path.find_dir("client/WEB-INF") |
| bld.install_files('${MSENVIRON}/webapps/client/WEB-INF', |
| start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), |
| cwd=start_path,relative_trick=True) |
| |
| start_path = bld.path.find_dir("client") |
| bld.install_files("${MSCONF}/resources",'WEB-INF/classes/resources/*.properties',chmod=0640, cwd=start_path) |
| |
| if not Options.options.PRESERVECONFIG: |
| bld.install_files_filtered("${MSCONF}","tomcatconf/*", cwd=start_path) |
| bld.install_files("${MSCONF}",'tomcatconf/db.properties',chmod=0640, cwd=start_path) |
| bld.setownership("${MSCONF}/db.properties","root",bld.env.MSUSER) |
| |
| start_path = bld.path.find_dir("server") |
| if not Options.options.PRESERVECONFIG: |
| bld.install_files_filtered("${SERVERSYSCONFDIR}","conf/*", cwd=start_path) |
| |
| def build_agent (): |
| start_path = bld.path.find_dir ("agent") |
| bld.install_files("${AGENTLIBDIR}", |
| start_path.ant_glob("storagepatch/**",src=True,bld=False,dir=False,flat=True), |
| cwd=start_path,relative_trick=True) |
| if not Options.options.PRESERVECONFIG: |
| bld.install_files_filtered("${AGENTSYSCONFDIR}","conf/*", cwd = start_path) |
| |
| def build_ui (): |
| # binary unsubstitutable files: |
| start_path = bld.path.find_dir ("ui") |
| bld.install_files ("${MSENVIRON}/webapps/client",start_path.ant_glob("*.ico **/*png **/*jpg **/*gif",src=True,bld=False,dir=False,flat=True),cwd=start_path,relative_trick=True) |
| |
| # text substitutable files (substitute with tokens from the environment bld.env): |
| bld.substitute ("*html **/*html **/*js **/*css **/*properties **/*jsp *jsp",install_to="${MSENVIRON}/webapps/client", cwd=start_path) |
| |
| def build_conf_files (): |
| # apply distro-specific config on top of the 'all' generic cloud-management config |
| globspec = _join("*","distro",bld.env.DISTRO.lower(),"*") # matches premium/distro/centos/SYSCONFDIR |
| #distrospecificdirs=_glob(globspec) + _glob(_join("cloudstack-proprietary",globspec)) |
| distrospecificdirs=_glob(globspec) |
| for dsdir in distrospecificdirs: |
| start_path = bld.srcnode.find_dir(dsdir) |
| subpath,varname = _split(dsdir) |
| dsdirwithvar = _join("${%s}"%varname) |
| files = filelist('%s/**'%dsdir,src=True,bld=False,dir=False,flat=True) |
| mode = 0644 |
| if "SYSCONFDIR" in dsdir: |
| mode = 0755 |
| if Options.options.PRESERVECONFIG: continue |
| bld.install_files_filtered(dsdirwithvar, files, cwd=start_path, relative_trick=True,chmod=mode) |
| |
| # cloudstack-proprietary still has db files for usage |
| def build_db_files (): |
| #bld.install_files_filtered("${SETUPDATADIR}",filelist("*/db/* cloudstack-proprietary/*/db/*",excl=Node.exclude_regs + "\ncloud-gate\ncloud-bridge")) |
| start_path = bld.path.find_dir ("setup/db") |
| bld.substitute('**',"${SETUPDATADIR}", cwd=start_path) |
| if buildpremium: |
| start_path = bld.path.find_dir ("cloudstack-proprietary/premium/db") |
| bld.substitute('**',"${SETUPDATADIR}", cwd=start_path) |
| |
| |
| def build_plugins (): |
| # ====================== Feature-specific plugins ======================== |
| |
| for plugin in _glob(_join("plugins","*")) + _glob(_join("cloudstack-proprietary","plugins","*")): |
| if not _exists(_join(plugin,"build.xml")): continue |
| pluginname = _basename(plugin) |
| target = 'target/jar/cloud-%s.jar' % pluginname |
| sources = filelist( '%s/**/*.java' % plugin.replace(sep,"/") , src=True, bld=False, dir=False ) |
| tgen = bld(rule=lambda x: runant("compile-%s"%pluginname), name='compile_%s'%pluginname, source=sources, target=target, after='runant') |
| bld.install_files('${PLUGINJAVADIR}',target) |
| |
| # ====================== End feature-specific plugins ==================== |
| |
| |
| # ====================== Vendor-specific plugins ======================== |
| |
| for vendor in _glob(_join("vendor","*")) + _glob(_join("cloudstack-proprietary","vendor","*")): |
| if not Options.options.PRESERVECONFIG: |
| bld.install_files_filtered("${MSCONF}/%s"%vendor,filelist("%s/tomcatconf/*"%vendor)) |
| |
| # ====================== End vendor-specific plugins ==================== |
| |
| def build_xml_api_description (): |
| def generate_xml_api_description(task): |
| relationship = Utils.relpath(sourcedir,os.getcwd()) |
| cp = [ _join(relationship,x) for x in task.generator.env.CLASSPATH.split(pathsep) ] |
| |
| jarnames = ['utils','server','core', 'api', 'server-extras'] |
| props = ["client/tomcatconf/commands.properties.in"] |
| |
| sources = [] |
| for i in jarnames: |
| str = 'target/jar/cloud-%s.jar' % i |
| sources.append (str) |
| sources.append ("client/tomcatconf/commands.properties.in") |
| if buildpremium: |
| sources.append("client/tomcatconf/commands-ext.properties.in") |
| |
| buildproducts = [] |
| for i in sources: |
| path = bld.path.abspath() + os.sep + i |
| buildproducts.append (path) |
| |
| jars = [ x for x in buildproducts if x.endswith("jar") ] |
| properties = [ x for x in buildproducts if x.endswith("properties.in") ] |
| cp += jars |
| cp = pathsep.join(cp) |
| arguments = ["-f",",".join(properties),"-d",builddir] |
| ret = Utils.exec_command(["java","-cp",cp,"com.cloud.api.doc.ApiXmlDocWriter"]+arguments,log=True) |
| return ret |
| |
| #TODO: We can't use 'source' token here because task_gen check if sources |
| # exist before all task_gen. This bring a problem that 'runant' task doesn't |
| # run when the check happen, which results in no source found at target/jar. |
| # Ask waf community to fix that |
| tgen = bld.new_task_gen ( |
| rule = generate_xml_api_description, |
| target = 'commands.xml', |
| name = 'xmlapi', |
| after = 'runant', |
| install_path="${CLIDIR}" |
| ) |
| |
| bld.install_as("${PYTHONDIR}/cloud_utils.py", 'python/lib/cloud_utils.py') |
| bld.install_files("${PYTHONDIR}/cloudtool", 'cloud-cli/cloudtool/*') |
| bld.install_files("${PYTHONDIR}/cloudutils", 'python/lib/cloudutils/*') |
| bld.install_as("${PYTHONDIR}/cloudapis.py", 'cloud-cli/cloudapis/cloud.py') |
| |
| def build_ovm(): |
| start_path = bld.path.find_dir("plugins/hypervisors/ovm/scripts") |
| bld.substitute('**',"${COMMONLIBDIR}/scripts",chmod=0755, cwd=start_path) |
| |
| def build_test(): |
| start_path = bld.path.find_dir("test/scripts") |
| bld.install_files('${LIBDIR}/${PACKAGE}/test', \ |
| start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \ |
| cwd=start_path,relative_trick=True) |
| |
| start_path = bld.path.find_dir("test/metadata") |
| bld.install_files('${SHAREDSTATEDIR}/${PACKAGE}/test', \ |
| start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True), \ |
| cwd=start_path,relative_trick=True) |
| |
| if not Options.options.PRESERVECONFIG: |
| start_path = bld.path.find_dir("test") |
| bld.install_files('${SYSCONFDIR}/${PACKAGE}/test','conf/*', cwd = start_path) |
| Utils.pprint ("GREEN", "Installed files of test/") |
| |
| def build_usage_dir (): |
| start_path = bld.path.find_dir ("usage") |
| bld.install_files_filtered("${USAGESYSCONFDIR}","conf/*", cwd = start_path) |
| bld.symlink_as("${USAGESYSCONFDIR}/db.properties", \ |
| Utils.subst_vars("${MSCONF}/db.properties",bld.env)) |
| Utils.pprint ("GREEN", "Installed files of usage/") |
| |
| |
| # Get started to execute here |
| #build_utils_docs () |
| build_jars () |
| build_premium () |
| #build_thirdparty_dir() |
| build_dependences () |
| #build_patches () |
| build_systemvm_patch () |
| build_dirs_symlinks () |
| build_scripts () |
| build_bin_exec_dirs () |
| build_server_client () |
| build_agent () |
| build_ui () |
| build_conf_files () |
| build_db_files () |
| build_plugins () |
| build_xml_api_description () |
| build_ovm () |
| #build_test() |
| build_usage_dir() |
| |
| # ====================== Magic! ========================================= |
| bld.use_the_magic() |