| #!/usr/bin/env python |
| # |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| from glob import glob |
| from optparse import OptionParser |
| from time import gmtime, strftime |
| import pprint |
| import os |
| import re |
| import sys |
| import urllib |
| import urllib2 |
| try: |
| import json |
| except ImportError: |
| import simplejson as json |
| |
| releaseVersion={} |
| namePattern = re.compile(r' \([0-9]+\)') |
| |
| asflicense=''' |
| <!--- |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| --> |
| ''' |
| |
| def clean(str): |
| return tableclean(re.sub(namePattern, "", str)) |
| |
| def formatComponents(str): |
| str = re.sub(namePattern, '', str).replace("'", "") |
| if str != "": |
| ret = str |
| else: |
| # some markdown parsers don't like empty tables |
| ret = "." |
| return clean(ret) |
| |
| # convert to utf-8 |
| # protect some known md metachars |
| # or chars that screw up doxia |
| def tableclean(str): |
| str=str.encode('utf-8') |
| str=str.replace("_","\_") |
| str=str.replace("\r","") |
| str=str.rstrip() |
| return str |
| |
| # same thing as tableclean, |
| # except table metachars are also |
| # escaped as well as more |
| # things we don't want doxia to |
| # screw up |
| def notableclean(str): |
| str=tableclean(str) |
| str=str.replace("|","\|") |
| str=str.replace("<","\<") |
| str=str.replace(">","\>") |
| str=str.replace("*","\*") |
| str=str.rstrip() |
| return str |
| |
| # clean output dir |
| def cleanOutputDir(dir): |
| files = os.listdir(dir) |
| for name in files: |
| os.remove(os.path.join(dir,name)) |
| os.rmdir(dir) |
| |
| def mstr(obj): |
| if (obj is None): |
| return "" |
| return unicode(obj) |
| |
| def buildindex(title,license): |
| versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*"))) |
| with open("index.md","w") as indexfile: |
| if license is True: |
| indexfile.write(asflicense) |
| for v in versions: |
| indexfile.write("* %s v%s\n" % (title,v)) |
| for k in ("Changes","Release Notes"): |
| indexfile.write(" * %s (%s/%s.%s.html)\n" \ |
| % (k,v,k.upper().replace(" ",""),v)) |
| indexfile.close() |
| |
| class GetVersions: |
| """ yo """ |
| def __init__(self,versions, projects): |
| versions = versions |
| projects = projects |
| self.newversions = [] |
| pp = pprint.PrettyPrinter(indent=4) |
| at=0 |
| end=1 |
| count=100 |
| versions.sort() |
| print "Looking for %s through %s"%(versions[0],versions[-1]) |
| for p in projects: |
| resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p) |
| data = json.loads(resp.read()) |
| for d in data: |
| if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= versions[-1]: |
| print "Adding %s to the list" % d['name'] |
| self.newversions.append(d['name']) |
| newlist=list(set(self.newversions)) |
| self.newversions=newlist |
| |
| def getlist(self): |
| pp = pprint.PrettyPrinter(indent=4) |
| return(self.newversions) |
| |
| class Version: |
| """Represents a version number""" |
| def __init__(self, data): |
| self.mod = False |
| self.data = data |
| found = re.match('^((\d+)(\.\d+)*).*$', data) |
| if (found): |
| self.parts = [ int(p) for p in found.group(1).split('.') ] |
| else: |
| self.parts = [] |
| # backfill version with zeroes if missing parts |
| self.parts.extend((0,) * (3 - len(self.parts))) |
| |
| def __str__(self): |
| if (self.mod): |
| return '.'.join([ str(p) for p in self.parts ]) |
| return self.data |
| |
| def __cmp__(self, other): |
| return cmp(self.parts, other.parts) |
| |
| class Jira: |
| """A single JIRA""" |
| |
| def __init__(self, data, parent): |
| self.key = data['key'] |
| self.fields = data['fields'] |
| self.parent = parent |
| self.notes = None |
| self.incompat = None |
| self.reviewed = None |
| |
| def getId(self): |
| return mstr(self.key) |
| |
| def getDescription(self): |
| return mstr(self.fields['description']) |
| |
| def getReleaseNote(self): |
| if (self.notes is None): |
| field = self.parent.fieldIdMap['Release Note'] |
| if (self.fields.has_key(field)): |
| self.notes=mstr(self.fields[field]) |
| else: |
| self.notes=self.getDescription() |
| return self.notes |
| |
| def getPriority(self): |
| ret = "" |
| pri = self.fields['priority'] |
| if(pri is not None): |
| ret = pri['name'] |
| return mstr(ret) |
| |
| def getAssignee(self): |
| ret = "" |
| mid = self.fields['assignee'] |
| if(mid is not None): |
| ret = mid['displayName'] |
| return mstr(ret) |
| |
| def getComponents(self): |
| if (len(self.fields['components'])>0): |
| return ", ".join([ comp['name'] for comp in self.fields['components'] ]) |
| else: |
| return "" |
| |
| def getSummary(self): |
| return self.fields['summary'] |
| |
| def getType(self): |
| ret = "" |
| mid = self.fields['issuetype'] |
| if(mid is not None): |
| ret = mid['name'] |
| return mstr(ret) |
| |
| def getReporter(self): |
| ret = "" |
| mid = self.fields['reporter'] |
| if(mid is not None): |
| ret = mid['displayName'] |
| return mstr(ret) |
| |
| def getProject(self): |
| ret = "" |
| mid = self.fields['project'] |
| if(mid is not None): |
| ret = mid['key'] |
| return mstr(ret) |
| |
| def __cmp__(self,other): |
| selfsplit=self.getId().split('-') |
| othersplit=other.getId().split('-') |
| v1=cmp(selfsplit[0],othersplit[0]) |
| if (v1!=0): |
| return v1 |
| else: |
| if selfsplit[1] < othersplit[1]: |
| return True |
| elif selfsplit[1] > othersplit[1]: |
| return False |
| return False |
| |
| def getIncompatibleChange(self): |
| if (self.incompat is None): |
| field = self.parent.fieldIdMap['Hadoop Flags'] |
| self.reviewed=False |
| self.incompat=False |
| if (self.fields.has_key(field)): |
| if self.fields[field]: |
| for hf in self.fields[field]: |
| if hf['value'] == "Incompatible change": |
| self.incompat=True |
| if hf['value'] == "Reviewed": |
| self.reviewed=True |
| return self.incompat |
| |
| def checkMissingComponent(self): |
| if (len(self.fields['components'])>0): |
| return False |
| return True |
| |
| def checkMissingAssignee(self): |
| if (self.fields['assignee'] is not None): |
| return False |
| return True |
| |
| def checkVersionString(self): |
| field = self.parent.fieldIdMap['Fix Version/s'] |
| for h in self.fields[field]: |
| found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name']) |
| if not found: |
| return True |
| return False |
| |
| def getReleaseDate(self,version): |
| for j in range(len(self.fields['fixVersions'])): |
| if self.fields['fixVersions'][j]==version: |
| return(self.fields['fixVersions'][j]['releaseDate']) |
| return None |
| |
| class JiraIter: |
| """An Iterator of JIRAs""" |
| |
| def __init__(self, version, projects): |
| self.version = version |
| self.projects = projects |
| v=str(version).replace("-SNAPSHOT","") |
| |
| resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field") |
| data = json.loads(resp.read()) |
| |
| self.fieldIdMap = {} |
| for part in data: |
| self.fieldIdMap[part['name']] = part['id'] |
| |
| self.jiras = [] |
| at=0 |
| end=1 |
| count=100 |
| while (at < end): |
| params = urllib.urlencode({'jql': "project in ('"+"' , '".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 'startAt':at, 'maxResults':count}) |
| resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params) |
| data = json.loads(resp.read()) |
| if (data.has_key('errorMessages')): |
| raise Exception(data['errorMessages']) |
| at = data['startAt'] + data['maxResults'] |
| end = data['total'] |
| self.jiras.extend(data['issues']) |
| |
| needaversion=False |
| if v not in releaseVersion: |
| needaversion=True |
| |
| if needaversion is True: |
| for i in range(len(data['issues'])): |
| for j in range(len(data['issues'][i]['fields']['fixVersions'])): |
| if 'releaseDate' in data['issues'][i]['fields']['fixVersions'][j]: |
| releaseVersion[data['issues'][i]['fields']['fixVersions'][j]['name']]=\ |
| data['issues'][i]['fields']['fixVersions'][j]['releaseDate'] |
| |
| self.iter = self.jiras.__iter__() |
| |
| def __iter__(self): |
| return self |
| |
| def next(self): |
| data = self.iter.next() |
| j = Jira(data, self) |
| return j |
| |
| class Outputs: |
| """Several different files to output to at the same time""" |
| |
| def __init__(self, base_file_name, file_name_pattern, keys, params={}): |
| self.params = params |
| self.base = open(base_file_name%params, 'w') |
| self.others = {} |
| for key in keys: |
| both = dict(params) |
| both['key'] = key |
| self.others[key] = open(file_name_pattern%both, 'w') |
| |
| def writeAll(self, pattern): |
| both = dict(self.params) |
| both['key'] = '' |
| self.base.write(pattern%both) |
| for key in self.others.keys(): |
| both = dict(self.params) |
| both['key'] = key |
| self.others[key].write(pattern%both) |
| |
| def writeKeyRaw(self, key, str): |
| self.base.write(str) |
| if (self.others.has_key(key)): |
| self.others[key].write(str) |
| |
| def close(self): |
| self.base.close() |
| for fd in self.others.values(): |
| fd.close() |
| |
| def writeList(self, mylist): |
| for jira in sorted(mylist): |
| line = '| [%s](https://issues.apache.org/jira/browse/%s) | %s | %s | %s | %s | %s |\n' \ |
| % (notableclean(jira.getId()), notableclean(jira.getId()), |
| notableclean(jira.getSummary()), |
| notableclean(jira.getPriority()), |
| formatComponents(jira.getComponents()), |
| notableclean(jira.getReporter()), |
| notableclean(jira.getAssignee())) |
| self.writeKeyRaw(jira.getProject(), line) |
| |
| def main(): |
| parser = OptionParser(usage="usage: %prog --project PROJECT [--project PROJECT] --version VERSION [--version VERSION2 ...]", |
| epilog= |
| "Markdown-formatted CHANGES and RELEASENOTES files will be stored in a directory" |
| " named after the highest version provided.") |
| parser.add_option("-i","--index", dest="index", action="store_true", |
| default=False, help="build an index file") |
| parser.add_option("-l","--license", dest="license", action="store_false", |
| default=True, help="Add an ASF license") |
| parser.add_option("-n","--lint", dest="lint", action="store_true", |
| help="use lint flag to exit on failures") |
| parser.add_option("-p", "--project", dest="projects", |
| action="append", type="string", |
| help="projects in JIRA to include in releasenotes", metavar="PROJECT") |
| parser.add_option("-r", "--range", dest="range", action="store_true", |
| default=False, help="Given versions are a range") |
| parser.add_option("-t", "--projecttitle", dest="title", |
| type="string", |
| help="Title to use for the project (default is Apache PROJECT)") |
| parser.add_option("-u","--usetoday", dest="usetoday", action="store_true", |
| default=False, help="use current date for unreleased versions") |
| parser.add_option("-v", "--version", dest="versions", |
| action="append", type="string", |
| help="versions in JIRA to include in releasenotes", metavar="VERSION") |
| (options, args) = parser.parse_args() |
| |
| if (options.versions is None): |
| options.versions = [] |
| |
| if (len(args) > 2): |
| options.versions.append(args[2]) |
| |
| if (len(options.versions) <= 0): |
| parser.error("At least one version needs to be supplied") |
| |
| proxy = urllib2.ProxyHandler() |
| opener = urllib2.build_opener(proxy) |
| urllib2.install_opener(opener) |
| |
| projects = options.projects |
| |
| if (options.range is True): |
| versions = [ Version(v) for v in GetVersions(options.versions, projects).getlist() ] |
| else: |
| versions = [ Version(v) for v in options.versions ] |
| versions.sort(); |
| |
| if (options.title is None): |
| title=projects[0] |
| else: |
| title=options.title |
| |
| haderrors=False |
| |
| for v in versions: |
| vstr=str(v) |
| jlist = JiraIter(vstr,projects) |
| |
| if vstr in releaseVersion: |
| reldate=releaseVersion[vstr] |
| elif options.usetoday: |
| reldate=strftime("%Y-%m-%d", gmtime()) |
| else: |
| reldate="Unreleased" |
| |
| if not os.path.exists(vstr): |
| os.mkdir(vstr) |
| |
| reloutputs = Outputs("%(ver)s/RELEASENOTES.%(ver)s.md", |
| "%(ver)s/RELEASENOTES.%(key)s.%(ver)s.md", |
| [], {"ver":v, "date":reldate, "title":title}) |
| choutputs = Outputs("%(ver)s/CHANGES.%(ver)s.md", |
| "%(ver)s/CHANGES.%(key)s.%(ver)s.md", |
| [], {"ver":v, "date":reldate, "title":title}) |
| |
| if (options.license is True): |
| reloutputs.writeAll(asflicense) |
| choutputs.writeAll(asflicense) |
| |
| relhead = '# %(title)s %(key)s %(ver)s Release Notes\n\n' \ |
| 'These release notes cover new developer and user-facing incompatibilities, features, and major improvements.\n\n' |
| chhead = '# %(title)s Changelog\n\n' \ |
| '## Release %(ver)s - %(date)s\n'\ |
| '\n' |
| |
| reloutputs.writeAll(relhead) |
| choutputs.writeAll(chhead) |
| errorCount=0 |
| warningCount=0 |
| lintMessage="" |
| incompatlist=[] |
| buglist=[] |
| improvementlist=[] |
| newfeaturelist=[] |
| subtasklist=[] |
| tasklist=[] |
| testlist=[] |
| otherlist=[] |
| |
| for jira in sorted(jlist): |
| if jira.getIncompatibleChange(): |
| incompatlist.append(jira) |
| elif jira.getType() == "Bug": |
| buglist.append(jira) |
| elif jira.getType() == "Improvement": |
| improvementlist.append(jira) |
| elif jira.getType() == "New Feature": |
| newfeaturelist.append(jira) |
| elif jira.getType() == "Sub-task": |
| subtasklist.append(jira) |
| elif jira.getType() == "Task": |
| tasklist.append(jira) |
| elif jira.getType() == "Test": |
| testlist.append(jira) |
| else: |
| otherlist.append(jira) |
| |
| line = '* [%s](https://issues.apache.org/jira/browse/%s) | *%s* | **%s**\n' \ |
| % (notableclean(jira.getId()), notableclean(jira.getId()), notableclean(jira.getPriority()), |
| notableclean(jira.getSummary())) |
| |
| if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0): |
| warningCount+=1 |
| reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") |
| reloutputs.writeKeyRaw(jira.getProject(), line) |
| line ='\n**WARNING: No release note provided for this incompatible change.**\n\n' |
| lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId())) |
| reloutputs.writeKeyRaw(jira.getProject(), line) |
| |
| if jira.checkVersionString(): |
| warningCount+=1 |
| lintMessage += "\nWARNING: Version string problem for %s " % jira.getId() |
| |
| if (jira.checkMissingComponent() or jira.checkMissingAssignee()): |
| errorCount+=1 |
| errorMessage=[] |
| jira.checkMissingComponent() and errorMessage.append("component") |
| jira.checkMissingAssignee() and errorMessage.append("assignee") |
| lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId()) |
| |
| if (len(jira.getReleaseNote())>0): |
| reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n") |
| reloutputs.writeKeyRaw(jira.getProject(), line) |
| line ='\n%s\n\n' % (tableclean(jira.getReleaseNote())) |
| reloutputs.writeKeyRaw(jira.getProject(), line) |
| |
| if (options.lint is True): |
| print lintMessage |
| print "=======================================" |
| print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount) |
| if (errorCount>0): |
| haderrors=True |
| cleanOutputDir(vstr) |
| continue |
| |
| reloutputs.writeAll("\n\n") |
| reloutputs.close() |
| |
| choutputs.writeAll("### INCOMPATIBLE CHANGES:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(incompatlist) |
| |
| choutputs.writeAll("\n\n### NEW FEATURES:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(newfeaturelist) |
| |
| choutputs.writeAll("\n\n### IMPROVEMENTS:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(improvementlist) |
| |
| choutputs.writeAll("\n\n### BUG FIXES:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(buglist) |
| |
| choutputs.writeAll("\n\n### TESTS:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(testlist) |
| |
| choutputs.writeAll("\n\n### SUB-TASKS:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(subtasklist) |
| |
| choutputs.writeAll("\n\n### OTHER:\n\n") |
| choutputs.writeAll("| JIRA | Summary | Priority | Component | Reporter | Contributor |\n") |
| choutputs.writeAll("|:---- |:---- | :--- |:---- |:---- |:---- |\n") |
| choutputs.writeList(otherlist) |
| choutputs.writeList(tasklist) |
| |
| choutputs.writeAll("\n\n") |
| choutputs.close() |
| |
| if options.index: |
| buildindex(title,options.license) |
| |
| if haderrors is True: |
| sys.exit(1) |
| |
| if __name__ == "__main__": |
| main() |
