Merge branch '42cc_4254' of ssh://git.code.sf.net/p/allura/git into 42cc_4254
diff --git a/Allura/allura/config/app_cfg.py b/Allura/allura/config/app_cfg.py
index 7e8925c..e8cd735 100644
--- a/Allura/allura/config/app_cfg.py
+++ b/Allura/allura/config/app_cfg.py
@@ -26,7 +26,7 @@
import allura
# needed for tg.configuration to work
-from allura.lib import app_globals
+from allura.lib import app_globals, helpers
log = logging.getLogger(__name__)
@@ -62,6 +62,7 @@
autoescape=True,
extensions=['jinja2.ext.do', 'jinja2.ext.i18n'])
jinja2_env.install_gettext_translations(pylons.i18n)
+ jinja2_env.filters['filesizeformat'] = helpers.do_filesizeformat
config['pylons.app_globals'].jinja2_env = jinja2_env
# Jinja's unable to request c's attributes without strict_c
config['pylons.strict_c'] = True
diff --git a/Allura/allura/controllers/repository.py b/Allura/allura/controllers/repository.py
index 14e5b1f..cd28efc 100644
--- a/Allura/allura/controllers/repository.py
+++ b/Allura/allura/controllers/repository.py
@@ -18,6 +18,7 @@
import allura.tasks
from allura.lib import security
+from allura.lib import utils
from allura.lib import helpers as h
from allura.lib import widgets as w
from allura.lib.decorators import require_post
@@ -487,8 +488,10 @@
else:
force_display = 'force' in kw
context = self._blob.context()
+ stats = utils.generate_code_stats(self._blob)
return dict(
blob=self._blob,
+ stats=stats,
prev=context.get('prev', None),
next=context.get('next', None),
force_display=force_display
diff --git a/Allura/allura/lib/markdown_extensions.py b/Allura/allura/lib/markdown_extensions.py
index ba4fcdf..477fdc9 100644
--- a/Allura/allura/lib/markdown_extensions.py
+++ b/Allura/allura/lib/markdown_extensions.py
@@ -170,7 +170,8 @@
def _expand_link(self, link):
reference = self.alinks.get(link)
- if not reference:
+ mailto = u'\x02amp\x03#109;\x02amp\x03#97;\x02amp\x03#105;\x02amp\x03#108;\x02amp\x03#116;\x02amp\x03#111;\x02amp\x03#58;'
+ if not reference and not link.startswith(mailto):
return 'notfound'
else:
return ''
@@ -261,6 +262,7 @@
return
if val.startswith('/'): return
if val.startswith('.'): return
+ if val.startswith('mailto:'): return
tag[attr] = '../' + val
def _rewrite_abs(self, tag, attr):
diff --git a/Allura/allura/lib/plugin.py b/Allura/allura/lib/plugin.py
index 5342068..0a71818 100644
--- a/Allura/allura/lib/plugin.py
+++ b/Allura/allura/lib/plugin.py
@@ -414,17 +414,17 @@
if 'tools' in project_template:
for i, tool in enumerate(project_template['tools'].keys()):
tool_config = project_template['tools'][tool]
+ tool_options = tool_config.get('options', {})
+ for k, v in tool_options.iteritems():
+ if isinstance(v, basestring):
+ tool_options[k] = \
+ string.Template(v).safe_substitute(
+ p.__dict__.get('root_project', {}))
app = p.install_app(tool,
mount_label=tool_config['label'],
mount_point=tool_config['mount_point'],
- ordinal=i+offset)
- if 'options' in tool_config:
- for option in tool_config['options']:
- value = tool_config['options'][option]
- if isinstance(value, basestring):
- value = string.Template(value).safe_substitute(
- p.__dict__.get('root_project', {}))
- app.config.options[option] = value
+ ordinal=i + offset,
+ **tool_options)
if tool == 'wiki':
from forgewiki import model as WM
text = tool_config.get('home_text',
diff --git a/Allura/allura/lib/utils.py b/Allura/allura/lib/utils.py
index b70078a..65c7240 100644
--- a/Allura/allura/lib/utils.py
+++ b/Allura/allura/lib/utils.py
@@ -8,6 +8,7 @@
import datetime
import random
import mimetypes
+import re
from itertools import groupby
import tg
@@ -410,3 +411,15 @@
yield (tup[0], '<div id="l%s" class="code_block">%s</div>' % (num, tup[1]))
num += 1
yield 0, '</pre>'
+
+def generate_code_stats(blob):
+ stats = {'line_count': 0,
+ 'code_size': 0,
+ 'data_line_count': 0}
+ code = blob.text
+ lines = code.split('\n')
+ stats['code_size'] = blob.size
+ stats['line_count'] = len(lines)
+ spaces = re.compile(r'^\s*$')
+ stats['data_line_count'] = sum([1 for l in lines if not spaces.match(l)])
+ return stats
diff --git a/Allura/allura/model/artifact.py b/Allura/allura/model/artifact.py
index f887323..6455360 100644
--- a/Allura/allura/model/artifact.py
+++ b/Allura/allura/model/artifact.py
@@ -576,6 +576,9 @@
'pubdate',
('artifact_ref.project_id', 'artifact_ref.mount_point'),
(('ref_id', pymongo.ASCENDING),
+ ('pubdate', pymongo.DESCENDING)),
+ (('project_id', pymongo.ASCENDING),
+ ('app_config_id', pymongo.ASCENDING),
('pubdate', pymongo.DESCENDING))]
_id = FieldProperty(S.ObjectId)
diff --git a/Allura/allura/model/project.py b/Allura/allura/model/project.py
index 5ca3f4c..ac62d41 100644
--- a/Allura/allura/model/project.py
+++ b/Allura/allura/model/project.py
@@ -34,6 +34,7 @@
class ProjectFile(File):
class __mongometa__:
session = main_orm_session
+ indexes = [('project_id', 'category')]
project_id=FieldProperty(S.ObjectId)
category=FieldProperty(str)
@@ -274,7 +275,7 @@
else:
self.acl.append(ace)
private = property(_get_private, _set_private)
-
+
@property
def is_user_project(self):
return self.shortname.startswith('u/')
@@ -415,7 +416,7 @@
for sm in app.sitemap:
entry = sm.bind_app(app)
entry.ui_icon='tool-%s' % ac.tool_name.lower()
- ordinal = ac.options.get('ordinal', 0) + delta_ordinal
+ ordinal = int(ac.options.get('ordinal', 0)) + delta_ordinal
if ordinal > max_ordinal:
max_ordinal = ordinal
entries.append({'ordinal':ordinal,'entry':entry})
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index c5a73b5..a070e4e 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -108,6 +108,10 @@
'''Return a file-like object that contains the contents of the blob'''
raise NotImplementedError, 'open_blob'
+ def blob_size(self, blob):
+ '''Return a blob size in bytes'''
+ raise NotImplementedError, 'blob_size'
+
@classmethod
def shorthand_for_commit(cls, oid):
return '[%s]' % oid[:6]
@@ -204,6 +208,8 @@
return self._impl.commit_context(commit)
def open_blob(self, blob):
return self._impl.open_blob(blob)
+ def blob_size(self, blob):
+ return self._impl.blob_size(blob)
def shorthand_for_commit(self, oid):
return self._impl.shorthand_for_commit(oid)
def symbolics_for_commit(self, commit):
@@ -1108,6 +1114,10 @@
return iter(self.open())
@LazyProperty
+ def size(self):
+ return self.repo.blob_size(self)
+
+ @LazyProperty
def text(self):
return self.open().read()
diff --git a/Allura/allura/templates/repo/file.html b/Allura/allura/templates/repo/file.html
index 9928258..7a60a49 100644
--- a/Allura/allura/templates/repo/file.html
+++ b/Allura/allura/templates/repo/file.html
@@ -68,9 +68,13 @@
{% elif blob.has_html_view or blob.has_pypeline_view or force_display %}
<p><a href="?format=raw">Download this file</a></p>
<div class="clip grid-19">
- <h3><span class="ico-l"><b data-icon="{{g.icons['table'].char}}" class="ico {{g.icons['table'].css}}"></b> {{h.really_unicode(blob.name)}}</span></h3>
+ <h3>
+ <span class="ico-l"><b data-icon="{{g.icons['table'].char}}" class="ico {{g.icons['table'].css}}"></b> {{h.really_unicode(blob.name)}}</span>
+
+ {{ stats.line_count }} lines ({{ stats.data_line_count }} with data), {{ stats.code_size|filesizeformat }}
+ </h3>
{% if blob.has_pypeline_view %}
- {{h.render_any_markup(blob.name, blob.text, code_mode=True, linenumbers_style=h.TABLE)}}
+ {{h.render_any_markup(blob.name, blob.text, code_mode=True)}}
{% else %}
{{g.highlight(blob.text, filename=blob.name)}}
{% endif %}
diff --git a/Allura/allura/tests/unit/test_utils.py b/Allura/allura/tests/unit/test_utils.py
new file mode 100644
index 0000000..2aecd61
--- /dev/null
+++ b/Allura/allura/tests/unit/test_utils.py
@@ -0,0 +1,28 @@
+import unittest
+from mock import Mock
+
+from alluratest.controller import setup_unit_test
+from allura.lib.utils import generate_code_stats
+
+class TestCodeStats(unittest.TestCase):
+
+ def setUp(self):
+ setup_unit_test()
+
+ def test_generate_code_stats(self):
+ blob = Mock()
+ blob.text = \
+"""class Person(object):
+
+ def __init__(self, name='Alice'):
+ self.name = name
+
+ def greetings(self):
+ print "Hello, %s" % self.name
+\t\t"""
+ blob.size = len(blob.text)
+
+ stats = generate_code_stats(blob)
+ assert stats['line_count'] == 8
+ assert stats['data_line_count'] == 5
+ assert stats['code_size'] == len(blob.text)
diff --git a/ForgeBlog/forgeblog/command/rssfeeds.py b/ForgeBlog/forgeblog/command/rssfeeds.py
index 6a21127..a02c1ff 100644
--- a/ForgeBlog/forgeblog/command/rssfeeds.py
+++ b/ForgeBlog/forgeblog/command/rssfeeds.py
@@ -15,7 +15,7 @@
from allura import model as M
from forgeblog import model as BM
from forgeblog import version
-from forgeblog.main import ForgeBlogApp
+from forgeblog.main import ForgeBlogApp
from allura.lib import exceptions
html2text.BODY_WIDTH = 0
@@ -78,10 +78,6 @@
else:
res_data = u"%s\n" % res_data
- if data[-1:] == "\n" and self.custom_tag_opened:
- res_data = u"%s%s" % (res_data, self.CUSTTAG_CLOSE)
- self.custom_tag_opened = False
-
self.result_doc = u"%s%s" % (self.result_doc, res_data)
def handle_comment(self, data):
@@ -174,29 +170,38 @@
content = u''
for ct in e.content:
if ct.type != 'text/html':
- content = u"%s<p>%s</p>" % (content, ct.value)
+ content += '[plain]%s[/plain]' % ct.value
else:
- content = content + ct.value
- else:
- content = e.summary
+ if False:
+ # FIXME: disabled until https://sourceforge.net/p/allura/tickets/4345
+ # because the bad formatting from [plain] is worse than bad formatting from unintentional markdown syntax
+ parser = MDHTMLParser()
+ parser.feed(ct.value)
+ parser.close() # must be before using the result_doc
+ markdown_content = html2text.html2text(parser.result_doc, baseurl=e.link)
+ else:
+ markdown_content = html2text.html2text(ct.value, baseurl=e.link)
- content = u'%s <a href="%s">link</a>' % (content, e.link)
- parser = MDHTMLParser()
- parser.feed(content)
- parser.close()
- content = html2text.html2text(parser.result_doc, e.link)
+ content += markdown_content
+ else:
+ content = '[plain]%s[/plain]' % getattr(e, 'summary',
+ getattr(e, 'subtitle',
+ getattr(e, 'title')))
+
+ # plain tags fix
+ content += u' [link](%s)' % e.link
updated = datetime.utcfromtimestamp(mktime(e.updated_parsed))
- base_slug = BM.BlogPost.make_base_slug(title, updated, feed_url)
- b_count = BM.BlogPost.query.find(dict(slug=base_slug)).count()
+ base_slug = BM.BlogPost.make_base_slug(title, updated)
+ b_count = BM.BlogPost.query.find(dict(slug=base_slug, app_config_id=appid)).count()
if b_count == 0:
post = BM.BlogPost(title=title, text=content, timestamp=updated,
app_config_id=appid,
tool_version={'blog': version.__version__},
state='published')
post.neighborhood_id=c.project.neighborhood_id
- post.make_slug(source=feed_url)
+ post.make_slug()
post.commit()
session(BM.BlogPost).flush()
diff --git a/ForgeBlog/forgeblog/model/blog.py b/ForgeBlog/forgeblog/model/blog.py
index 0644634..8b7f03e 100644
--- a/ForgeBlog/forgeblog/model/blog.py
+++ b/ForgeBlog/forgeblog/model/blog.py
@@ -139,26 +139,17 @@
return '%s@%s%s' % (self.title.replace('/', '.'), domain, config.common_suffix)
@staticmethod
- def make_base_slug(title, timestamp, source = None):
+ def make_base_slug(title, timestamp):
slugsafe = ''.join(
ch.lower()
for ch in title.replace(' ', '-')
if ch.isalnum() or ch == '-')
- if source is None:
- base = '%s/%s' % (
+ return '%s/%s' % (
timestamp.strftime('%Y/%m'),
slugsafe)
- else:
- m = hashlib.md5()
- m.update(source)
- link_hash_key = m.hexdigest()[:16]
- base = '%s/%s/%s' % (
- timestamp.strftime('%Y/%m'),
- link_hash_key, slugsafe)
- return base
- def make_slug(self, source = None):
- base = BlogPost.make_base_slug(self.title, self.timestamp, source)
+ def make_slug(self):
+ base = BlogPost.make_base_slug(self.title, self.timestamp)
self.slug = base
while True:
try:
@@ -166,7 +157,6 @@
return self.slug
except DuplicateKeyError:
self.slug = base + '-%.3d' % randint(0,999)
- return self.slug
def url(self):
return self.app.url + self.slug + '/'
diff --git a/ForgeGit/forgegit/model/git_repo.py b/ForgeGit/forgegit/model/git_repo.py
index 7601b06..54a8dc1 100644
--- a/ForgeGit/forgegit/model/git_repo.py
+++ b/ForgeGit/forgegit/model/git_repo.py
@@ -272,6 +272,9 @@
return _OpenedGitBlob(
self._object(blob.object_id).data_stream)
+ def blob_size(self, blob):
+ return self._object(blob.object_id).data_stream.size
+
def _setup_hooks(self):
'Set up the git post-commit hook'
text = self.post_receive_template.substitute(
diff --git a/ForgeHg/forgehg/model/hg.py b/ForgeHg/forgehg/model/hg.py
index 75a858e..2c0017f 100644
--- a/ForgeHg/forgehg/model/hg.py
+++ b/ForgeHg/forgehg/model/hg.py
@@ -257,6 +257,10 @@
fctx = self._hg[blob.commit.object_id][h.really_unicode(blob.path()).encode('utf-8')[1:]]
return StringIO(fctx.data())
+ def blob_size(self, blob):
+ fctx = self._hg[blob.commit.object_id][h.really_unicode(blob.path()).encode('utf-8')[1:]]
+ return fctx.size()
+
def _setup_hooks(self):
'Set up the hg changegroup hook'
cp = ConfigParser()
diff --git a/ForgeSVN/forgesvn/model/svn.py b/ForgeSVN/forgesvn/model/svn.py
index 128dc2e..d6b7819 100644
--- a/ForgeSVN/forgesvn/model/svn.py
+++ b/ForgeSVN/forgesvn/model/svn.py
@@ -431,6 +431,24 @@
revision=self._revision(blob.commit.object_id))
return StringIO(data)
+ def blob_size(self, blob):
+ try:
+ data = self._svn.list(
+ self._url + blob.path(),
+ revision=self._revision(blob.commit.object_id),
+ dirent_fields=pysvn.SVN_DIRENT_SIZE)
+ except pysvn.ClientError:
+ log.info('ClientError getting filesize %r %r, returning 0', blob.path(), self._repo, exc_info=True)
+ return 0
+
+ try:
+ size = data[0][0]['size']
+ except (IndexError, KeyError):
+ log.info('Error getting filesize: bad data from svn client %r %r, returning 0', blob.path(), self._repo, exc_info=True)
+ size = 0
+
+ return size
+
def _setup_hooks(self):
'Set up the post-commit and pre-revprop-change hooks'
text = self.post_receive_template.substitute(
diff --git a/ForgeSVN/forgesvn/widgets.py b/ForgeSVN/forgesvn/widgets.py
index eaf3c82..1cfda7d 100644
--- a/ForgeSVN/forgesvn/widgets.py
+++ b/ForgeSVN/forgesvn/widgets.py
@@ -1,3 +1,5 @@
+import re
+
from formencode import validators as fev
import ew as ew_core
@@ -5,8 +7,20 @@
from allura.lib.widgets.forms import ForgeForm
+class ValidateSvnUrl(fev.URL):
+ url_re = re.compile(r'''
+ ^(http|https|svn)://
+ (?:[%:\w]*@)? # authenticator
+ (?P<domain>[a-z0-9][a-z0-9\-]{,62}\.)* # subdomain
+ (?P<tld>[a-z]{2,63}|xn--[a-z0-9\-]{2,59}) # top level domain
+ (?::[0-9]{1,5})? # port
+ # files/delims/etc
+ (?P<path>/[a-z0-9\-\._~:/\?#\[\]@!%\$&\'\(\)\*\+,;=]*)?
+ $
+ ''', re.I | re.VERBOSE)
+
class ImportForm(ForgeForm):
submit_text='Import'
class fields(ew_core.NameList):
checkout_url = ew.TextField(label='Checkout URL',
- validator=fev.URL())
+ validator=ValidateSvnUrl(not_empty=True))
diff --git a/ForgeTracker/forgetracker/model/ticket.py b/ForgeTracker/forgetracker/model/ticket.py
index 0e81a91..d60d8e1 100644
--- a/ForgeTracker/forgetracker/model/ticket.py
+++ b/ForgeTracker/forgetracker/model/ticket.py
@@ -211,6 +211,7 @@
'ticket_num',
'app_config_id',
('app_config_id', 'custom_fields._milestone'),
+ 'import_id',
]
unique_indexes = [
('app_config_id', 'ticket_num'),
diff --git a/ForgeWiki/forgewiki/tests/functional/test_root.py b/ForgeWiki/forgewiki/tests/functional/test_root.py
index 43dd786..d99abc4 100644
--- a/ForgeWiki/forgewiki/tests/functional/test_root.py
+++ b/ForgeWiki/forgewiki/tests/functional/test_root.py
@@ -488,3 +488,31 @@
response = self.app.get('/wiki/browse_pages/')
assert 'aaa' in response
assert '?deleted=True">bbb' in response
+
+ def test_mailto_links(self):
+ self.app.get('/wiki/test_mailto/')
+ params = {
+ 'title':'test_mailto',
+ 'text':'''
+* Automatic mailto #1 <darth.vader@deathstar.org>
+* Automatic mailto #2 <mailto:luke.skywalker@tatooine.org>
+* Handmaid mailto <a href="mailto:yoda@jedi.org">Email Yoda</a>
+''',
+ 'labels':'',
+ 'labels_old':'',
+ 'viewable_by-0.id':'all'}
+ self.app.post('/wiki/test_mailto/update', params=params)
+ r = self.app.get('/wiki/test_mailto/')
+ mailto_links = 0
+ for link in r.html.findAll('a'):
+ if link.get('href') == 'mailto:darth.vader@deathstar.org':
+ assert 'notfound' not in link.get('class', '')
+ mailto_links +=1
+ if link.get('href') == 'mailto:luke.skywalker@tatooine.org':
+ assert 'notfound' not in link.get('class', '')
+ mailto_links += 1
+ if link.get('href') == 'mailto:yoda@jedi.org':
+ assert link.contents == ['Email Yoda']
+ assert 'notfound' not in link.get('class', '')
+ mailto_links += 1
+ assert mailto_links == 3, 'Wrong number of mailto links'
diff --git a/scripts/project-import.py b/scripts/project-import.py
index 3b2685a..b7cd686 100644
--- a/scripts/project-import.py
+++ b/scripts/project-import.py
@@ -4,6 +4,7 @@
import logging
import multiprocessing
import re
+import string
import sys
import colander as col
@@ -165,6 +166,7 @@
shortname = p.shortname or p.name.shortname
project = M.Project.query.get(shortname=shortname,
neighborhood_id=nbhd._id)
+ project_template = nbhd.get_project_template()
if project and not (options.update and p.shortname):
log.warning('[%s] Skipping existing project "%s". To update an existing '
@@ -186,6 +188,22 @@
log.info('[%s] Updating project "%s".' % (worker_name, shortname))
project.notifications_disabled = True
+
+ if options.ensure_tools and 'tools' in project_template:
+ for i, tool in enumerate(project_template['tools'].iterkeys()):
+ tool_config = project_template['tools'][tool]
+ if project.app_instance(tool_config['mount_point']):
+ continue
+ tool_options = tool_config.get('options', {})
+ for k, v in tool_options.iteritems():
+ if isinstance(v, basestring):
+ tool_options[k] = string.Template(v).safe_substitute(
+ project.root_project.__dict__.get('root_project', {}))
+ project.install_app(tool,
+ mount_label=tool_config['label'],
+ mount_point=tool_config['mount_point'],
+ **tool_options)
+
project.summary = p.summary
project.short_description = p.description
project.external_homepage = p.external_homepage
@@ -267,6 +285,10 @@
action='store_true',
help='Update existing projects. Without this option, existing '
'projects will be skipped.')
+ parser.add_argument('--ensure-tools', dest='ensure_tools', default=False,
+ action='store_true',
+ help='Check nbhd project template for default tools, and install '
+ 'them on the project(s) if not already installed.')
parser.add_argument('--nprocs', '-n', action='store', dest='nprocs', type=int,
help='Number of processes to divide the work among.',
default=multiprocessing.cpu_count())
