Ubuntu Developer Portal

Merge lp:~developer-ubuntu-com-dev/developer-ubuntu-com/importer-versioning into lp:developer-ubuntu-com

  1. importer-versioning
  2. Merge into stable
Proposed by Daniel Holbach
Status: Needs review
Proposed branch: lp:~developer-ubuntu-com-dev/developer-ubuntu-com/importer-versioning
Merge into: lp:developer-ubuntu-com
Diff against target: 552 lines (+300/-30)
10 files modified
md_importer/importer/article.py (+4/-7)
md_importer/importer/process.py (+49/-5)
md_importer/importer/publish.py (+19/-5)
md_importer/importer/repo.py (+19/-10)
md_importer/migrations/0005_add_versioning.py (+41/-0)
md_importer/models.py (+37/-0)
md_importer/tests/test_process.py (+95/-0)
md_importer/tests/test_publish.py (+20/-1)
md_importer/tests/utils.py (+2/-2)
templates/cms_page_base.html (+14/-0)
To merge this branch: bzr merge lp:~developer-ubuntu-com-dev/developer-ubuntu-com/importer-versioning
Reviewer Review Type Date Requested Status
Ubuntu App Developer site developers Pending
Review via email: mp+294517@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Daniel Holbach (dholbach) wrote :

What's still needed: make the version picker look nice.

Reply
lp:~developer-ubuntu-com-dev/developer-ubuntu-com/importer-versioning updated
237. By Daniel Holbach

fix dissociation of version information

238. By Daniel Holbach

fix urls to not include 'None'

Unmerged revisions

238. By Daniel Holbach

fix urls to not include 'None'

237. By Daniel Holbach

fix dissociation of version information

236. By Daniel Holbach

merge trunk

235. By Daniel Holbach

exclude current version

234. By Daniel Holbach

make sure we add the alternative url info

233. By Daniel Holbach

fix template

232. By Daniel Holbach

first cut at adding version/url info to page template

231. By Daniel Holbach

fix logic

230. By Daniel Holbach

clarify tests, add a new test to check the extension data of the page - currently fails

229. By Daniel Holbach

simplify code and make version extension handling actually work

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'md_importer/importer/article.py'
2--- md_importer/importer/article.py 2016-04-21 12:26:42 +0000
3+++ md_importer/importer/article.py 2016-05-14 20:13:14 +0000
4@@ -6,7 +6,6 @@
5 import sys
6
7 from . import (
8- DEFAULT_LANG,
9 DEFAULT_TEMPLATE,
10 logger,
11 MARKDOWN_EXTENSIONS,
12@@ -14,6 +13,7 @@
13 )
14 from .publish import (
15 ArticlePage,
16+ clean_url,
17 ParentNotFoundException,
18 slugify,
19 )
20@@ -25,14 +25,13 @@
21
22
23 class Article:
24- def __init__(self, fn, write_to, advertise=True,
25+ def __init__(self, fn, full_url, advertise=True,
26 template=DEFAULT_TEMPLATE):
27 self.html = None
28 self.article_page = None
29 self.title = ""
30 self.fn = fn
31- self.write_to = slugify(self.fn)
32- self.full_url = write_to
33+ self.full_url = full_url
34 self.slug = os.path.basename(self.full_url)
35 self.links_rewritten = False
36 self.local_images = []
37@@ -129,9 +128,7 @@
38 in_navigation=self.advertise, template=self.template)
39 except ParentNotFoundException:
40 return False
41- self.full_url = re.sub(
42- r'^\/None\/', '/{}/'.format(DEFAULT_LANG),
43- self.article_page.draft.get_absolute_url())
44+ self.full_url = clean_url(self.article_page.draft.get_absolute_url())
45 return True
46
47 def publish(self):
48
49=== modified file 'md_importer/importer/process.py'
50--- md_importer/importer/process.py 2016-04-27 15:14:30 +0000
51+++ md_importer/importer/process.py 2016-05-14 20:13:14 +0000
52@@ -3,21 +3,27 @@
53 import shutil
54 import tempfile
55
56-from django.core.management import call_command
57-
58-from md_importer.importer import logger
59-from md_importer.importer.publish import find_page
60+from md_importer.importer import (
61+ DEFAULT_LANG,
62+ logger,
63+)
64+from md_importer.importer.publish import (
65+ find_page,
66+ remove_none_from_url,
67+)
68 from md_importer.importer.repo import Repo
69 from md_importer.models import (
70+ ExternalDocsBranch,
71 ExternalDocsBranchImportDirective,
72 ImportedArticle,
73+ VersionExtension,
74 )
75
76
77 def process_branch(branch):
78 tempdir = tempfile.mkdtemp()
79 repo = Repo(tempdir, branch.origin, branch.branch_name,
80- branch.post_checkout_command)
81+ branch.post_checkout_command, branch.version)
82 if repo.get() != 0:
83 return None
84 for directive in ExternalDocsBranchImportDirective.objects.filter(
85@@ -46,6 +52,43 @@
86 url=url,
87 last_import=timestamp)
88
89+ if repo.version:
90+ # Updates pages with information about other versions of the article
91+ ebs = ExternalDocsBranch.objects.filter(origin=repo.origin)
92+ available_versions = [a.version for a in ebs]
93+ urls = [remove_none_from_url(i.url)
94+ for i in ImportedArticle.objects.all()]
95+
96+ for page in repo.pages:
97+ url = page.get_absolute_url()
98+ alternative_urls = [
99+ {
100+ 'version': v,
101+ 'url': remove_none_from_url(url.replace(repo.version, v)),
102+ }
103+ for v in available_versions if v != repo.version
104+ ]
105+ # Check which of the alternative versions of the article
106+ # actually exist
107+ for useless_entry in [x for x in alternative_urls
108+ if x['url'] not in urls]:
109+ alternative_urls.remove(useless_entry)
110+ draft = page.get_draft_object()
111+ if not hasattr(draft, 'versionextension'):
112+ version_ext = VersionExtension(extended_object=draft)
113+ version_ext.save()
114+ draft.versionextension = version_ext
115+ if draft.versionextension.version != repo.version:
116+ draft.versionextension.version = repo.version
117+ draft.versionextension.update_alternatives(alternative_urls)
118+ draft.versionextension.save()
119+ draft.publish(DEFAULT_LANG)
120+ else:
121+ for page in repo.pages:
122+ if hasattr(page, 'versionextension'):
123+ page.versionextension.delete()
124+ page.publish(DEFAULT_LANG)
125+
126 # Remove old entries
127 for imported_article in ImportedArticle.objects.filter(
128 branch=branch, last_import__lt=timestamp):
129@@ -61,6 +104,7 @@
130 if p.changed_by in ['python-api', 'script']]
131 ImportedArticle.objects.filter(
132 branch=branch).filter(id__in=imported_page_ids).delete()
133+
134 shutil.rmtree(tempdir)
135
136 return repo
137
138=== modified file 'md_importer/importer/publish.py'
139--- md_importer/importer/publish.py 2016-04-21 12:26:42 +0000
140+++ md_importer/importer/publish.py 2016-05-14 20:13:14 +0000
141@@ -134,12 +134,26 @@
142 return os.path.basename(filename).replace('.md', '').replace('.html', '')
143
144
145+def remove_none_from_url(url):
146+ return re.sub(
147+ r'^\/None|{}\/'.format(DEFAULT_LANG),
148+ '',
149+ url)
150+
151+
152 def clean_url(url):
153- return remove_leading_and_trailing_slash(
154- re.sub(
155- r'^\/None|{}\/'.format(DEFAULT_LANG),
156- '',
157- url))
158+ return remove_leading_and_trailing_slash(remove_none_from_url(url))
159+
160+
161+def generate_url(import_to, article_fn, version=None):
162+ if not version:
163+ if not article_fn:
164+ # Unversioned index page
165+ return clean_url(import_to)
166+ # Unversioned 'regular' article
167+ return clean_url(os.path.join(import_to, slugify(article_fn)))
168+ # Versioned article
169+ return clean_url(os.path.join(import_to, version, slugify(article_fn)))
170
171
172 def find_page(url, draft=False):
173
174=== modified file 'md_importer/importer/repo.py'
175--- md_importer/importer/repo.py 2016-04-21 12:26:42 +0000
176+++ md_importer/importer/repo.py 2016-05-14 20:13:14 +0000
177@@ -5,9 +5,9 @@
178 )
179 from .article import Article
180 from .publish import (
181+ generate_url,
182 IndexPage,
183 ParentNotFoundException,
184- slugify,
185 )
186 from .source import SourceCode
187
188@@ -18,7 +18,8 @@
189
190
191 class Repo:
192- def __init__(self, tempdir, origin, branch_name, post_checkout_command):
193+ def __init__(self, tempdir, origin, branch_name,
194+ post_checkout_command=None, version=None):
195 self.directives = []
196 self.imported_articles = []
197 self.url_map = {}
198@@ -31,9 +32,13 @@
199 self.origin = origin
200 self.branch_name = branch_name
201 self.post_checkout_command = post_checkout_command
202+ self.version = version
203 self.branch_nick = os.path.basename(self.origin.replace('.git', ''))
204 self.checkout_location = os.path.join(tempdir, self.branch_nick)
205
206+ def __str__(self):
207+ return '{} -- {}'.format(self.origin, self.branch_name)
208+
209 def get(self):
210 sourcecode = SourceCode(self.origin, self.checkout_location,
211 self.branch_name, self.post_checkout_command)
212@@ -79,29 +84,33 @@
213 for fn in glob.glob('{}/*'.format(directive['import_from'])):
214 if fn not in [a[0] for a in import_list]:
215 import_list += [
216- (fn, os.path.join(directive['write_to'], slugify(fn)),
217+ (fn,
218+ generate_url(directive['write_to'], fn, self.version),
219 directive['advertise'], directive['template'])
220 ]
221 # If we import into a namespace and don't have an index doc,
222 # we need to write one.
223 # XXX: Right now we just create one index doc, this will change in
224 # the near future.
225- if directive['write_to'] not in [x[1] for x in import_list] and \
226+ index_url = generate_url(directive['write_to'], '', self.version)
227+ if index_url not in [x[1] for x in import_list] and \
228 not self.index_page:
229 try:
230 self.index_page = IndexPage(
231- title=self.branch_nick, full_url=directive['write_to'],
232+ title=self.branch_nick, full_url=index_url,
233 in_navigation=directive['advertise'], html='',
234 menu_title=None, template=DEFAULT_TEMPLATE)
235- self.pages.extend([self.index_page.page])
236 except ParentNotFoundException:
237 return self._abort_import(
238 'Could not create fake index page at {}'.format(
239- directive['write_to']))
240+ index_url))
241+ self.pages.extend([self.index_page.page])
242+
243 # The actual import
244 for entry in import_list:
245 article = self._read_article(
246- entry[0], entry[1], entry[2], entry[3])
247+ fn=entry[0], full_url=entry[1], advertise=entry[2],
248+ template=entry[3])
249 if article:
250 self.imported_articles += [article]
251 self.titles[article.fn] = article.title
252@@ -121,8 +130,8 @@
253 logger.error('Importing of {} aborted: {}.'.format(self.origin, msg))
254 return False
255
256- def _read_article(self, fn, write_to, advertise, template):
257- article = Article(fn, write_to, advertise, template)
258+ def _read_article(self, fn, full_url, advertise, template):
259+ article = Article(fn, full_url, advertise, template)
260 if article.read():
261 return article
262 return None
263
264=== added file 'md_importer/migrations/0005_add_versioning.py'
265--- md_importer/migrations/0005_add_versioning.py 1970-01-01 00:00:00 +0000
266+++ md_importer/migrations/0005_add_versioning.py 2016-05-14 20:13:14 +0000
267@@ -0,0 +1,41 @@
268+# -*- coding: utf-8 -*-
269+from __future__ import unicode_literals
270+
271+from django.db import migrations, models
272+
273+
274+class Migration(migrations.Migration):
275+
276+ dependencies = [
277+ ('cms', '0014_auto_20160404_1908'),
278+ ('md_importer', '0004_move_from_page_object_to_url_string'),
279+ ]
280+
281+ operations = [
282+ migrations.CreateModel(
283+ name='VersionedURL',
284+ fields=[
285+ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
286+ ('version', models.CharField(max_length=10)),
287+ ('url', models.URLField(max_length=1000)),
288+ ],
289+ ),
290+ migrations.CreateModel(
291+ name='VersionExtension',
292+ fields=[
293+ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
294+ ('version', models.CharField(max_length=10, blank=True)),
295+ ('alternative_urls', models.ManyToManyField(to='md_importer.VersionedURL', blank=True)),
296+ ('extended_object', models.OneToOneField(editable=False, to='cms.Page')),
297+ ('public_extension', models.OneToOneField(related_name='draft_extension', null=True, editable=False, to='md_importer.VersionExtension')),
298+ ],
299+ options={
300+ 'abstract': False,
301+ },
302+ ),
303+ migrations.AddField(
304+ model_name='externaldocsbranch',
305+ name='version',
306+ field=models.CharField(help_text='If this is a versioned import, set the version, e.g. "15.04" here.', max_length=10, blank=True),
307+ ),
308+ ]
309
310=== modified file 'md_importer/models.py'
311--- md_importer/models.py 2016-04-18 08:56:51 +0000
312+++ md_importer/models.py 2016-05-14 20:13:14 +0000
313@@ -2,6 +2,9 @@
314 from django.db import models
315 from django.utils.translation import ugettext_lazy as _
316
317+from cms.extensions import PageExtension
318+from cms.extensions.extension_pool import extension_pool
319+
320
321 if settings.CMS_TEMPLATES:
322 cms_templates = settings.CMS_TEMPLATES
323@@ -26,6 +29,11 @@
324 help_text=_('Command to run after checkout of the branch.'),
325 blank=True)
326 active = models.BooleanField(default=True)
327+ version = models.CharField(
328+ max_length=10,
329+ help_text=_('If this is a versioned import, set the version, e.g. '
330+ '"15.04" here.'),
331+ blank=True)
332
333 def __str__(self):
334 if self.branch_name:
335@@ -80,3 +88,32 @@
336 def __str__(self):
337 return '{} -- {} -- {}'.format(
338 self.url, self.branch, self.last_import)
339+
340+
341+class VersionedURL(models.Model):
342+ version = models.CharField(max_length=10)
343+ url = models.URLField(max_length=1000)
344+
345+
346+class VersionExtension(PageExtension):
347+ version = models.CharField(
348+ max_length=10,
349+ blank=True)
350+ alternative_urls = models.ManyToManyField(VersionedURL, blank=True)
351+
352+ def update_alternatives(self, alternative_urls):
353+ for alternative in alternative_urls:
354+ versioned_url, created = VersionedURL.objects.get_or_create(
355+ version=alternative['version'],
356+ url=alternative['url'])
357+ if versioned_url not in self.alternative_urls.all():
358+ self.alternative_urls.add(versioned_url)
359+ for listed_url in self.alternative_urls.all():
360+ if not [a for a in alternative_urls
361+ if a['url'] == listed_url.url]:
362+ self.alternative_urls.remove(listed_url)
363+
364+ def copy_relations(self, oldinstance, language):
365+ self.alternative_urls = oldinstance.alternative_urls.all()
366+
367+extension_pool.register(VersionExtension)
368
369=== renamed file 'md_importer/tests/test_import_process.py' => 'md_importer/tests/test_process.py'
370--- md_importer/tests/test_import_process.py 2016-04-18 10:30:14 +0000
371+++ md_importer/tests/test_process.py 2016-05-14 20:13:14 +0000
372@@ -1,4 +1,5 @@
373 import os
374+import re
375
376 from django.test import TestCase
377 from cms.api import publish_pages
378@@ -328,3 +329,97 @@
379 self.assertTrue(published_pages.has_at_least_size(20))
380 for imported_article in ImportedArticle.objects.all():
381 self.assertTrue(check_imported_article(imported_article))
382+
383+
384+class TestVersionedImportProcessPasses(TestCase):
385+ def runTest(self):
386+ db_empty_page_list()
387+ root = db_create_root_page()
388+ snappy_page = db_add_empty_page('Snappy', root)
389+ guides = db_add_empty_page('Guides', snappy_page)
390+ publish_pages([snappy_page, guides])
391+ ExternalDocsBranch.objects.all().delete()
392+ ExternalDocsBranchImportDirective.objects.all().delete()
393+ ImportedArticle.objects.all().delete()
394+ branch, created = ExternalDocsBranch.objects.get_or_create(
395+ origin=os.path.join(
396+ os.path.dirname(__file__), 'data/snappy-test'),
397+ branch_name='',
398+ version='16.04')
399+ a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
400+ import_from='docs', write_to='snappy/guides',
401+ external_docs_branch=branch)
402+ self.assertIsNotNone(process_branch(branch))
403+
404+
405+class TestTwoVersionedImportProcessPass(TestCase):
406+ def runTest(self):
407+ db_empty_page_list()
408+ root = db_create_root_page()
409+ snappy_page = db_add_empty_page('Snappy', root)
410+ guides = db_add_empty_page('Guides', snappy_page)
411+ publish_pages([snappy_page, guides])
412+ ExternalDocsBranch.objects.all().delete()
413+ ExternalDocsBranchImportDirective.objects.all().delete()
414+ ImportedArticle.objects.all().delete()
415+ branch, created = ExternalDocsBranch.objects.get_or_create(
416+ origin=os.path.join(
417+ os.path.dirname(__file__), 'data/snappy-test'),
418+ branch_name='',
419+ version='16.04')
420+ a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
421+ import_from='docs', write_to='snappy/guides',
422+ external_docs_branch=branch)
423+ self.assertIsNotNone(process_branch(branch))
424+ branch2, created = ExternalDocsBranch.objects.get_or_create(
425+ origin=os.path.join(
426+ os.path.dirname(__file__), 'data/snappy-test'),
427+ branch_name='',
428+ version='15.04')
429+ b, created = ExternalDocsBranchImportDirective.objects.get_or_create(
430+ import_from='docs', write_to='snappy/guides',
431+ external_docs_branch=branch2)
432+ self.assertIsNotNone(process_branch(branch2))
433+
434+
435+class TestTwoVersionedImportsValidExtensionData(TestCase):
436+ def runTest(self):
437+ db_empty_page_list()
438+ root = db_create_root_page()
439+ snappy_page = db_add_empty_page('Snappy', root)
440+ guides = db_add_empty_page('Guides', snappy_page)
441+ publish_pages([snappy_page, guides])
442+ ExternalDocsBranch.objects.all().delete()
443+ ExternalDocsBranchImportDirective.objects.all().delete()
444+ ImportedArticle.objects.all().delete()
445+ branch, created = ExternalDocsBranch.objects.get_or_create(
446+ origin=os.path.join(
447+ os.path.dirname(__file__), 'data/snappy-test'),
448+ branch_name='',
449+ version='16.04')
450+ a, created = ExternalDocsBranchImportDirective.objects.get_or_create(
451+ import_from='docs', write_to='snappy/guides',
452+ external_docs_branch=branch)
453+ self.assertIsNotNone(process_branch(branch))
454+ branch2, created = ExternalDocsBranch.objects.get_or_create(
455+ origin=os.path.join(
456+ os.path.dirname(__file__), 'data/snappy-test'),
457+ branch_name='',
458+ version='15.04')
459+ b, created = ExternalDocsBranchImportDirective.objects.get_or_create(
460+ import_from='docs', write_to='snappy/guides',
461+ external_docs_branch=branch2)
462+ self.assertIsNotNone(process_branch(branch2))
463+
464+ # Check if the REST article was imported into 15.04 and 16.04 namespace
465+ published_pages = PublishedPages()
466+ self.assertTrue(published_pages.contains_url(
467+ '/{}/snappy/guides/15.04/rest/'.format(DEFAULT_LANG)))
468+ self.assertTrue(published_pages.contains_url(
469+ '/{}/snappy/guides/16.04/rest/'.format(DEFAULT_LANG)))
470+
471+ for page in published_pages.pages:
472+ if re.findall(
473+ '\/{}\/snappy\/guides\/1\d\.04\/(\S+?)\/'.format(DEFAULT_LANG),
474+ page.get_absolute_url()):
475+ self.assertTrue(hasattr(page, 'versionextension'))
476
477=== modified file 'md_importer/tests/test_publish.py'
478--- md_importer/tests/test_publish.py 2016-04-21 10:39:58 +0000
479+++ md_importer/tests/test_publish.py 2016-05-14 20:13:14 +0000
480@@ -5,7 +5,10 @@
481 from developer_portal.models import RawHtml
482
483 from md_importer.importer import DEFAULT_LANG
484-from md_importer.importer.publish import ArticlePage
485+from md_importer.importer.publish import (
486+ ArticlePage,
487+ generate_url,
488+)
489
490 from .utils import (
491 db_empty_page_list,
492@@ -75,3 +78,19 @@
493 plugins = placeholder.get_plugins()
494 self.assertEqual(plugins.count(), 1)
495 self.assertIsInstance(plugins[0].get_plugin_instance()[0], RawHtml)
496+
497+
498+class TestGenerateURL(TestCase):
499+ def runTest(self):
500+ self.assertEqual(
501+ generate_url('snappy/guides', ''),
502+ 'snappy/guides')
503+ self.assertEqual(
504+ generate_url('snappy/guides', 'interfaces.md'),
505+ 'snappy/guides/interfaces')
506+ self.assertEqual(
507+ generate_url('snappy/guides', '', '16.04'),
508+ 'snappy/guides/16.04')
509+ self.assertEqual(
510+ generate_url('snappy/guides', 'interfaces.md', '16.04'),
511+ 'snappy/guides/16.04/interfaces')
512
513=== modified file 'md_importer/tests/utils.py'
514--- md_importer/tests/utils.py 2016-04-18 10:30:14 +0000
515+++ md_importer/tests/utils.py 2016-05-14 20:13:14 +0000
516@@ -52,10 +52,10 @@
517 self.root.get_absolute_url(),
518 '/{}/'.format(DEFAULT_LANG))
519
520- def create_repo(self, docs_path):
521+ def create_repo(self, docs_path, version=None):
522 origin = os.path.join(os.path.dirname(__file__), docs_path)
523 self.assertTrue(os.path.exists(origin))
524- self.repo = Repo(self.tempdir, origin, '', '')
525+ self.repo = Repo(self.tempdir, origin, '', '', version=version)
526 self.fetch_retcode = self.repo.get()
527 self.assertEqual(self.fetch_retcode, 0)
528
529
530=== modified file 'templates/cms_page_base.html'
531--- templates/cms_page_base.html 2015-10-21 14:41:23 +0000
532+++ templates/cms_page_base.html 2016-05-14 20:13:14 +0000
533@@ -9,5 +9,19 @@
534 {% endblock extrameta %}
535
536 {% block content %}
537+{% if request.current_page.versionextension %}
538+ <div>
539+ <p>Current version {{ request.current_page.versionextension.version }}</p>
540+ {% if request.current_page.versionextension.alternative_urls %}
541+ <p>Other versions:
542+ <ul>
543+ {% for alternative in request.current_page.versionextension.alternative_urls.all %}
544+ <li><a href="{{ alternative.url }}">{{ alternative.version }}</a></li>
545+ {% endfor %}
546+ </ul>
547+ </p>
548+ {% endif %}
549+ </div>
550+{% endif %}
551 {% placeholder page_content %}
552 {% endblock %}

Subscribers

People subscribed via source and target branches