1 # Copyright (C) 2009-2012 Chris Ball <cjb@laptop.org>
2 # Valtteri Kokkoniemi <rvk@iki.fi>
3 # W. Trevor King <wking@tremily.us>
5 # This file is part of Bugs Everywhere.
7 # Bugs Everywhere is free software: you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License as published by the Free
9 # Software Foundation, either version 2 of the License, or (at your option) any
12 # Bugs Everywhere is distributed in the hope that it will be useful, but
13 # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 # You should have received a copy of the GNU General Public License along with
18 # Bugs Everywhere. If not, see <http://www.gnu.org/licenses/>.
23 try: # import core module, Python >= 2.5
24 from xml.etree import ElementTree
25 except ImportError: # look for non-core module
26 from elementtree import ElementTree
32 import libbe.command.util
34 import libbe.util.encoding
36 import libbe.util.utility
38 if libbe.TESTING == True:
46 class Import_XML (libbe.command.Command):
47 """Import comments and bugs from XML
51 >>> import libbe.bugdir
52 >>> bd = libbe.bugdir.SimpleBugDir(memory=False)
53 >>> io = libbe.command.StringInputOutput()
54 >>> io.stdout = sys.stdout
55 >>> ui = libbe.command.UserInterface(io=io)
56 >>> ui.storage_callbacks.set_storage(bd.storage)
57 >>> cmd = Import_XML(ui=ui)
59 >>> ui.io.set_stdin('<be-xml><comment><uuid>c</uuid><body>This is a comment about a</body></comment></be-xml>')
60 >>> ret = ui.run(cmd, {'root':'/a'}, ['-'])
62 >>> bug = bd.bug_from_uuid('a')
63 >>> bug.load_comments(load_full=False)
64 >>> comment = bug.comment_root[0]
65 >>> print comment.body
66 This is a comment about a
68 >>> comment.time <= int(time.time())
70 >>> comment.in_reply_to is None
77 def __init__(self, *args, **kwargs):
78 libbe.command.Command.__init__(self, *args, **kwargs)
80 libbe.command.Option(name='ignore-missing-references', short_name='i',
81 help="If any comment's <in-reply-to> refers to a non-existent comment, ignore it (instead of raising an exception)."),
82 libbe.command.Option(name='add-only', short_name='a',
83 help='If any bug or comment listed in the XML file already exists in the bug repository, do not alter the repository version.'),
84 libbe.command.Option(name='preserve-uuids', short_name='p',
85 help='Preserve UUIDs for trusted input (potential name collisions).'),
86 libbe.command.Option(name='root', short_name='r',
87 help='Supply a bugdir, bug, or comment ID as the root of '
88 'any non-bugdir elements that are direct children of the '
89 '<be-xml> element. If any such elements exist, you are '
90 'required to set this option.',
91 arg=libbe.command.Argument(
92 name='root', metavar='ID',
93 completion_callback=libbe.command.util.complete_bug_comment_id)),
96 libbe.command.Argument(
97 name='xml-file', metavar='XML-FILE'),
100 def _run(self, **params):
101 storage = self._get_storage()
102 bugdirs = self._get_bugdirs()
103 writeable = storage.writeable
104 storage.writeable = False
105 if params['root'] != None:
106 root_bugdir,root_bug,root_comment = (
107 libbe.command.util.bugdir_bug_comment_from_user_id(
108 bugdirs, params['root']))
110 root_bugdir,root_bug,root_comment = (None, None, None)
112 xml = self._read_xml(storage, params)
113 version,root_bugdirs,root_bugs,root_comments = self._parse_xml(
116 if params['add-only']:
117 accept_changes = False
118 accept_extra_strings = False
120 accept_changes = True
121 accept_extra_strings = True
123 dirty_items = list(self._merge_comments(
124 bugdirs, root_bug, root_comment, root_comments,
125 params, accept_changes, accept_extra_strings))
126 dirty_items.extend(self._merge_bugs(
127 bugdirs, root_bugdir, root_bugs,
128 params, accept_changes, accept_extra_strings))
129 dirty_items.extend(self._merge_bugdirs(
130 bugdirs, root_bugdirs,
131 params, accept_changes, accept_extra_strings))
133 # protect against programmer error causing data loss:
134 if root_bug is not None:
135 # check for each of the new comments
137 for c in root_bug.comments():
140 comms.append(c.alt_id)
141 if root_comment.uuid == libbe.comment.INVALID_UUID:
142 root_text = root_bug.id.user()
144 root_text = root_comment.id.user()
145 for new in root_comments:
146 assert new.uuid in comms or new.alt_id in comms, \
147 "comment %s (alt: %s) wasn't added to %s" \
148 % (new.uuid, new.alt_id, root_text)
149 for new in root_bugs:
150 # check for each of the new bugs
152 libbe.command.util.bug_from_uuid(bugdirs, new.uuid)
153 except libbe.bugdir.NoBugMatches:
155 libbe.command.util.bug_from_uuid(bugdirs, new.alt_id)
156 except libbe.bugdir.NoBugMatches:
157 raise AssertionError(
158 "bug {} (alt: {}) wasn't added to {}".format(
159 new.uuid, new.alt_id, root_bugdir.id.user()))
160 for new in root_bugdirs:
161 assert new.uuid in bugdirs or new.alt_id in bugdirs, (
162 "bugdir {} wasn't added to {}".format(
163 new.uuid, sorted(bugdirs.keys())))
165 # save new information
166 storage.writeable = writeable
167 for item in dirty_items:
170 def _read_xml(self, storage, params):
171 if params['xml-file'] == '-':
172 return self.stdin.read().encode(self.stdin.encoding)
174 self._check_restricted_access(storage, params['xml-file'])
175 return libbe.util.encoding.get_file_contents(params['xml-file'])
177 def _parse_xml(self, xml, params):
182 be_xml = ElementTree.XML(xml)
183 if be_xml.tag != 'be-xml':
184 raise libbe.util.utility.InvalidXML(
185 'import-xml', be_xml, 'root element must be <be-xml>')
186 for child in be_xml.getchildren():
187 if child.tag == 'bugdir':
188 new = libbe.bugdir.BugDir(storage=None)
189 new.from_xml(child, preserve_uuids=params['preserve-uuids'])
190 root_bugdirs.append(new)
191 elif child.tag == 'bug':
192 new = libbe.bug.Bug()
193 new.from_xml(child, preserve_uuids=params['preserve-uuids'])
194 root_bugs.append(new)
195 elif child.tag == 'comment':
196 new = libbe.comment.Comment()
197 new.from_xml(child, preserve_uuids=params['preserve-uuids'])
198 root_comments.append(new)
199 elif child.tag == 'version':
200 for gchild in child.getchildren():
201 if child.tag in ['tag', 'nick', 'revision', 'revision-id']:
202 text = xml.sax.saxutils.unescape(child.text)
203 text = text.decode('unicode_escape').strip()
204 version[child.tag] = text
207 'ignoring unknown tag {} in {}\n'.format(
208 gchild.tag, child.tag))
210 sys.stderr.write('ignoring unknown tag {} in {}\n'.format(
211 child.tag, be_xml.tag))
212 return (version, root_bugdirs, root_bugs, root_comments)
214 def _merge_comments(self, bugdirs, bug, root_comment, comments,
215 params, accept_changes, accept_extra_strings,
216 accept_comments=True):
217 if len(comments) == 0:
220 raise libbe.command.UserError(
221 'No root bug for merging comments:\n{}'.format(
222 '\n\n'.join([c.string() for c in comments])))
223 bug.load_comments(load_full=True)
224 if root_comment.uuid == libbe.comment.INVALID_UUID:
225 root_comment = bug.comment_root
227 root_comment = bug.comment_from_uuid(root_comment.uuid)
228 new_bug = libbe.bug.Bug(bugdir=bug.bugdir, uuid=bug.uuid)
229 new_bug.explicit_attrs = []
230 new_bug.comment_root = copy.deepcopy(bug.comment_root)
231 if root_comment.uuid == libbe.comment.INVALID_UUID:
232 new_root_comment = new_bug.comment_root
234 new_root_comment = new_bug.comment_from_uuid(
236 for new in new_bug.comments():
237 new.explicit_attrs = []
239 new_bug.add_comments(
241 default_parent=root_comment,
242 ignore_missing_references=params['ignore-missing-references'])
243 except libbe.comment.MissingReference as e:
244 raise libbe.command.UserError(e)
245 bug.merge(new_bug, accept_changes=accept_changes,
246 accept_extra_strings=accept_extra_strings,
247 accept_comments=accept_comments)
250 def _merge_bugs(self, bugdirs, bugdir, bugs,
251 params, accept_changes, accept_extra_strings,
252 accept_comments=True):
255 old = bugdir.bug_from_uuid(new.alt_id)
257 bugdir.append(new, update=True)
260 old.load_comments(load_full=True)
261 old.merge(new, accept_changes=accept_changes,
262 accept_extra_strings=accept_extra_strings,
263 accept_comments=accept_comments)
266 def _merge_bugdirs(self, bugdirs, new_bugdirs,
267 params, accept_changes, accept_extra_strings,
268 accept_comments=True):
269 for new in new_bugdirs:
270 if new.alt_id in bugdirs:
271 old = bugdirs[new.alt_id]
273 old.merge(new, accept_changes=accept_changes,
274 accept_extra_strings=accept_extra_strings,
276 accept_comments=accept_comments)
279 bugdirs[new.uuid] = new
280 new.storage = self._get_storage()
283 def _long_help(self):
285 Import comments and bugs from XMLFILE. If XMLFILE is '-', the file is
288 This command provides a fallback mechanism for passing bugs between
289 repositories, in case the repositories VCSs are incompatible. If the
290 VCSs are compatible, it's better to use their builtin merge/push/pull
291 to share this information, as that will preserve a more detailed
294 The XML file should be formatted similarly to:
299 <branch-nick>be</branch-nick>
301 <revision-id>a@b.com-20091119214553-iqyw2cpqluww3zna</revision-id>
306 <comment>...</comment>
307 <comment>...</comment>
313 <comment>...</comment>
314 <comment>...</comment>
317 where the ellipses mark output commpatible with BugDir.xml(),
318 Bug.xml(), and Comment.xml(). Take a look at the output of `be show
319 --xml` for some explicit examples. Unrecognized tags are ignored.
320 Missing tags are left at the default value. The version tag is not
321 required, but is strongly recommended.
323 The bugdir, bug, and comment UUIDs are always auto-generated, so if
324 you set a <uuid> field, but no <alt-id> field, your <uuid> will be
325 used as the object's <alt-id>. An exception is raised if <alt-id>
326 conflicts with an existing object. Bugdirs and bugs do not have a
327 permantent alt-id, so they the <uuid>s you specify are not saved. The
328 <uuid>s _are_ used to match agains prexisting bug and comment uuids,
329 and comment alt-ids, and fields explicitly given in the XML file will
330 replace old versions unless the --add-only flag.
332 *.extra_strings recieves special treatment, and if --add-only is not
333 set, the resulting list concatenates both source lists and removes
336 Here's an example of import activity:
339 bug (uuid=B, creator=John, status=open)
340 estr (don't forget your towel)
341 estr (helps with space travel)
342 com (uuid=C1, author=Jane, body=Hello)
343 com (uuid=C2, author=Jess, body=World)
346 bug (uuid=B, status=fixed)
347 estr (don't forget your towel)
348 estr (watch out for flying dolphins)
349 com (uuid=C1, body=So long)
350 com (uuid=C3, author=Jed, body=And thanks)
353 bug (uuid=B, creator=John, status=fixed)
354 estr (don't forget your towel)
355 estr (helps with space travel)
356 estr (watch out for flying dolphins)
357 com (uuid=C1, author=Jane, body=So long)
358 com (uuid=C2, author=Jess, body=World)
359 com (uuid=C4, alt-id=C3, author=Jed, body=And thanks)
360 Result, with --add-only
362 bug (uuid=B, creator=John, status=open)
363 estr (don't forget your towel)
364 estr (helps with space travel)
365 com (uuid=C1, author=Jane, body=Hello)
366 com (uuid=C2, author=Jess, body=World)
367 com (uuid=C4, alt-id=C3, author=Jed, body=And thanks)
371 Import comments (e.g. emails from a mailbox) and append to bug /XYZ:
373 $ be-mail-to-xml mail.mbox | be import-xml -r /XYZ -
375 Or you can append those emails underneath the prexisting comment /XYZ/3:
377 $ be-mail-to-xml mail.mbox | be import-xml -r /XYZ/3 -
379 User creates a new bug:
381 user$ be new "The demuxulizer is broken"
382 Created bug with ID 48f
387 User exports bug as xml and emails it to the developers:
389 user$ be show --xml 48f > 48f.xml
390 user$ cat 48f.xml | mail -s "Demuxulizer bug xml" devs@b.com
391 or equivalently (with a slightly fancier be-handle-mail compatible
393 user$ be email-bugs 48f
395 Devs recieve email, and save it's contents as demux-bug.xml:
397 dev$ cat demux-bug.xml | be import-xml -
401 Import_xml = Import_XML # alias for libbe.command.base.get_command_class()
403 if libbe.TESTING == True:
404 class LonghelpTestCase (unittest.TestCase):
406 Test import scenarios given in longhelp.
409 self.bugdir = libbe.bugdir.SimpleBugDir(memory=False)
410 io = libbe.command.StringInputOutput()
411 self.ui = libbe.command.UserInterface(io=io)
412 self.ui.storage_callbacks.set_storage(self.bugdir.storage)
413 self.cmd = Import_XML(ui=self.ui)
414 self.cmd._storage = self.bugdir.storage
415 self.cmd._setup_io = lambda i_enc,o_enc : None
416 bugA = self.bugdir.bug_from_uuid('a')
417 self.bugdir.remove_bug(bugA)
418 self.bugdir.storage.writeable = False
419 bugB = self.bugdir.bug_from_uuid('b')
420 bugB.creator = 'John'
422 bugB.extra_strings += ["don't forget your towel"]
423 bugB.extra_strings += ['helps with space travel']
424 comm1 = bugB.comment_root.new_reply(body='Hello\n')
426 comm1.author = 'Jane'
427 comm2 = bugB.comment_root.new_reply(body='World\n')
429 comm2.author = 'Jess'
430 self.bugdir.storage.writeable = True
438 <status>fixed</status>
439 <summary>a test bug</summary>
440 <extra-string>don't forget your towel</extra-string>
441 <extra-string>watch out for flying dolphins</extra-string>
449 <body>And thanks</body>
455 self.root_comment_xml = """
464 <body>And thanks</body>
469 self.bugdir.cleanup()
471 def _execute(self, xml, params={}, args=[]):
472 self.ui.io.set_stdin(xml)
473 self.ui.run(self.cmd, params, args)
474 self.bugdir.flush_reload()
475 def testCleanBugdir(self):
476 uuids = list(self.bugdir.uuids())
477 self.failUnless(uuids == ['b'], uuids)
478 def testNotAddOnly(self):
479 bugB = self.bugdir.bug_from_uuid('b')
480 self._execute(self.xml, {}, ['-'])
481 uuids = list(self.bugdir.uuids())
482 self.failUnless(uuids == ['b'], uuids)
483 bugB = self.bugdir.bug_from_uuid('b')
484 self.failUnless(bugB.uuid == 'b', bugB.uuid)
485 self.failUnless(bugB.creator == 'John', bugB.creator)
486 self.failUnless(bugB.status == 'fixed', bugB.status)
487 self.failUnless(bugB.summary == 'a test bug', bugB.summary)
488 estrs = ["don't forget your towel",
489 'helps with space travel',
490 'watch out for flying dolphins']
491 self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
492 comments = list(bugB.comments())
493 self.failUnless(len(comments) == 3,
494 ['%s (%s, %s)' % (c.uuid, c.alt_id, c.body)
496 c1 = bugB.comment_from_uuid('c1')
498 self.failUnless(c1.uuid == 'c1', c1.uuid)
499 self.failUnless(c1.alt_id == None, c1.alt_id)
500 self.failUnless(c1.author == 'Jane', c1.author)
501 self.failUnless(c1.body == 'So long\n', c1.body)
502 c2 = bugB.comment_from_uuid('c2')
504 self.failUnless(c2.uuid == 'c2', c2.uuid)
505 self.failUnless(c2.alt_id == None, c2.alt_id)
506 self.failUnless(c2.author == 'Jess', c2.author)
507 self.failUnless(c2.body == 'World\n', c2.body)
509 self.failUnless(len(c4.uuid) == 36, c4.uuid)
510 self.failUnless(c4.alt_id == 'c3', c4.alt_id)
511 self.failUnless(c4.author == 'Jed', c4.author)
512 self.failUnless(c4.body == 'And thanks\n', c4.body)
513 def testAddOnly(self):
514 bugB = self.bugdir.bug_from_uuid('b')
515 initial_bugB_summary = bugB.summary
516 self._execute(self.xml, {'add-only':True}, ['-'])
517 uuids = list(self.bugdir.uuids())
518 self.failUnless(uuids == ['b'], uuids)
519 bugB = self.bugdir.bug_from_uuid('b')
520 self.failUnless(bugB.uuid == 'b', bugB.uuid)
521 self.failUnless(bugB.creator == 'John', bugB.creator)
522 self.failUnless(bugB.status == 'open', bugB.status)
523 self.failUnless(bugB.summary == initial_bugB_summary, bugB.summary)
524 estrs = ["don't forget your towel",
525 'helps with space travel']
526 self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
527 comments = list(bugB.comments())
528 self.failUnless(len(comments) == 3,
529 ['%s (%s)' % (c.uuid, c.alt_id) for c in comments])
530 c1 = bugB.comment_from_uuid('c1')
532 self.failUnless(c1.uuid == 'c1', c1.uuid)
533 self.failUnless(c1.alt_id == None, c1.alt_id)
534 self.failUnless(c1.author == 'Jane', c1.author)
535 self.failUnless(c1.body == 'Hello\n', c1.body)
536 c2 = bugB.comment_from_uuid('c2')
538 self.failUnless(c2.uuid == 'c2', c2.uuid)
539 self.failUnless(c2.alt_id == None, c2.alt_id)
540 self.failUnless(c2.author == 'Jess', c2.author)
541 self.failUnless(c2.body == 'World\n', c2.body)
543 self.failUnless(len(c4.uuid) == 36, c4.uuid)
544 self.failUnless(c4.alt_id == 'c3', c4.alt_id)
545 self.failUnless(c4.author == 'Jed', c4.author)
546 self.failUnless(c4.body == 'And thanks\n', c4.body)
547 def testRootCommentsNotAddOnly(self):
548 bugB = self.bugdir.bug_from_uuid('b')
549 initial_bugB_summary = bugB.summary
550 self._execute(self.root_comment_xml, {'root':'/b'}, ['-'])
551 uuids = list(self.bugdir.uuids())
552 uuids = list(self.bugdir.uuids())
553 self.failUnless(uuids == ['b'], uuids)
554 bugB = self.bugdir.bug_from_uuid('b')
555 self.failUnless(bugB.uuid == 'b', bugB.uuid)
556 self.failUnless(bugB.creator == 'John', bugB.creator)
557 self.failUnless(bugB.status == 'open', bugB.status)
558 self.failUnless(bugB.summary == initial_bugB_summary, bugB.summary)
559 estrs = ["don't forget your towel",
560 'helps with space travel']
561 self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
562 comments = list(bugB.comments())
563 self.failUnless(len(comments) == 3,
564 ['%s (%s, %s)' % (c.uuid, c.alt_id, c.body)
566 c1 = bugB.comment_from_uuid('c1')
568 self.failUnless(c1.uuid == 'c1', c1.uuid)
569 self.failUnless(c1.alt_id == None, c1.alt_id)
570 self.failUnless(c1.author == 'Jane', c1.author)
571 self.failUnless(c1.body == 'So long\n', c1.body)
572 c2 = bugB.comment_from_uuid('c2')
574 self.failUnless(c2.uuid == 'c2', c2.uuid)
575 self.failUnless(c2.alt_id == None, c2.alt_id)
576 self.failUnless(c2.author == 'Jess', c2.author)
577 self.failUnless(c2.body == 'World\n', c2.body)
579 self.failUnless(len(c4.uuid) == 36, c4.uuid)
580 self.failUnless(c4.alt_id == 'c3', c4.alt_id)
581 self.failUnless(c4.author == 'Jed', c4.author)
582 self.failUnless(c4.body == 'And thanks\n', c4.body)
583 def testRootCommentsAddOnly(self):
584 bugB = self.bugdir.bug_from_uuid('b')
585 initial_bugB_summary = bugB.summary
586 self._execute(self.root_comment_xml,
587 {'root':'/b', 'add-only':True}, ['-'])
588 uuids = list(self.bugdir.uuids())
589 self.failUnless(uuids == ['b'], uuids)
590 bugB = self.bugdir.bug_from_uuid('b')
591 self.failUnless(bugB.uuid == 'b', bugB.uuid)
592 self.failUnless(bugB.creator == 'John', bugB.creator)
593 self.failUnless(bugB.status == 'open', bugB.status)
594 self.failUnless(bugB.summary == initial_bugB_summary, bugB.summary)
595 estrs = ["don't forget your towel",
596 'helps with space travel']
597 self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
598 comments = list(bugB.comments())
599 self.failUnless(len(comments) == 3,
600 ['%s (%s)' % (c.uuid, c.alt_id) for c in comments])
601 c1 = bugB.comment_from_uuid('c1')
603 self.failUnless(c1.uuid == 'c1', c1.uuid)
604 self.failUnless(c1.alt_id == None, c1.alt_id)
605 self.failUnless(c1.author == 'Jane', c1.author)
606 self.failUnless(c1.body == 'Hello\n', c1.body)
607 c2 = bugB.comment_from_uuid('c2')
609 self.failUnless(c2.uuid == 'c2', c2.uuid)
610 self.failUnless(c2.alt_id == None, c2.alt_id)
611 self.failUnless(c2.author == 'Jess', c2.author)
612 self.failUnless(c2.body == 'World\n', c2.body)
614 self.failUnless(len(c4.uuid) == 36, c4.uuid)
615 self.failUnless(c4.alt_id == 'c3', c4.alt_id)
616 self.failUnless(c4.author == 'Jed', c4.author)
617 self.failUnless(c4.body == 'And thanks\n', c4.body)
619 unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
620 suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()])