a8906691edc3a228b868b9920ee914e9e6705a41
[be.git] / libbe / command / import_xml.py
1 # Copyright (C) 2009-2010 W. Trevor King <wking@drexel.edu>
2 #
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 2 of the License, or
6 # (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
11 # GNU General Public License for more details.
12 #
13 # You should have received a copy of the GNU General Public License along
14 # with this program; if not, write to the Free Software Foundation, Inc.,
15 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
16
17 import copy
18 import os
19 import sys
20 try: # import core module, Python >= 2.5
21     from xml.etree import ElementTree
22 except ImportError: # look for non-core module
23     from elementtree import ElementTree
24
25 import libbe
26 import libbe.bug
27 import libbe.command
28 import libbe.command.util
29 import libbe.comment
30 import libbe.util.encoding
31 import libbe.util.utility
32
33 if libbe.TESTING == True:
34     import doctest
35     import StringIO
36     import unittest
37
38     import libbe.bugdir
39
40 class Import_XML (libbe.command.Command):
41     """Import comments and bugs from XML
42
43     >>> import time
44     >>> import StringIO
45     >>> import libbe.bugdir
46     >>> bd = libbe.bugdir.SimpleBugDir(memory=False)
47     >>> io = libbe.command.StringInputOutput()
48     >>> io.stdout = sys.stdout
49     >>> ui = libbe.command.UserInterface(io=io)
50     >>> ui.storage_callbacks.set_storage(bd.storage)
51     >>> cmd = Import_XML(ui=ui)
52
53     >>> ui.io.set_stdin('<be-xml><comment><uuid>c</uuid><body>This is a comment about a</body></comment></be-xml>')
54     >>> ret = ui.run(cmd, {'comment-root':'/a'}, ['-'])
55     >>> bd.flush_reload()
56     >>> bug = bd.bug_from_uuid('a')
57     >>> bug.load_comments(load_full=False)
58     >>> comment = bug.comment_root[0]
59     >>> print comment.body
60     This is a comment about a
61     <BLANKLINE>
62     >>> comment.time <= int(time.time())
63     True
64     >>> comment.in_reply_to is None
65     True
66     >>> ui.cleanup()
67     >>> bd.cleanup()
68     """
69     name = 'import-xml'
70
71     def __init__(self, *args, **kwargs):
72         libbe.command.Command.__init__(self, *args, **kwargs)
73         self.options.extend([
74                 libbe.command.Option(name='ignore-missing-references', short_name='i',
75                     help="If any comment's <in-reply-to> refers to a non-existent comment, ignore it (instead of raising an exception)."),
76                 libbe.command.Option(name='add-only', short_name='a',
77                     help='If any bug or comment listed in the XML file already exists in the bug repository, do not alter the repository version.'),
78                 libbe.command.Option(name='comment-root', short_name='c',
79                     help='Supply a bug or comment ID as the root of any <comment> elements that are direct children of the <be-xml> element.  If any such <comment> elements exist, you are required to set this option.',
80                     arg=libbe.command.Argument(
81                         name='comment-root', metavar='ID',
82                         completion_callback=libbe.command.util.complete_bug_comment_id)),
83                 ])
84         self.args.extend([
85                 libbe.command.Argument(
86                     name='xml-file', metavar='XML-FILE'),
87                 ])
88
89     def _run(self, **params):
90         bugdir = self._get_bugdir()
91         writeable = bugdir.storage.writeable
92         bugdir.storage.writeable = False
93         if params['comment-root'] != None:
94             croot_bug,croot_comment = \
95                 libbe.command.util.bug_comment_from_user_id(
96                     bugdir, params['comment-root'])
97             croot_bug.load_comments(load_full=True)
98             if croot_comment.uuid == libbe.comment.INVALID_UUID:
99                 croot_comment = croot_bug.comment_root
100             else:
101                 croot_comment = croot_bug.comment_from_uuid(croot_comment.uuid)
102             new_croot_bug = libbe.bug.Bug(bugdir=bugdir, uuid=croot_bug.uuid)
103             new_croot_bug.explicit_attrs = []
104             new_croot_bug.comment_root = copy.deepcopy(croot_bug.comment_root)
105             if croot_comment.uuid == libbe.comment.INVALID_UUID:
106                 new_croot_comment = new_croot_bug.comment_root
107             else:
108                 new_croot_comment = \
109                     new_croot_bug.comment_from_uuid(croot_comment.uuid)
110             for new in new_croot_bug.comments():
111                 new.explicit_attrs = []
112         else:
113             croot_bug,croot_comment = (None, None)
114
115         if params['xml-file'] == '-':
116             xml = self.stdin.read().encode(self.stdin.encoding)
117         else:
118             self._check_restricted_access(bugdir.storage, params['xml-file'])
119             xml = libbe.util.encoding.get_file_contents(
120                 params['xml-file'])
121
122         # parse the xml
123         root_bugs = []
124         root_comments = []
125         version = {}
126         be_xml = ElementTree.XML(xml)
127         if be_xml.tag != 'be-xml':
128             raise libbe.util.utility.InvalidXML(
129                 'import-xml', be_xml, 'root element must be <be-xml>')
130         for child in be_xml.getchildren():
131             if child.tag == 'bug':
132                 new = libbe.bug.Bug(bugdir=bugdir)
133                 new.from_xml(child)
134                 root_bugs.append(new)
135             elif child.tag == 'comment':
136                 new = libbe.comment.Comment(croot_bug)
137                 new.from_xml(child)
138                 root_comments.append(new)
139             elif child.tag == 'version':
140                 for gchild in child.getchildren():
141                     if child.tag in ['tag', 'nick', 'revision', 'revision-id']:
142                         text = xml.sax.saxutils.unescape(child.text)
143                         text = text.decode('unicode_escape').strip()
144                         version[child.tag] = text
145                     else:
146                         print >> sys.stderr, 'ignoring unknown tag %s in %s' \
147                             % (gchild.tag, child.tag)
148             else:
149                 print >> sys.stderr, 'ignoring unknown tag %s in %s' \
150                     % (child.tag, comment_list.tag)
151
152         # merge the new root_comments
153         if params['add-only'] == True:
154             accept_changes = False
155             accept_extra_strings = False
156         else:
157             accept_changes = True
158             accept_extra_strings = True
159         accept_comments = True
160         if len(root_comments) > 0:
161             if croot_bug == None:
162                 raise libbe.command.UserError(
163                     '--comment-root option is required for your root comments:\n%s'
164                     % '\n\n'.join([c.string() for c in root_comments]))
165             try:
166                 # link new comments
167                 new_croot_bug.add_comments(root_comments,
168                                            default_parent=new_croot_comment,
169                                            ignore_missing_references= \
170                                                params['ignore-missing-references'])
171             except libbe.comment.MissingReference, e:
172                 raise libbe.command.UserError(e)
173             croot_bug.merge(new_croot_bug, accept_changes=accept_changes,
174                             accept_extra_strings=accept_extra_strings,
175                             accept_comments=accept_comments)
176
177         # merge the new croot_bugs
178         merged_bugs = []
179         old_bugs = []
180         for new in root_bugs:
181             try:
182                 old = bugdir.bug_from_uuid(new.alt_id)
183             except KeyError:
184                 old = None
185             if old == None:
186                 bd.append(new)
187             else:
188                 old.load_comments(load_full=True)
189                 old.merge(new, accept_changes=accept_changes,
190                           accept_extra_strings=accept_extra_strings,
191                           accept_comments=accept_comments)
192                 merged_bugs.append(new)
193                 old_bugs.append(old)
194
195         # protect against programmer error causing data loss:
196         if croot_bug != None:
197             comms = []
198             for c in croot_comment.traverse():
199                 comms.append(c.uuid)
200                 if c.alt_id != None:
201                     comms.append(c.alt_id)
202             if croot_comment.uuid == libbe.comment.INVALID_UUID:
203                 root_text = croot_bug.id.user()
204             else:
205                 root_text = croot_comment.id.user()
206             for new in root_comments:
207                 assert new.uuid in comms or new.alt_id in comms, \
208                     "comment %s (alt: %s) wasn't added to %s" \
209                     % (new.uuid, new.alt_id, root_text)
210         for new in root_bugs:
211             if not new in merged_bugs:
212                 assert bugdir.has_bug(new.uuid), \
213                     "bug %s wasn't added" % (new.uuid)
214
215         # save new information
216         bugdir.storage.writeable = writeable
217         if croot_bug != None:
218             croot_bug.save()
219         for new in root_bugs:
220             if not new in merged_bugs:
221                 new.save()
222         for old in old_bugs:
223             old.save()
224
225     def _long_help(self):
226         return """
227 Import comments and bugs from XMLFILE.  If XMLFILE is '-', the file is
228 read from stdin.
229
230 This command provides a fallback mechanism for passing bugs between
231 repositories, in case the repositories VCSs are incompatible.  If the
232 VCSs are compatible, it's better to use their builtin merge/push/pull
233 to share this information, as that will preserve a more detailed
234 history.
235
236 The XML file should be formatted similarly to
237   <be-xml>
238     <version>
239       <tag>1.0.0</tag>
240       <branch-nick>be</branch-nick>
241       <revno>446</revno>
242       <revision-id>a@b.com-20091119214553-iqyw2cpqluww3zna</revision-id>
243     <version>
244     <bug>
245       ...
246       <comment>...</comment>
247       <comment>...</comment>
248     </bug>
249     <bug>...</bug>
250     <comment>...</comment>
251     <comment>...</comment>
252   </be-xml>
253 where the ellipses mark output commpatible with Bug.xml() and
254 Comment.xml().  Take a look at the output of `be show --xml` for some
255 explicit examples.  Unrecognized tags are ignored.  Missing tags are
256 left at the default value.  The version tag is not required, but is
257 strongly recommended.
258
259 The bug and comment UUIDs are always auto-generated, so if you set a
260 <uuid> field, but no <alt-id> field, your <uuid> will be used as the
261 comment's <alt-id>.  An exception is raised if <alt-id> conflicts with
262 an existing comment.  Bugs do not have a permantent alt-id, so they
263 the <uuid>s you specify are not saved.  The <uuid>s _are_ used to
264 match agains prexisting bug and comment uuids, and comment alt-ids,
265 and fields explicitly given in the XML file will replace old versions
266 unless the --add-only flag.
267
268 *.extra_strings recieves special treatment, and if --add-only is not
269 set, the resulting list concatenates both source lists and removes
270 repeats.
271
272 Here's an example of import activity:
273   Repository
274    bug (uuid=B, creator=John, status=open)
275      estr (don't forget your towel)
276      estr (helps with space travel)
277      com (uuid=C1, author=Jane, body=Hello)
278      com (uuid=C2, author=Jess, body=World)
279   XML
280    bug (uuid=B, status=fixed)
281      estr (don't forget your towel)
282      estr (watch out for flying dolphins)
283      com (uuid=C1, body=So long)
284      com (uuid=C3, author=Jed, body=And thanks)
285   Result
286    bug (uuid=B, creator=John, status=fixed)
287      estr (don't forget your towel)
288      estr (helps with space travel)
289      estr (watch out for flying dolphins)
290      com (uuid=C1, author=Jane, body=So long)
291      com (uuid=C2, author=Jess, body=World)
292      com (uuid=C4, alt-id=C3, author=Jed, body=And thanks)
293   Result, with --add-only
294    bug (uuid=B, creator=John, status=open)
295      estr (don't forget your towel)
296      estr (helps with space travel)
297      com (uuid=C1, author=Jane, body=Hello)
298      com (uuid=C2, author=Jess, body=World)
299      com (uuid=C4, alt-id=C3, author=Jed, body=And thanks)
300
301 Examples:
302
303 Import comments (e.g. emails from an mbox) and append to bug XYZ
304   $ be-mbox-to-xml mail.mbox | be import-xml --c XYZ -
305 Or you can append those emails underneath the prexisting comment XYZ-3
306   $ be-mbox-to-xml mail.mbox | be import-xml --c XYZ-3 -
307
308 User creates a new bug
309   user$ be new "The demuxulizer is broken"
310   Created bug with ID 48f
311   user$ be comment 48f
312   <Describe bug>
313   ...
314 User exports bug as xml and emails it to the developers
315   user$ be show --xml 48f > 48f.xml
316   user$ cat 48f.xml | mail -s "Demuxulizer bug xml" devs@b.com
317 or equivalently (with a slightly fancier be-handle-mail compatible
318 email):
319   user$ be email-bugs 48f
320 Devs recieve email, and save it's contents as demux-bug.xml
321   dev$ cat demux-bug.xml | be import-xml -
322 """
323
324
325 Import_xml = Import_XML # alias for libbe.command.base.get_command_class()
326
327 if libbe.TESTING == True:
328     class LonghelpTestCase (unittest.TestCase):
329         """
330         Test import scenarios given in longhelp.
331         """
332         def setUp(self):
333             self.bugdir = libbe.bugdir.SimpleBugDir(memory=False)
334             io = libbe.command.StringInputOutput()
335             self.ui = libbe.command.UserInterface(io=io)
336             self.ui.storage_callbacks.set_storage(self.bugdir.storage)
337             self.cmd = Import_XML(ui=self.ui)
338             self.cmd._storage = self.bugdir.storage
339             self.cmd._setup_io = lambda i_enc,o_enc : None
340             bugA = self.bugdir.bug_from_uuid('a')
341             self.bugdir.remove_bug(bugA)
342             self.bugdir.storage.writeable = False
343             bugB = self.bugdir.bug_from_uuid('b')
344             bugB.creator = 'John'
345             bugB.status = 'open'
346             bugB.extra_strings += ["don't forget your towel"]
347             bugB.extra_strings += ['helps with space travel']
348             comm1 = bugB.comment_root.new_reply(body='Hello\n')
349             comm1.uuid = 'c1'
350             comm1.author = 'Jane'
351             comm2 = bugB.comment_root.new_reply(body='World\n')
352             comm2.uuid = 'c2'
353             comm2.author = 'Jess'
354             self.bugdir.storage.writeable = True
355             bugB.save()
356             self.xml = """
357             <be-xml>
358               <bug>
359                 <uuid>b</uuid>
360                 <status>fixed</status>
361                 <summary>a test bug</summary>
362                 <extra-string>don't forget your towel</extra-string>
363                 <extra-string>watch out for flying dolphins</extra-string>
364                 <comment>
365                   <uuid>c1</uuid>
366                   <body>So long</body>
367                 </comment>
368                 <comment>
369                   <uuid>c3</uuid>
370                   <author>Jed</author>
371                   <body>And thanks</body>
372                 </comment>
373               </bug>
374             </be-xml>
375             """
376             self.root_comment_xml = """
377             <be-xml>
378               <comment>
379                 <uuid>c1</uuid>
380                 <body>So long</body>
381               </comment>
382               <comment>
383                 <uuid>c3</uuid>
384                 <author>Jed</author>
385                 <body>And thanks</body>
386               </comment>
387             </be-xml>
388             """
389         def tearDown(self):
390             self.bugdir.cleanup()
391             self.ui.cleanup()
392         def _execute(self, xml, params={}, args=[]):
393             self.ui.io.set_stdin(xml)
394             self.ui.run(self.cmd, params, args)
395             self.bugdir.flush_reload()
396         def testCleanBugdir(self):
397             uuids = list(self.bugdir.uuids())
398             self.failUnless(uuids == ['b'], uuids)
399         def testNotAddOnly(self):
400             bugB = self.bugdir.bug_from_uuid('b')
401             self._execute(self.xml, {}, ['-'])
402             uuids = list(self.bugdir.uuids())
403             self.failUnless(uuids == ['b'], uuids)
404             bugB = self.bugdir.bug_from_uuid('b')
405             self.failUnless(bugB.uuid == 'b', bugB.uuid)
406             self.failUnless(bugB.creator == 'John', bugB.creator)
407             self.failUnless(bugB.status == 'fixed', bugB.status)
408             self.failUnless(bugB.summary == 'a test bug', bugB.summary)
409             estrs = ["don't forget your towel",
410                      'helps with space travel',
411                      'watch out for flying dolphins']
412             self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
413             comments = list(bugB.comments())
414             self.failUnless(len(comments) == 3,
415                             ['%s (%s, %s)' % (c.uuid, c.alt_id, c.body)
416                              for c in comments])
417             c1 = bugB.comment_from_uuid('c1')
418             comments.remove(c1)
419             self.failUnless(c1.uuid == 'c1', c1.uuid)
420             self.failUnless(c1.alt_id == None, c1.alt_id)
421             self.failUnless(c1.author == 'Jane', c1.author)
422             self.failUnless(c1.body == 'So long\n', c1.body)
423             c2 = bugB.comment_from_uuid('c2')
424             comments.remove(c2)
425             self.failUnless(c2.uuid == 'c2', c2.uuid)
426             self.failUnless(c2.alt_id == None, c2.alt_id)
427             self.failUnless(c2.author == 'Jess', c2.author)
428             self.failUnless(c2.body == 'World\n', c2.body)
429             c4 = comments[0]
430             self.failUnless(len(c4.uuid) == 36, c4.uuid)
431             self.failUnless(c4.alt_id == 'c3', c4.alt_id)
432             self.failUnless(c4.author == 'Jed', c4.author)
433             self.failUnless(c4.body == 'And thanks\n', c4.body)
434         def testAddOnly(self):
435             bugB = self.bugdir.bug_from_uuid('b')
436             initial_bugB_summary = bugB.summary
437             self._execute(self.xml, {'add-only':True}, ['-'])
438             uuids = list(self.bugdir.uuids())
439             self.failUnless(uuids == ['b'], uuids)
440             bugB = self.bugdir.bug_from_uuid('b')
441             self.failUnless(bugB.uuid == 'b', bugB.uuid)
442             self.failUnless(bugB.creator == 'John', bugB.creator)
443             self.failUnless(bugB.status == 'open', bugB.status)
444             self.failUnless(bugB.summary == initial_bugB_summary, bugB.summary)
445             estrs = ["don't forget your towel",
446                      'helps with space travel']
447             self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
448             comments = list(bugB.comments())
449             self.failUnless(len(comments) == 3,
450                             ['%s (%s)' % (c.uuid, c.alt_id) for c in comments])
451             c1 = bugB.comment_from_uuid('c1')
452             comments.remove(c1)
453             self.failUnless(c1.uuid == 'c1', c1.uuid)
454             self.failUnless(c1.alt_id == None, c1.alt_id)
455             self.failUnless(c1.author == 'Jane', c1.author)
456             self.failUnless(c1.body == 'Hello\n', c1.body)
457             c2 = bugB.comment_from_uuid('c2')
458             comments.remove(c2)
459             self.failUnless(c2.uuid == 'c2', c2.uuid)
460             self.failUnless(c2.alt_id == None, c2.alt_id)
461             self.failUnless(c2.author == 'Jess', c2.author)
462             self.failUnless(c2.body == 'World\n', c2.body)
463             c4 = comments[0]
464             self.failUnless(len(c4.uuid) == 36, c4.uuid)
465             self.failUnless(c4.alt_id == 'c3', c4.alt_id)
466             self.failUnless(c4.author == 'Jed', c4.author)
467             self.failUnless(c4.body == 'And thanks\n', c4.body)
468         def testRootCommentsNotAddOnly(self):
469             bugB = self.bugdir.bug_from_uuid('b')
470             initial_bugB_summary = bugB.summary
471             self._execute(self.root_comment_xml, {'comment-root':'/b'}, ['-'])
472             uuids = list(self.bugdir.uuids())
473             uuids = list(self.bugdir.uuids())
474             self.failUnless(uuids == ['b'], uuids)
475             bugB = self.bugdir.bug_from_uuid('b')
476             self.failUnless(bugB.uuid == 'b', bugB.uuid)
477             self.failUnless(bugB.creator == 'John', bugB.creator)
478             self.failUnless(bugB.status == 'open', bugB.status)
479             self.failUnless(bugB.summary == initial_bugB_summary, bugB.summary)
480             estrs = ["don't forget your towel",
481                      'helps with space travel']
482             self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
483             comments = list(bugB.comments())
484             self.failUnless(len(comments) == 3,
485                             ['%s (%s, %s)' % (c.uuid, c.alt_id, c.body)
486                              for c in comments])
487             c1 = bugB.comment_from_uuid('c1')
488             comments.remove(c1)
489             self.failUnless(c1.uuid == 'c1', c1.uuid)
490             self.failUnless(c1.alt_id == None, c1.alt_id)
491             self.failUnless(c1.author == 'Jane', c1.author)
492             self.failUnless(c1.body == 'So long\n', c1.body)
493             c2 = bugB.comment_from_uuid('c2')
494             comments.remove(c2)
495             self.failUnless(c2.uuid == 'c2', c2.uuid)
496             self.failUnless(c2.alt_id == None, c2.alt_id)
497             self.failUnless(c2.author == 'Jess', c2.author)
498             self.failUnless(c2.body == 'World\n', c2.body)
499             c4 = comments[0]
500             self.failUnless(len(c4.uuid) == 36, c4.uuid)
501             self.failUnless(c4.alt_id == 'c3', c4.alt_id)
502             self.failUnless(c4.author == 'Jed', c4.author)
503             self.failUnless(c4.body == 'And thanks\n', c4.body)
504         def testRootCommentsAddOnly(self):
505             bugB = self.bugdir.bug_from_uuid('b')
506             initial_bugB_summary = bugB.summary
507             self._execute(self.root_comment_xml,
508                           {'comment-root':'/b', 'add-only':True}, ['-'])
509             uuids = list(self.bugdir.uuids())
510             self.failUnless(uuids == ['b'], uuids)
511             bugB = self.bugdir.bug_from_uuid('b')
512             self.failUnless(bugB.uuid == 'b', bugB.uuid)
513             self.failUnless(bugB.creator == 'John', bugB.creator)
514             self.failUnless(bugB.status == 'open', bugB.status)
515             self.failUnless(bugB.summary == initial_bugB_summary, bugB.summary)
516             estrs = ["don't forget your towel",
517                      'helps with space travel']
518             self.failUnless(bugB.extra_strings == estrs, bugB.extra_strings)
519             comments = list(bugB.comments())
520             self.failUnless(len(comments) == 3,
521                             ['%s (%s)' % (c.uuid, c.alt_id) for c in comments])
522             c1 = bugB.comment_from_uuid('c1')
523             comments.remove(c1)
524             self.failUnless(c1.uuid == 'c1', c1.uuid)
525             self.failUnless(c1.alt_id == None, c1.alt_id)
526             self.failUnless(c1.author == 'Jane', c1.author)
527             self.failUnless(c1.body == 'Hello\n', c1.body)
528             c2 = bugB.comment_from_uuid('c2')
529             comments.remove(c2)
530             self.failUnless(c2.uuid == 'c2', c2.uuid)
531             self.failUnless(c2.alt_id == None, c2.alt_id)
532             self.failUnless(c2.author == 'Jess', c2.author)
533             self.failUnless(c2.body == 'World\n', c2.body)
534             c4 = comments[0]
535             self.failUnless(len(c4.uuid) == 36, c4.uuid)
536             self.failUnless(c4.alt_id == 'c3', c4.alt_id)
537             self.failUnless(c4.author == 'Jed', c4.author)
538             self.failUnless(c4.body == 'And thanks\n', c4.body)
539
540     unitsuite =unittest.TestLoader().loadTestsFromModule(sys.modules[__name__])
541     suite = unittest.TestSuite([unitsuite, doctest.DocTestSuite()])