download.py 17.4 KB
Newer Older
1
# Copyright (c) 2009, 2011 Google Inc. All rights reserved.
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# Copyright (c) 2009 Apple Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
# 
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
# 
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

30
from webkitpy.tool import steps
31

32
33
from webkitpy.common.checkout.changelog import ChangeLog
from webkitpy.common.config import urls
34
from webkitpy.common.system.executive import ScriptError
35
from webkitpy.tool.commands.abstractsequencedcommand import AbstractSequencedCommand
36
from webkitpy.tool.commands.stepsequence import StepSequence
37
from webkitpy.tool.comments import bug_comment_from_commit_text
38
from webkitpy.tool.grammar import pluralize
39
from webkitpy.tool.multicommandtool import AbstractDeclarativeCommand
40
from webkitpy.common.system.deprecated_logging import error, log
41
42


43
44
45
46
47
48
49
50
51
52
53
class Clean(AbstractSequencedCommand):
    name = "clean"
    help_text = "Clean the working copy"
    steps = [
        steps.CleanWorkingDirectory,
    ]

    def _prepare_state(self, options, args, tool):
        options.force_clean = True


54
55
56
57
58
59
60
61
62
class Update(AbstractSequencedCommand):
    name = "update"
    help_text = "Update working copy (used internally)"
    steps = [
        steps.CleanWorkingDirectory,
        steps.Update,
    ]


63
class Build(AbstractSequencedCommand):
64
65
66
    name = "build"
    help_text = "Update working copy and build"
    steps = [
67
68
69
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.Build,
70
71
    ]

72
73
74
    def _prepare_state(self, options, args, tool):
        options.build = True

75

76
class BuildAndTest(AbstractSequencedCommand):
77
78
79
    name = "build-and-test"
    help_text = "Update working copy, build, and run the tests"
    steps = [
80
81
82
83
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.Build,
        steps.RunTests,
84
85
86
    ]


87
class Land(AbstractSequencedCommand):
88
    name = "land"
89
90
    help_text = "Land the current working directory diff and updates the associated bug if any"
    argument_names = "[BUGID]"
91
    show_in_main_help = True
92
    steps = [
93
        steps.AddSvnMimetypeForPng,
94
        steps.UpdateChangeLogsWithReviewer,
95
        steps.ValidateReviewer,
96
        steps.ValidateChangeLogs, # We do this after UpdateChangeLogsWithReviewer to avoid not having to cache the diff twice.
97
98
99
100
        steps.Build,
        steps.RunTests,
        steps.Commit,
        steps.CloseBugForLandDiff,
101
    ]
102
    long_help = """land commits the current working copy diff (just as svn or git commit would).
103
land will NOT build and run the tests before committing, but you can use the --build option for that.
104
If a bug id is provided, or one can be found in the ChangeLog land will update the bug after committing."""
105

106
    def _prepare_state(self, options, args, tool):
107
        changed_files = self._tool.scm().changed_files(options.git_commit)
108
        return {
109
110
            "changed_files": changed_files,
            "bug_id": (args and args[0]) or tool.checkout().bug_id_for_this_commit(options.git_commit, changed_files),
111
112
113
        }


114
115
116
117
118
119
class LandCowboy(AbstractSequencedCommand):
    name = "land-cowboy"
    help_text = "Prepares a ChangeLog and lands the current working directory diff."
    steps = [
        steps.PrepareChangeLog,
        steps.EditChangeLog,
120
        steps.CheckStyle,
121
122
123
124
        steps.ConfirmDiff,
        steps.Build,
        steps.RunTests,
        steps.Commit,
125
        steps.CloseBugForLandDiff,
126
127
    ]

128
129
130
    def _prepare_state(self, options, args, tool):
        options.check_style_filter = "-changelog"

131

132
133
134
135
136
class LandCowhand(LandCowboy):
    # Gender-blind term for cowboy, see: http://en.wiktionary.org/wiki/cowhand
    name = "land-cowhand"


137
138
139
140
141
142
143
144
class CheckStyleLocal(AbstractSequencedCommand):
    name = "check-style-local"
    help_text = "Run check-webkit-style on the current working directory diff"
    steps = [
        steps.CheckStyle,
    ]


145
class AbstractPatchProcessingCommand(AbstractDeclarativeCommand):
146
147
148
149
150
    # Subclasses must implement the methods below.  We don't declare them here
    # because we want to be able to implement them with mix-ins.
    #
    # def _fetch_list_of_patches_to_process(self, options, args, tool):
    # def _prepare_to_process(self, options, args, tool):
151
152
153
154
155

    @staticmethod
    def _collect_patches_by_bug(patches):
        bugs_to_patches = {}
        for patch in patches:
156
            bugs_to_patches[patch.bug_id()] = bugs_to_patches.get(patch.bug_id(), []) + [patch]
157
158
159
160
161
162
163
164
165
166
167
168
169
170
        return bugs_to_patches

    def execute(self, options, args, tool):
        self._prepare_to_process(options, args, tool)
        patches = self._fetch_list_of_patches_to_process(options, args, tool)

        # It's nice to print out total statistics.
        bugs_to_patches = self._collect_patches_by_bug(patches)
        log("Processing %s from %s." % (pluralize("patch", len(patches)), pluralize("bug", len(bugs_to_patches))))

        for patch in patches:
            self._process_patch(patch, options, args, tool)


171
172
173
class AbstractPatchSequencingCommand(AbstractPatchProcessingCommand):
    prepare_steps = None
    main_steps = None
174

175
    def __init__(self):
176
        options = []
177
178
179
        self._prepare_sequence = StepSequence(self.prepare_steps)
        self._main_sequence = StepSequence(self.main_steps)
        options = sorted(set(self._prepare_sequence.options() + self._main_sequence.options()))
180
        AbstractPatchProcessingCommand.__init__(self, options)
181
182

    def _prepare_to_process(self, options, args, tool):
183
        self._prepare_sequence.run_and_handle_errors(tool, options)
184
185

    def _process_patch(self, patch, options, args, tool):
186
187
        state = { "patch" : patch }
        self._main_sequence.run_and_handle_errors(tool, options, state)
188
189


190
191
class ProcessAttachmentsMixin(object):
    def _fetch_list_of_patches_to_process(self, options, args, tool):
192
        return map(lambda patch_id: tool.bugs.fetch_attachment(patch_id), args)
193
194
195
196
197
198


class ProcessBugsMixin(object):
    def _fetch_list_of_patches_to_process(self, options, args, tool):
        all_patches = []
        for bug_id in args:
199
200
201
            patches = tool.bugs.fetch_bug(bug_id).reviewed_patches()
            log("%s found on bug %s." % (pluralize("reviewed patch", len(patches)), bug_id))
            all_patches += patches
202
203
204
205
206
207
        if not all_patches:
            log("No reviewed patches found, looking for unreviewed patches.")
            for bug_id in args:
                patches = tool.bugs.fetch_bug(bug_id).patches()
                log("%s found on bug %s." % (pluralize("patch", len(patches)), bug_id))
                all_patches += patches
208
209
210
        return all_patches


eric@webkit.org's avatar
eric@webkit.org committed
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
class ProcessURLsMixin(object):
    def _fetch_list_of_patches_to_process(self, options, args, tool):
        all_patches = []
        for url in args:
            bug_id = urls.parse_bug_id(url)
            if bug_id:
                patches = tool.bugs.fetch_bug(bug_id).patches()
                log("%s found on bug %s." % (pluralize("patch", len(patches)), bug_id))
                all_patches += patches

            attachment_id = urls.parse_attachment_id(url)
            if attachment_id:
                all_patches += tool.bugs.fetch_attachment(attachment_id)

        return all_patches


228
class CheckStyle(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
229
    name = "check-style"
230
231
    help_text = "Run check-webkit-style on the specified attachments"
    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
232
    main_steps = [
233
234
235
236
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.ApplyPatch,
        steps.CheckStyle,
237
    ]
238
239


240
class BuildAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
241
    name = "build-attachment"
242
243
    help_text = "Apply and build patches from bugzilla"
    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
244
    main_steps = [
245
246
247
248
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.ApplyPatch,
        steps.Build,
249
    ]
250

251

252
253
254
255
256
257
258
259
260
261
262
263
264
class BuildAndTestAttachment(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
    name = "build-and-test-attachment"
    help_text = "Apply, build, and test patches from bugzilla"
    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
    main_steps = [
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.ApplyPatch,
        steps.Build,
        steps.RunTests,
    ]


265
266
class AbstractPatchApplyingCommand(AbstractPatchSequencingCommand):
    prepare_steps = [
267
268
269
        steps.EnsureLocalCommitIfNeeded,
        steps.CleanWorkingDirectoryWithLocalCommits,
        steps.Update,
270
271
    ]
    main_steps = [
272
        steps.ApplyPatchWithLocalCommit,
273
    ]
274
275
    long_help = """Updates the working copy.
Downloads and applies the patches, creating local commits if necessary."""
276
277


278
class ApplyAttachment(AbstractPatchApplyingCommand, ProcessAttachmentsMixin):
279
    name = "apply-attachment"
280
281
    help_text = "Apply an attachment to the local working directory"
    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
282
283
284
    show_in_main_help = True


285
286
class ApplyFromBug(AbstractPatchApplyingCommand, ProcessBugsMixin):
    name = "apply-from-bug"
287
288
    help_text = "Apply reviewed patches from provided bugs to the local working directory"
    argument_names = "BUGID [BUGIDS]"
289
290
291
    show_in_main_help = True


292
293
class ApplyWatchList(AbstractPatchSequencingCommand, ProcessAttachmentsMixin):
    name = "apply-watchlist"
294
    help_text = "Applies the watchlist to the specified attachments"
295
296
297
298
299
300
301
302
303
304
305
    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
    main_steps = [
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.ApplyPatch,
        steps.ApplyWatchList,
    ]
    long_help = """"Applies the watchlist to the specified attachments.
Downloads the attachment, applies it locally, runs the watchlist against it, and updates the bug with the result."""


306
307
class AbstractPatchLandingCommand(AbstractPatchSequencingCommand):
    main_steps = [
308
309
310
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.ApplyPatch,
311
        steps.ValidateChangeLogs,
312
        steps.ValidateReviewer,
313
314
315
316
317
        steps.Build,
        steps.RunTests,
        steps.Commit,
        steps.ClosePatch,
        steps.CloseBug,
318
    ]
319
320
321
322
323
324
325
326
    long_help = """Checks to make sure builders are green.
Updates the working copy.
Applies the patch.
Builds.
Runs the layout tests.
Commits the patch.
Clears the flags on the patch.
Closes the bug if no patches are marked for review."""
327
328


329
class LandAttachment(AbstractPatchLandingCommand, ProcessAttachmentsMixin):
330
    name = "land-attachment"
331
332
    help_text = "Land patches from bugzilla, optionally building and testing them first"
    argument_names = "ATTACHMENT_ID [ATTACHMENT_IDS]"
333
    show_in_main_help = True
334
335


336
337
class LandFromBug(AbstractPatchLandingCommand, ProcessBugsMixin):
    name = "land-from-bug"
338
339
    help_text = "Land all patches on the given bugs, optionally building and testing them first"
    argument_names = "BUGID [BUGIDS]"
340
    show_in_main_help = True
341
342


eric@webkit.org's avatar
eric@webkit.org committed
343
344
345
346
347
348
class LandFromURL(AbstractPatchLandingCommand, ProcessURLsMixin):
    name = "land-from-url"
    help_text = "Land all patches on the given URLs, optionally building and testing them first"
    argument_names = "URL [URLS]"


349
350
351
352
353
354
355
356
357
358
359
360
class ValidateChangelog(AbstractSequencedCommand):
    name = "validate-changelog"
    help_text = "Validate that the ChangeLogs and reviewers look reasonable"
    long_help = """Examines the current diff to see whether the ChangeLogs
and the reviewers listed in the ChangeLogs look reasonable.
"""
    steps = [
        steps.ValidateChangeLogs,
        steps.ValidateReviewer,
    ]


361
class AbstractRolloutPrepCommand(AbstractSequencedCommand):
362
    argument_names = "REVISION [REVISIONS] REASON"
363

364
    def _commit_info(self, revision):
365
        commit_info = self._tool.checkout().commit_info_for_revision(revision)
366
        if commit_info and commit_info.bug_id():
367
368
369
370
371
            # Note: Don't print a bug URL here because it will confuse the
            #       SheriffBot because the SheriffBot just greps the output
            #       of create-rollout for bug URLs.  It should do better
            #       parsing instead.
            log("Preparing rollout for bug %s." % commit_info.bug_id())
372
373
374
        else:
            log("Unable to parse bug number from diff.")
        return commit_info
375
376

    def _prepare_state(self, options, args, tool):
377
378
379
380
381
382
383
384
385
386
        revision_list = []
        for revision in str(args[0]).split():
            if revision.isdigit():
                revision_list.append(int(revision))
            else:
                raise ScriptError(message="Invalid svn revision number: " + revision)
        revision_list.sort()

        # We use the earliest revision for the bug info
        earliest_revision = revision_list[0]
387
        state = {
388
389
            "revision": earliest_revision,
            "revision_list": revision_list,
390
            "reason": args[1],
391
        }
392
393
394
395
396
397
398
399
400
        commit_info = self._commit_info(earliest_revision)
        if commit_info:
            state["bug_id"] = commit_info.bug_id()
            cc_list = sorted([party.bugzilla_email()
                            for party in commit_info.responsible_parties()
                            if party.bugzilla_email()])
            # FIXME: We should used the list as the canonical representation.
            state["bug_cc"] = ",".join(cc_list)
        return state
401
402
403
404


class PrepareRollout(AbstractRolloutPrepCommand):
    name = "prepare-rollout"
405
    help_text = "Revert the given revision(s) in the working copy and prepare ChangeLogs with revert reason"
406
    long_help = """Updates the working copy.
407
Applies the inverse diff for the provided revision(s).
408
409
410
411
412
413
414
415
416
Creates an appropriate rollout ChangeLog, including a trac link and bug link.
"""
    steps = [
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.RevertRevision,
        steps.PrepareChangeLogForRevert,
    ]

417

418
419
class CreateRollout(AbstractRolloutPrepCommand):
    name = "create-rollout"
420
    help_text = "Creates a bug to track the broken SVN revision(s) and uploads a rollout patch."
421
422
423
    steps = [
        steps.CleanWorkingDirectory,
        steps.Update,
424
425
        steps.RevertRevision,
        steps.CreateBug,
426
427
428
        steps.PrepareChangeLogForRevert,
        steps.PostDiffForRevert,
    ]
429

430
431
432
433
434
435
436
437
438
439
440
    def _prepare_state(self, options, args, tool):
        state = AbstractRolloutPrepCommand._prepare_state(self, options, args, tool)
        # Currently, state["bug_id"] points to the bug that caused the
        # regression.  We want to create a new bug that blocks the old bug
        # so we move state["bug_id"] to state["bug_blocked"] and delete the
        # old state["bug_id"] so that steps.CreateBug will actually create
        # the new bug that we want (and subsequently store its bug id into
        # state["bug_id"])
        state["bug_blocked"] = state["bug_id"]
        del state["bug_id"]
        state["bug_title"] = "REGRESSION(r%s): %s" % (state["revision"], state["reason"])
441
        state["bug_description"] = "%s broke the build:\n%s" % (urls.view_revision_url(state["revision"]), state["reason"])
442
443
444
445
446
447
448
449
450
451
452
        # FIXME: If we had more context here, we could link to other open bugs
        #        that mention the test that regressed.
        if options.parent_command == "sheriff-bot":
            state["bug_description"] += """

This is an automatic bug report generated by the sheriff-bot. If this bug
report was created because of a flaky test, please file a bug for the flaky
test (if we don't already have one on file) and dup this bug against that bug
so that we can track how often these flaky tests case pain.

"Only you can prevent forest fires." -- Smokey the Bear
453
"""
454
455
        return state

456
457

class Rollout(AbstractRolloutPrepCommand):
458
    name = "rollout"
459
    show_in_main_help = True
460
    help_text = "Revert the given revision(s) in the working copy and optionally commit the revert and re-open the original bug"
461
462
463
464
465
466
    long_help = """Updates the working copy.
Applies the inverse diff for the provided revision.
Creates an appropriate rollout ChangeLog, including a trac link and bug link.
Opens the generated ChangeLogs in $EDITOR.
Shows the prepared diff for confirmation.
Commits the revert and updates the bug (including re-opening the bug if necessary)."""
467
    steps = [
468
469
470
471
        steps.CleanWorkingDirectory,
        steps.Update,
        steps.RevertRevision,
        steps.PrepareChangeLogForRevert,
472
473
        steps.EditChangeLog,
        steps.ConfirmDiff,
474
475
476
        steps.Build,
        steps.Commit,
        steps.ReopenBugAfterRollout,
477
    ]