Commit 0a3a187d authored by eric@webkit.org's avatar eric@webkit.org
Browse files

2010-03-13 Victor Wang <victorw@chromium.org>

        Add appengine app to host and serve webkit layout test results.

        The app allows you post test result files (json) and serve them up.
        Chromium flakiness dashboard will first use this app to host results.json
        and expectations.json, but the files hosted by this app are not limited
        to chromium results or json files. It can be used to host other files if needed.

        https://bugs.webkit.org/show_bug.cgi?id=35944

        * TestResultServer: Added.
        * TestResultServer/app.yaml: Added.
        * TestResultServer/handlers: Added.
        * TestResultServer/handlers/__init__.py: Added.
        * TestResultServer/handlers/menu.py: Added.
        * TestResultServer/handlers/testfilehandler.py: Added.
        * TestResultServer/index.yaml: Added.
        * TestResultServer/main.py: Added.
        * TestResultServer/model: Added.
        * TestResultServer/model/__init__.py: Added.
        * TestResultServer/model/testfile.py: Added.
        * TestResultServer/stylesheets: Added.
        * TestResultServer/stylesheets/form.css: Added.
        * TestResultServer/stylesheets/menu.css: Added.
        * TestResultServer/stylesheets/testfile.css: Added.
        * TestResultServer/templates: Added.
        * TestResultServer/templates/menu.html: Added.
        * TestResultServer/templates/showfilelist.html: Added.
        * TestResultServer/templates/uploadform.html: Added.

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@55960 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent 9f7b68b2
2010-03-13 Victor Wang <victorw@chromium.org>
Add appengine app to host and serve webkit layout test results.
The app allows you post test result files (json) and serve them up.
Chromium flakiness dashboard will first use this app to host results.json
and expectations.json, but the files hosted by this app are not limited
to chromium results or json files. It can be used to host other files if needed.
https://bugs.webkit.org/show_bug.cgi?id=35944
* TestResultServer: Added.
* TestResultServer/app.yaml: Added.
* TestResultServer/handlers: Added.
* TestResultServer/handlers/__init__.py: Added.
* TestResultServer/handlers/menu.py: Added.
* TestResultServer/handlers/testfilehandler.py: Added.
* TestResultServer/index.yaml: Added.
* TestResultServer/main.py: Added.
* TestResultServer/model: Added.
* TestResultServer/model/__init__.py: Added.
* TestResultServer/model/testfile.py: Added.
* TestResultServer/stylesheets: Added.
* TestResultServer/stylesheets/form.css: Added.
* TestResultServer/stylesheets/menu.css: Added.
* TestResultServer/stylesheets/testfile.css: Added.
* TestResultServer/templates: Added.
* TestResultServer/templates/menu.html: Added.
* TestResultServer/templates/showfilelist.html: Added.
* TestResultServer/templates/uploadform.html: Added.
2010-03-13 Dimitri Glazkov <dglazkov@chromium.org>
 
Reviewed by David Levin.
......
application: test-results
version: 1
runtime: python
api_version: 1
handlers:
- url: /stylesheets
static_dir: stylesheets
- url: /.*
script: main.py
# Required for Python to search this directory for module files
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
menu = [
["List of files", "/getfile"],
["List of results.json files", "/getfile?name=results.json"],
["List of expectations.json files", "/getfile?name=expectations.json"],
["Upload file", "/uploadform"],
]
class Menu(webapp.RequestHandler):
def get(self):
template_values = {
"menu": menu,
}
self.response.out.write(
template.render("templates/menu.html", template_values))
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import logging
import urllib
from google.appengine.ext import blobstore
from google.appengine.ext import webapp
from google.appengine.ext.webapp import blobstore_handlers
from google.appengine.ext.webapp import template
from model.testfile import TestFile
PARAM_BUILDER = "builder"
PARAM_DIR = "dir"
PARAM_FILE = "file"
PARAM_NAME = "name"
PARAM_KEY = "key"
PARAM_TEST_TYPE = "testtype"
class DeleteFile(webapp.RequestHandler):
"""Delete test file for a given builder and name from datastore (metadata) and blobstore (file data)."""
def get(self):
key = self.request.get(PARAM_KEY)
builder = self.request.get(PARAM_BUILDER)
test_type = self.request.get(PARAM_TEST_TYPE)
name = self.request.get(PARAM_NAME)
logging.debug(
"Deleting File, builder: %s, test_type: %s, name: %s, blob key: %s.",
builder, test_type, name, key)
TestFile.delete_file(key, builder, test_type, name, 100)
# Display file list after deleting the file.
self.redirect("/getfile?builder=%s&testtype=%s&name=%s"
% (builder, test_type, name))
class GetFile(blobstore_handlers.BlobstoreDownloadHandler):
"""Get file content or list of files for given builder and name."""
def get_file_list(self, builder, test_type, name):
"""Get and display a list of files that matches builder and file name.
Args:
builder: builder name
test_type: type of the test
name: file name
"""
files = TestFile.get_files(builder, test_type, name, 100)
if not files:
logging.info("File not found, builder: %s, test_type: %s, name: %s.",
builder, test_type, name)
self.response.out.write("File not found")
return
template_values = {
"builder": builder,
"test_type": test_type,
"name": name,
"files": files,
}
self.response.out.write(template.render("templates/showfilelist.html",
template_values))
def get_file_content(self, builder, test_type, name):
"""Return content of the file that matches builder and file name.
Args:
builder: builder name
test_type: type of the test
name: file name
"""
files = TestFile.get_files(builder, test_type, name, 1)
if not files:
logging.info("File not found, builder: %s, test_type: %s, name: %s.",
builder, test_type, name)
return
blob_key = files[0].blob_key
blob_info = blobstore.get(blob_key)
if blob_info:
self.send_blob(blob_info, "text/plain")
def get(self):
builder = self.request.get(PARAM_BUILDER)
test_type = self.request.get(PARAM_TEST_TYPE)
name = self.request.get(PARAM_NAME)
dir = self.request.get(PARAM_DIR)
logging.debug(
"Getting files, builder: %s, test_type: %s, name: %s.",
builder, test_type, name)
# If parameter "dir" is specified or there is no builder or filename
# specified in the request, return list of files, otherwise, return
# file content.
if dir or not builder or not name:
return self.get_file_list(builder, test_type, name)
else:
return self.get_file_content(builder, test_type, name)
class GetUploadUrl(webapp.RequestHandler):
"""Get an url for uploading file to blobstore. A special url is required for each blobsotre upload."""
def get(self):
upload_url = blobstore.create_upload_url("/upload")
logging.info("Getting upload url: %s.", upload_url)
self.response.out.write(upload_url)
class Upload(blobstore_handlers.BlobstoreUploadHandler):
"""Upload file to blobstore."""
def post(self):
uploaded_files = self.get_uploads("file")
if not uploaded_files:
return self._upload_done([("Missing upload file field.")])
builder = self.request.get(PARAM_BUILDER)
if not builder:
for blob_info in uploaded_files:
blob_info.delete()
return self._upload_done([("Missing builder parameter in upload request.")])
test_type = self.request.get(PARAM_TEST_TYPE)
logging.debug(
"Processing upload request, builder: %s, test_type: %s.",
builder, test_type)
errors = []
for blob_info in uploaded_files:
tf = TestFile.update_file(builder, test_type, blob_info)
if not tf:
errors.append(
"Upload failed, builder: %s, test_type: %s, name: %s." %
(builder, test_type, blob_info.filename))
blob_info.delete()
return self._upload_done(errors)
def _upload_done(self, errors):
logging.info("upload done.")
error_messages = []
for error in errors:
logging.info(error)
error_messages.append("error=%s" % urllib.quote(error))
if error_messages:
redirect_url = "/uploadfail?%s" % "&".join(error_messages)
else:
redirect_url = "/uploadsuccess"
logging.info(redirect_url)
# BlobstoreUploadHandler requires redirect at the end.
self.redirect(redirect_url)
class UploadForm(webapp.RequestHandler):
"""Show a form so user can submit a file to blobstore."""
def get(self):
upload_url = blobstore.create_upload_url("/upload")
template_values = {
"upload_url": upload_url,
}
self.response.out.write(template.render("templates/uploadform.html",
template_values))
class UploadStatus(webapp.RequestHandler):
"""Return status of file uploading"""
def get(self):
if self.request.path == "/uploadsuccess":
self.response.set_status(200)
self.response.out.write("OK")
else:
errors = self.request.params.getall("error")
if errors:
messages = "FAIL: " + "\n".join(errors)
logging.warning(messages)
self.response.out.write(messages)
self.response.set_status(500)
indexes:
# AUTOGENERATED
# This index.yaml is automatically updated whenever the dev_appserver
# detects that a new type of query is run. If you want to manage the
# index.yaml file manually, remove the above marker line (the line
# saying "# AUTOGENERATED"). If you want to manage some indexes
# manually, move them above the marker line. The index.yaml file is
# automatically uploaded to the admin console when you next deploy
# your application using appcfg.py.
- kind: TestFile
properties:
- name: builder
- name: date
direction: desc
- kind: TestFile
properties:
- name: builder
- name: name
- name: date
direction: desc
- kind: TestFile
properties:
- name: builder
- name: name
- name: test_type
- name: date
direction: desc
- kind: TestFile
properties:
- name: builder=
- name: name=
- name: date
direction: desc
- kind: TestFile
properties:
- name: name
- name: date
direction: desc
- kind: TestFile
properties:
- name: test_type
- name: date
direction: desc
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Request a modern Django
from google.appengine.dist import use_library
use_library('django', '1.1')
from google.appengine.ext import webapp
from google.appengine.ext.webapp.util import run_wsgi_app
from handlers.menu import Menu
from handlers.testfilehandler import DeleteFile
from handlers.testfilehandler import GetFile
from handlers.testfilehandler import GetUploadUrl
from handlers.testfilehandler import Upload
from handlers.testfilehandler import UploadForm
from handlers.testfilehandler import UploadStatus
routes = [
('/deletefile', DeleteFile),
('/getfile', GetFile),
('/getuploadurl', GetUploadUrl),
('/upload', Upload),
('/uploadfail', UploadStatus),
('/uploadform', UploadForm),
('/uploadsuccess', UploadStatus),
('/*|/menu', Menu),
]
application = webapp.WSGIApplication(routes, debug=True)
def main():
run_wsgi_app(application)
if __name__ == "__main__":
main()
# Required for Python to search this directory for module files
# Copyright (C) 2010 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from datetime import datetime
import logging
from google.appengine.ext import blobstore
from google.appengine.ext import db
class TestFile(db.Model):
builder = db.StringProperty()
name = db.StringProperty()
test_type = db.StringProperty()
blob_key = db.StringProperty()
date = db.DateTimeProperty(auto_now_add=True)
@classmethod
def delete_file(cls, key, builder, test_type, name, limit):
if key:
file = db.get(key)
if not file:
logging.warning("File not found, key: %s.", key)
return False
file._delete_all()
else:
files = cls.get_files(builder, test_type, name, limit)
if not files:
logging.warning(
"File not found, builder: %s, test_type:%s, name: %s.",
builder, test_type, name)
return False
for file in files:
file._delete_all()
return True
@classmethod
def get_files(cls, builder, test_type, name, limit):
query = TestFile.all()
if builder:
query = query.filter("builder =", builder)
if test_type:
query = query.filter("test_type =", test_type)
if name:
query = query.filter("name =", name)
return query.order("-date").fetch(limit)
@classmethod
def add_file(cls, builder, test_type, blob_info):
file = TestFile()
file.builder = builder
file.test_type = test_type
file.name = blob_info.filename
file.blob_key = str(blob_info.key())
file.put()
logging.info(
"File saved, builder: %s, test_type: %s, name: %s, blob key: %s.",
builder, file.name, file.blob_key)
return file
@classmethod
def update_file(cls, builder, test_type, blob_info):
files = cls.get_files(builder, test_type, blob_info.filename, 1)
if not files:
return cls.add_file(builder, test_type, blob_info)
file = files[0]
old_blob_info = blobstore.BlobInfo.get(file.blob_key)
if old_blob_info:
old_blob_info.delete()
file.builder = builder
file.test_type = test_type
file.name = blob_info.filename
file.blob_key = str(blob_info.key())
file.date = datetime.now()
file.put()
logging.info(
"File replaced, builder: %s, test_type: %s, name: %s, blob key: %s.",
builder, test_type, file.name, file.blob_key)
return file
def _delete_all(self):
if self.blob_key:
blob_info = blobstore.BlobInfo.get(self.blob_key)
if blob_info:
blob_info.delete()
self.delete()
body {
font-family: Verdana;
padding: 0px;
color: #444;
}
h1 {
color: #444;
font-size: 14pt;
font-style: italic;
margin: 0px;
padding: 5px;
}
.label {
margin: 1px;
padding: 5px;
font-size: 11pt;
width: 90px;
}
.inputtext {
font-size: 11pt;
}
.button {
margin: 1px;
padding: 1px;
font-size: 11pt;
}
body {
font-family: Verdana, Helvetica, sans-serif;
}
h1 {
background-color: #EEE;
color: #444;
font-size: 14pt;
font-style: italic;
margin: 0px;
padding: 5px;
}
ul {
margin: 0px;
padding: 20px;
list-style: none;
}