Commit 8d5ba53c authored by oliver@apple.com's avatar oliver@apple.com

<rdar://problem/6164797> Add Canvas API to allow drawing of <video> frames

<https://bugs.webkit.org/show_bug.cgi?id=25920>

Reviewed by Sam Weinig and Dave Hyatt.

Add support for drawing the contents of the video element to the canvas
in accordance with the current HTML5 draft.

Test: media/video-canvas.html


git-svn-id: http://svn.webkit.org/repository/webkit/trunk@45060 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent 8fda65e2
2009-06-23 Oliver Hunt <oliver@apple.com> and Eric Carlson <eric.carlson@apple.com>
Reviewed by Sam Weinig and Dave Hyatt.
<rdar://problem/6164797> Add Canvas API to allow drawing of <video> frames
<https://bugs.webkit.org/show_bug.cgi?id=25920>
Add tests for drawing a video to the canvas element.
* media/video-canvas-expected.txt: Added.
* media/video-canvas.html: Added.
2009-06-23 Oliver Hunt <oliver@apple.com>
Reviewed by Gavin Barraclough.
......
Test <video> as a source for <canvas>.
RUN(video.src = 'content/counting.mp4')
EVENT(loadedmetadata)
EXPECTED (r == '255') OK
EXPECTED (g == '255') OK
EXPECTED (b == '0') OK
RUN(video.currentTime = 2)
EXPECTED (r == '8') OK
EXPECTED (g == '0') OK
EXPECTED (b == '226') OK
RUN(video.currentTime = 4)
EXPECTED (r == '0') OK
EXPECTED (g == '24') OK
EXPECTED (b == '197') OK
RUN(video.currentTime = 6)
EXPECTED (r == '0') OK
EXPECTED (g == '46') OK
EXPECTED (b == '166') OK
RUN(video.currentTime = 8)
EXPECTED (r == '0') OK
EXPECTED (g == '66') OK
EXPECTED (b == '136') OK
RUN(video.currentTime = 10)
EXPECTED (r == '0') OK
EXPECTED (g == '85') OK
EXPECTED (b == '112') OK
END OF TEST
<html>
<head>
<title>drawing &lt;video&gt; to &lt;canvas&gt;</title>
<script src=video-test.js></script>
<script>
var ctx;
var results = {
current: 0,
values: [
{ time:0, r:255, g:255, b:0 },
{ time:2, r:8, g:0, b:226 },
{ time:4, r:0, g:24, b:197 },
{ time:6, r:0, g:46, b:166 },
{ time:8, r:0, g:66, b:136 },
{ time:10, r:0, g:85, b:112 },
]
};
var width;
var height;
function testPixel()
{
var expected = results.values[results.current];
if (expected.time)
ctx.drawImage(video, 0, 0, width, height);
var frame = ctx.getImageData(0, 0, width, height);
r = frame.data[4 + 0];
g = frame.data[4 + 1];
b = frame.data[4 + 2];
testExpected("r", expected.r);
testExpected("g", expected.g);
testExpected("b", expected.b);
if (++results.current >= results.values.length)
endTest();
else
setTimeout(testFrame, 0);
}
function testFrame()
{
consoleWrite("");
var expected = results.values[results.current];
if (expected.time)
run("video.currentTime = " + expected.time);
setTimeout(testPixel, 100);
}
function loadedmetadata()
{
width = video.videoWidth / 2;
height = video.videoHeight / 2;
ctx = canvas.getContext("2d");
ctx.fillStyle = 'yellow';
ctx.fillRect(0, 0, width, height);
testFrame();
}
function start()
{
findMediaElement();
canvas = document.getElementsByTagName('canvas')[0];
waitForEvent('loadedmetadata', loadedmetadata);
run("video.src = 'content/counting.mp4'");
}
</script>
</head>
<body onload="start()" >
<div>
<video controls="true"></video>
<canvas width="160" height="120" ></canvas>
</div>
<p>Test &lt;video&gt; as a source for &lt;canvas&gt;.</p>
</body>
</html>
\ No newline at end of file
2009-06-23 Oliver Hunt <oliver@apple.com> and Eric Carlson <eric.carlson@apple.com>
Reviewed by Sam Weinig and Dave Hyatt.
<rdar://problem/6164797> Add Canvas API to allow drawing of <video> frames
<https://bugs.webkit.org/show_bug.cgi?id=25920>
Add support for drawing the contents of the video element to the canvas
in accordance with the current HTML5 draft.
Test: media/video-canvas.html
* bindings/js/JSCanvasRenderingContext2DCustom.cpp:
(WebCore::JSCanvasRenderingContext2D::drawImage):
Standard custom bindings stuff we need to do for all canvas methods.
* html/CanvasRenderingContext2D.cpp:
(WebCore::size): Helper function for finding the size of a video element
(WebCore::CanvasRenderingContext2D::checkOrigin): moved up in the file.
(WebCore::CanvasRenderingContext2D::drawImage): The various overloads of HTML5's drawImage(<video>)
* html/CanvasRenderingContext2D.h:
* html/HTMLMediaElement.h:
(WebCore::HTMLMediaElement::hasSingleSecurityOrigin):
hasSingleSecurityOrigin is needed for security, currently all implementations are trivial
as we force QT to maintain a single origin.
* html/HTMLVideoElement.cpp:
(WebCore::HTMLVideoElement::paint):
Paint routine on video so we don't have to look at MediaPlayer directly
* html/HTMLVideoElement.h:
* platform/graphics/MediaPlayer.cpp:
(WebCore::NullMediaPlayerPrivate::hasSingleSecurityOrigin):
(WebCore::MediaPlayer::hasSingleSecurityOrigin):
Default implementations of hasSingleSecurityOrigin
* platform/graphics/MediaPlayer.h:
* platform/graphics/MediaPlayerPrivate.h:
* platform/graphics/mac/MediaPlayerPrivateQTKit.h:
* platform/graphics/mac/MediaPlayerPrivateQTKit.mm:
(WebCore::MediaPlayerPrivate::setUpVideoRendering):
A video may need a player now even if it is not visible.
(WebCore::MediaPlayerPrivate::hasSingleSecurityOrigin):
Always return true due to restrictions we've placed on QT.
* platform/graphics/win/MediaPlayerPrivateQuickTimeWin.cpp:
(WebCore::MediaPlayerPrivate::paint):
Jump through some hoops to allow windows QT to draw to an intermediate buffer.
In the long term we'd like to cache the HDC, but this will do for now.
(WebCore::MediaPlayerPrivate::hasSingleSecurityOrigin):
* platform/graphics/win/MediaPlayerPrivateQuickTimeWin.h:
As for Mac we force QT to only allow same origin loads.
2009-06-23 Adam Langley <agl@google.com>
Reviewed by Eric Seidel.
......
/*
* Copyright (C) 2006, 2007 Apple Inc. All rights reserved.
* Copyright (C) 2006, 2007, 2009 Apple Inc. All rights reserved.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
......@@ -28,11 +28,13 @@
#include "FloatRect.h"
#include "HTMLCanvasElement.h"
#include "HTMLImageElement.h"
#include "HTMLVideoElement.h"
#include "ImageData.h"
#include "JSCanvasGradient.h"
#include "JSCanvasPattern.h"
#include "JSHTMLCanvasElement.h"
#include "JSHTMLImageElement.h"
#include "JSHTMLVideoElement.h"
#include "JSImageData.h"
#include <runtime/Error.h>
......@@ -230,6 +232,29 @@ JSValue JSCanvasRenderingContext2D::drawImage(ExecState* exec, const ArgList& ar
default:
return throwError(exec, SyntaxError);
}
#if ENABLE(VIDEO)
} else if (o->inherits(&JSHTMLVideoElement::s_info)) {
HTMLVideoElement* video = static_cast<HTMLVideoElement*>(static_cast<JSHTMLElement*>(o)->impl());
switch (args.size()) {
case 3:
context->drawImage(video, args.at(1).toFloat(exec), args.at(2).toFloat(exec));
break;
case 5:
context->drawImage(video, args.at(1).toFloat(exec), args.at(2).toFloat(exec),
args.at(3).toFloat(exec), args.at(4).toFloat(exec), ec);
setDOMException(exec, ec);
break;
case 9:
context->drawImage(video, FloatRect(args.at(1).toFloat(exec), args.at(2).toFloat(exec),
args.at(3).toFloat(exec), args.at(4).toFloat(exec)),
FloatRect(args.at(5).toFloat(exec), args.at(6).toFloat(exec),
args.at(7).toFloat(exec), args.at(8).toFloat(exec)), ec);
setDOMException(exec, ec);
break;
default:
return throwError(exec, SyntaxError);
}
#endif
} else {
setDOMException(exec, TYPE_MISMATCH_ERR);
}
......
/*
* Copyright (C) 2004, 2005, 2006, 2007, 2008 Apple Inc. All rights reserved.
* Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved.
* Copyright (C) 2008 Nokia Corporation and/or its subsidiary(-ies)
* Copyright (C) 2007 Alp Toker <alp@atoker.com>
* Copyright (C) 2008 Eric Seidel <eric@webkit.org>
......@@ -56,6 +56,7 @@
#include "Settings.h"
#include "StrokeStyleApplier.h"
#include "TextMetrics.h"
#include "HTMLVideoElement.h"
#include <stdio.h>
#include <wtf/ByteArray.h>
......@@ -911,6 +912,13 @@ static IntSize size(HTMLImageElement* image)
return cachedImage->imageSize(1.0f); // FIXME: Not sure about this.
return IntSize();
}
static IntSize size(HTMLVideoElement* video)
{
if (MediaPlayer* player = video->player())
return player->naturalSize();
return IntSize();
}
static inline FloatRect normalizeRect(const FloatRect& rect)
{
......@@ -920,6 +928,13 @@ static inline FloatRect normalizeRect(const FloatRect& rect)
max(rect.height(), -rect.height()));
}
void CanvasRenderingContext2D::checkOrigin(const KURL& url)
{
RefPtr<SecurityOrigin> origin = SecurityOrigin::create(url);
if (!m_canvas->document()->securityOrigin()->canAccess(origin.get()))
m_canvas->setOriginTainted();
}
void CanvasRenderingContext2D::drawImage(HTMLImageElement* image, float x, float y)
{
ASSERT(image);
......@@ -936,13 +951,6 @@ void CanvasRenderingContext2D::drawImage(HTMLImageElement* image,
drawImage(image, FloatRect(0, 0, s.width(), s.height()), FloatRect(x, y, width, height), ec);
}
void CanvasRenderingContext2D::checkOrigin(const KURL& url)
{
RefPtr<SecurityOrigin> origin = SecurityOrigin::create(url);
if (!m_canvas->document()->securityOrigin()->canAccess(origin.get()))
m_canvas->setOriginTainted();
}
void CanvasRenderingContext2D::drawImage(HTMLImageElement* image, const FloatRect& srcRect, const FloatRect& dstRect,
ExceptionCode& ec)
{
......@@ -1033,6 +1041,62 @@ void CanvasRenderingContext2D::drawImage(HTMLCanvasElement* canvas, const FloatR
// FIXME: Arguably willDraw should become didDraw and occur after drawing calls and not before them to avoid problems like this.
}
void CanvasRenderingContext2D::drawImage(HTMLVideoElement* video, float x, float y)
{
ASSERT(video);
IntSize s = size(video);
ExceptionCode ec;
drawImage(video, x, y, s.width(), s.height(), ec);
}
void CanvasRenderingContext2D::drawImage(HTMLVideoElement* video,
float x, float y, float width, float height, ExceptionCode& ec)
{
ASSERT(video);
IntSize s = size(video);
drawImage(video, FloatRect(0, 0, s.width(), s.height()), FloatRect(x, y, width, height), ec);
}
void CanvasRenderingContext2D::drawImage(HTMLVideoElement* video, const FloatRect& srcRect, const FloatRect& dstRect,
ExceptionCode& ec)
{
ASSERT(video);
ec = 0;
FloatRect videoRect = FloatRect(FloatPoint(), size(video));
if (!videoRect.contains(normalizeRect(srcRect)) || srcRect.width() == 0 || srcRect.height() == 0) {
ec = INDEX_SIZE_ERR;
return;
}
if (!dstRect.width() || !dstRect.height())
return;
GraphicsContext* c = drawingContext();
if (!c)
return;
if (!state().m_invertibleCTM)
return;
if (m_canvas->originClean())
checkOrigin(video->src());
if (m_canvas->originClean() && !video->hasSingleSecurityOrigin())
m_canvas->setOriginTainted();
FloatRect sourceRect = c->roundToDevicePixels(srcRect);
FloatRect destRect = c->roundToDevicePixels(dstRect);
willDraw(destRect);
c->save();
c->clip(destRect);
c->translate(destRect.x(), destRect.y());
c->scale(FloatSize(destRect.width()/sourceRect.width(), destRect.height()/sourceRect.height()));
c->translate(-sourceRect.x(), -sourceRect.y());
video->paint(c, IntRect(IntPoint(), size(video)));
c->restore();
}
// FIXME: Why isn't this just another overload of drawImage? Why have a different name?
void CanvasRenderingContext2D::drawImageFromRect(HTMLImageElement* image,
float sx, float sy, float sw, float sh,
......
/*
* Copyright (C) 2006, 2007 Apple Inc. All rights reserved.
* Copyright (C) 2006, 2007, 2009 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
......@@ -47,6 +47,7 @@ namespace WebCore {
class GraphicsContext;
class HTMLCanvasElement;
class HTMLImageElement;
class HTMLVideoElement;
class ImageData;
class KURL;
class TextMetrics;
......@@ -159,6 +160,9 @@ namespace WebCore {
void drawImage(HTMLCanvasElement*, float x, float y);
void drawImage(HTMLCanvasElement*, float x, float y, float width, float height, ExceptionCode&);
void drawImage(HTMLCanvasElement*, const FloatRect& srcRect, const FloatRect& dstRect, ExceptionCode&);
void drawImage(HTMLVideoElement*, float x, float y);
void drawImage(HTMLVideoElement*, float x, float y, float width, float height, ExceptionCode&);
void drawImage(HTMLVideoElement*, const FloatRect& srcRect, const FloatRect& dstRect, ExceptionCode&);
void drawImageFromRect(HTMLImageElement*, float sx, float sy, float sw, float sh,
float dx, float dy, float dw, float dh, const String& compositeOperation);
......
......@@ -143,6 +143,8 @@ public:
virtual void finishParsingChildren();
#endif
bool hasSingleSecurityOrigin() const { return !m_player || m_player->hasSingleSecurityOrigin(); }
protected:
float getTimeOffsetAttribute(const QualifiedName&, float valueOnError) const;
void setTimeOffsetAttribute(const QualifiedName&, float value);
......
/*
* Copyright (C) 2007, 2008 Apple Inc. All rights reserved.
* Copyright (C) 2007, 2008, 2009 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
......@@ -186,5 +186,17 @@ void HTMLVideoElement::updatePosterImage()
#endif
}
void HTMLVideoElement::paint(GraphicsContext* context, const IntRect& r)
{
// FIXME: We should also be able to paint the poster image.
MediaPlayer* player = HTMLMediaElement::player();
if (!player)
return;
player->setVisible(true); // Make player visible or it won't draw.
player->paint(context, r);
}
}
#endif
/*
* Copyright (C) 2007, 2008 Apple Inc. All rights reserved.
* Copyright (C) 2007, 2008, 2009 Apple Inc. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
......@@ -66,6 +66,9 @@ public:
void updatePosterImage();
// Used by canvas to gain raw pixel access
void paint(GraphicsContext*, const IntRect&);
private:
OwnPtr<HTMLImageLoader> m_imageLoader;
bool m_shouldShowPosterImage;
......
......@@ -103,6 +103,8 @@ public:
virtual void deliverNotification(MediaPlayerProxyNotificationType) { }
virtual void setMediaPlayerProxy(WebMediaPlayerProxy*) { }
#endif
virtual bool hasSingleSecurityOrigin() const { return true; }
};
static MediaPlayerPrivateInterface* createNullMediaPlayer(MediaPlayer* player)
......@@ -529,5 +531,10 @@ void MediaPlayer::rateChanged()
m_mediaPlayerClient->mediaPlayerRateChanged(this);
}
bool MediaPlayer::hasSingleSecurityOrigin() const
{
return m_private->hasSingleSecurityOrigin();
}
}
#endif
......@@ -191,6 +191,8 @@ public:
void acceleratedRenderingStateChanged();
#endif
bool hasSingleSecurityOrigin() const;
private:
static void initializeMediaEngines();
......
......@@ -99,6 +99,8 @@ public:
// called when the rendering system flips the into or out of accelerated rendering mode.
virtual void acceleratedRenderingStateChanged() { }
#endif
virtual bool hasSingleSecurityOrigin() const { return false; }
};
}
......
......@@ -68,6 +68,7 @@ public:
void timeChanged();
void didEnd();
bool hasSingleSecurityOrigin() const;
private:
MediaPlayerPrivate(MediaPlayer*);
......
......@@ -497,18 +497,17 @@ void MediaPlayerPrivate::setUpVideoRendering()
{
MediaRenderingMode currentMode = currentRenderingMode();
MediaRenderingMode preferredMode = preferredRenderingMode();
if (currentMode == preferredMode)
if (currentMode == preferredMode && currentMode != MediaRenderingNone)
return;
if (currentMode != MediaRenderingNone)
tearDownVideoRendering();
switch (preferredMode) {
case MediaRenderingNone:
break;
case MediaRenderingMovieView:
createQTMovieView();
break;
case MediaRenderingNone:
case MediaRenderingSoftwareRenderer:
createQTVideoRenderer();
break;
......@@ -1309,6 +1308,13 @@ void MediaPlayerPrivate::acceleratedRenderingStateChanged()
}
#endif
bool MediaPlayerPrivate::hasSingleSecurityOrigin() const
{
// We tell quicktime to disallow resources that come from different origins
// so we know all media is single origin.
return true;
}
} // namespace WebCore
@implementation WebCoreMovieObserver
......
......@@ -427,9 +427,33 @@ void MediaPlayerPrivate::paint(GraphicsContext* p, const IntRect& r)
{
if (p->paintingDisabled() || !m_qtMovie || m_hasUnsupportedTracks)
return;
bool usingTempBitmap = false;
OwnPtr<GraphicsContext::WindowsBitmap> bitmap;
HDC hdc = p->getWindowsContext(r);
if (!hdc) {
// The graphics context doesn't have an associated HDC so create a temporary
// bitmap where QTMovieWin can draw the frame and we can copy it.
usingTempBitmap = true;
bitmap.set(p->createWindowsBitmap(r.size()));
hdc = bitmap->hdc();
// FIXME: is this necessary??
XFORM xform;
xform.eM11 = 1.0f;
xform.eM12 = 0.0f;
xform.eM21 = 0.0f;
xform.eM22 = 1.0f;
xform.eDx = -r.x();
xform.eDy = -r.y();
SetWorldTransform(hdc, &xform);
}
m_qtMovie->paint(hdc, r.x(), r.y());
p->releaseWindowsContext(hdc, r);
if (usingTempBitmap)
p->drawWindowsBitmap(bitmap.get(), r.topLeft());
else
p->releaseWindowsContext(hdc, r);
#if DRAW_FRAME_RATE
if (m_frameCountWhilePlaying > 10) {
......@@ -541,6 +565,13 @@ void MediaPlayerPrivate::movieNewImageAvailable(QTMovieWin* movie)
m_player->repaint();
}
bool MediaPlayerPrivate::hasSingleSecurityOrigin() const
{
// We tell quicktime to disallow resources that come from different origins
// so we all media is single origin.
return true;
}
}
#endif
......
......@@ -90,6 +90,7 @@ public:
void paint(GraphicsContext*, const IntRect&);
bool hasSingleSecurityOrigin() const;
private:
MediaPlayerPrivate(MediaPlayer*);
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment