Commit 4fd3685a authored by alp@webkit.org's avatar alp@webkit.org

2007-12-16 Alp Toker <alp@atoker.com>

        Reviewed by Maciej.

        http://bugs.webkit.org/show_bug.cgi?id=16356
        [GTK] Integrate GStreamer video with the graphics backend

        Integrate the GStreamer media backend with the Cairo graphics backend.
        There are still some issues: Data is copied more often than necessary,
        and repaint() is not called, causing transformed video not to update
        sometimes.

        * WebCore.pro:
        * platform/graphics/gtk/MediaPlayerPrivateGStreamer.cpp:
        (WebCore::MediaPlayerPrivate::MediaPlayerPrivate):
        (WebCore::MediaPlayerPrivate::~MediaPlayerPrivate):
        (WebCore::MediaPlayerPrivate::currentTime):
        (WebCore::MediaPlayerPrivate::setEndTime):
        (WebCore::MediaPlayerPrivate::seeking):
        (WebCore::MediaPlayerPrivate::naturalSize):
        (WebCore::MediaPlayerPrivate::setMuted):
        (WebCore::MediaPlayerPrivate::setRect):
        (WebCore::MediaPlayerPrivate::setVisible):
        (WebCore::MediaPlayerPrivate::repaint):
        (WebCore::MediaPlayerPrivate::paint):
        (WebCore::MediaPlayerPrivate::createGSTPlayBin):
        * platform/graphics/gtk/MediaPlayerPrivateGStreamer.h:
        * platform/graphics/gtk/VideoSinkGStreamer.cpp: Added.
        (webkit_video_sink_base_init):
        (webkit_video_sink_init):
        (webkit_video_sink_idle_func):
        (webkit_video_sink_render):
        (webkit_video_sink_set_caps):
        (webkit_video_sink_dispose):
        (webkit_video_sink_finalize):
        (webkit_video_sink_set_property):
        (webkit_video_sink_get_property):
        (webkit_video_sink_stop):
        (webkit_video_sink_class_init):
        (webkit_video_sink_new):
        (webkit_video_sink_set_surface):
        (plugin_init):
        * platform/graphics/gtk/VideoSinkGStreamer.h: Added.


git-svn-id: http://svn.webkit.org/repository/webkit/trunk@28792 268f45cc-cd09-0410-ab3c-d52691b4dbfc
parent bdeb5cd5
2007-12-16 Alp Toker <alp@atoker.com>
Reviewed by Maciej.
http://bugs.webkit.org/show_bug.cgi?id=16356
[GTK] Integrate GStreamer video with the graphics backend
Integrate the GStreamer media backend with the Cairo graphics backend.
There are still some issues: Data is copied more often than necessary,
and repaint() is not called, causing transformed video not to update
sometimes.
* WebCore.pro:
* platform/graphics/gtk/MediaPlayerPrivateGStreamer.cpp:
(WebCore::MediaPlayerPrivate::MediaPlayerPrivate):
(WebCore::MediaPlayerPrivate::~MediaPlayerPrivate):
(WebCore::MediaPlayerPrivate::currentTime):
(WebCore::MediaPlayerPrivate::setEndTime):
(WebCore::MediaPlayerPrivate::seeking):
(WebCore::MediaPlayerPrivate::naturalSize):
(WebCore::MediaPlayerPrivate::setMuted):
(WebCore::MediaPlayerPrivate::setRect):
(WebCore::MediaPlayerPrivate::setVisible):
(WebCore::MediaPlayerPrivate::repaint):
(WebCore::MediaPlayerPrivate::paint):
(WebCore::MediaPlayerPrivate::createGSTPlayBin):
* platform/graphics/gtk/MediaPlayerPrivateGStreamer.h:
* platform/graphics/gtk/VideoSinkGStreamer.cpp: Added.
(webkit_video_sink_base_init):
(webkit_video_sink_init):
(webkit_video_sink_idle_func):
(webkit_video_sink_render):
(webkit_video_sink_set_caps):
(webkit_video_sink_dispose):
(webkit_video_sink_finalize):
(webkit_video_sink_set_property):
(webkit_video_sink_get_property):
(webkit_video_sink_stop):
(webkit_video_sink_class_init):
(webkit_video_sink_new):
(webkit_video_sink_set_surface):
(plugin_init):
* platform/graphics/gtk/VideoSinkGStreamer.h: Added.
2007-12-16 Mark Rowe <mrowe@apple.com>
Mac build fix.
......
......@@ -1111,7 +1111,8 @@ contains(DEFINES, ENABLE_VIDEO=1) {
gtk-port {
SOURCES += \
platform/graphics/gtk/MediaPlayerPrivateGStreamer.cpp
platform/graphics/gtk/MediaPlayerPrivateGStreamer.cpp \
platform/graphics/gtk/VideoSinkGStreamer.cpp
PKGCONFIG += gstreamer-0.10 gstreamer-plugins-base-0.10 gnome-vfs-2.0
LIBS += -lgstinterfaces-0.10 -lgstbase-0.10 -lgstvideo-0.10
......
/*
* Copyright (C) 2007 Apple Inc. All rights reserved.
* Copyright (C) 2007 Collabora Ltd. All rights reserved.
* Copyright (C) 2007 Alp Toker <alp@atoker.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
......@@ -23,8 +24,8 @@
#if ENABLE(VIDEO)
#include "MediaPlayerPrivateGStreamer.h"
#include "VideoSinkGStreamer.h"
#include "CString.h"
#include "CString.h"
#include "GraphicsContext.h"
#include "IntRect.h"
......@@ -116,15 +117,30 @@ MediaPlayerPrivate::MediaPlayerPrivate(MediaPlayer* player)
, m_readyState(MediaPlayer::DataUnavailable)
, m_startedPlaying(false)
, m_isStreaming(false)
, m_rect(IntRect())
, m_visible(true)
{
static bool gstInitialized = false;
// FIXME: We should pass the arguments from the command line
gst_init(0, NULL);
if (!gstInitialized) {
gst_init(0, NULL);
gstInitialized = true;
}
// FIXME: The size shouldn't be fixed here, this is just a quick hack.
m_surface = cairo_image_surface_create(CAIRO_FORMAT_ARGB32, 640, 480);
}
MediaPlayerPrivate::~MediaPlayerPrivate()
{
gst_element_set_state(m_playBin, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(m_playBin));
if (m_surface)
cairo_surface_destroy(m_surface);
if (m_playBin) {
gst_element_set_state(m_playBin, GST_STATE_NULL);
gst_object_unref(GST_OBJECT(m_playBin));
}
}
void MediaPlayerPrivate::load(String url)
......@@ -192,12 +208,9 @@ float MediaPlayerPrivate::currentTime() const
return m_endTime;
float ret;
GstQuery* query;
gboolean res;
query = gst_query_new_position(GST_FORMAT_TIME);
res = gst_element_query(m_playBin, query);
if (res) {
GstQuery* query = gst_query_new_position(GST_FORMAT_TIME);
if (gst_element_query(m_playBin, query)) {
gint64 position;
gst_query_parse_position(query, NULL, &position);
ret = (float) (position / 1000000000.0);
......@@ -207,6 +220,7 @@ float MediaPlayerPrivate::currentTime() const
ret = 0.0;
}
gst_query_unref(query);
return ret;
}
......@@ -242,7 +256,7 @@ void MediaPlayerPrivate::setEndTime(float time)
GstClockTime end = (GstClockTime)(time * GST_SECOND);
LOG_VERBOSE(Media, "setEndTime: %" GST_TIME_FORMAT, GST_TIME_ARGS(end));
// FIXME: What happens when the seeked position is not available?
if (!gst_element_seek( m_playBin, m_rate,
if (!gst_element_seek(m_playBin, m_rate,
GST_FORMAT_TIME,
(GstSeekFlags)(GST_SEEK_FLAG_FLUSH | GST_SEEK_FLAG_ACCURATE),
GST_SEEK_TYPE_SET, start,
......@@ -273,19 +287,21 @@ bool MediaPlayerPrivate::paused() const
bool MediaPlayerPrivate::seeking() const
{
return false;;
return false;
}
// Returns the size of the video
IntSize MediaPlayerPrivate::naturalSize()
{
if (!hasVideo())
return IntSize();
int x = 0, y = 0;
if (hasVideo()) {
GstPad* pad = NULL;
pad = gst_element_get_pad(m_videoSink, "sink");
if (pad)
gst_video_get_size(GST_PAD(pad), &x, &y);
if (GstPad* pad = gst_element_get_static_pad(m_videoSink, "sink")) {
gst_video_get_size(GST_PAD(pad), &x, &y);
gst_object_unref(GST_OBJECT(pad));
}
return IntSize(x, y);
}
......@@ -306,7 +322,7 @@ void MediaPlayerPrivate::setVolume(float volume)
void MediaPlayerPrivate::setMuted(bool b)
{
if (!m_playBin)
if (!m_playBin)
return;
if (b) {
......@@ -537,23 +553,39 @@ void MediaPlayerPrivate::loadingFailed()
}
}
void MediaPlayerPrivate::setRect(const IntRect& r)
void MediaPlayerPrivate::setRect(const IntRect& rect)
{
notImplemented();
m_rect = rect;
}
void MediaPlayerPrivate::setVisible(bool b)
void MediaPlayerPrivate::setVisible(bool visible)
{
notImplemented();
m_visible = visible;
}
void MediaPlayerPrivate::paint(GraphicsContext* p, const IntRect& r)
void MediaPlayerPrivate::repaint()
{
// FIXME: do the real thing
if (p->paintingDisabled())
m_player->repaint();
}
void MediaPlayerPrivate::paint(GraphicsContext* context, const IntRect& rect)
{
if (context->paintingDisabled())
return;
// For now draw a placeholder rectangle
p->drawRect(r);
if (!m_visible)
return;
//TODO: m_rect vs rect?
cairo_t* cr = context->platformContext();
cairo_save(cr);
cairo_set_operator(cr, CAIRO_OPERATOR_SOURCE);
cairo_translate(cr, rect.x(), rect.y());
cairo_rectangle(cr, 0, 0, rect.width(), rect.height());
cairo_set_source_surface(cr, m_surface, 0, 0);
cairo_fill(cr);
cairo_restore(cr);
}
void MediaPlayerPrivate::getSupportedTypes(HashSet<String>& types)
......@@ -565,25 +597,21 @@ void MediaPlayerPrivate::getSupportedTypes(HashSet<String>& types)
void MediaPlayerPrivate::createGSTPlayBin(String url)
{
GstElement* audioSink;
GstBus* bus;
ASSERT(!m_playBin);
m_playBin = gst_element_factory_make("playbin", "play");
bus = gst_pipeline_get_bus(GST_PIPELINE(m_playBin));
GstBus* bus = gst_pipeline_get_bus(GST_PIPELINE(m_playBin));
gst_bus_add_signal_watch(bus);
g_signal_connect(bus, "message::error", G_CALLBACK(mediaPlayerPrivateErrorCallback), this);
g_signal_connect(bus, "message::eos", G_CALLBACK(mediaPlayerPrivateEOSCallback), this);
g_signal_connect(bus, "message::state-changed", G_CALLBACK(mediaPlayerPrivateStateCallback), this);
g_signal_connect(bus, "message::buffering", G_CALLBACK(mediaPlayerPrivateBufferingCallback), this);
gst_object_unref(bus);
g_object_set(G_OBJECT(m_playBin), "uri", url.utf8().data(), NULL);
audioSink = gst_element_factory_make("gconfaudiosink", NULL);
m_videoSink = gst_element_factory_make("gconfvideosink", NULL);
GstElement* audioSink = gst_element_factory_make("gconfaudiosink", NULL);
m_videoSink = webkit_video_sink_new(m_surface);
g_object_set(m_playBin, "audio-sink", audioSink, NULL);
g_object_set(m_playBin, "video-sink", m_videoSink, NULL);
......
/*
* Copyright (C) 2007 Apple Inc. All rights reserved.
* Copyright (C) 2007 Collabora Ltd. All rights reserved.
* Copyright (C) 2007 Alp Toker <alp@atoker.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
......@@ -25,7 +26,6 @@
#include "MediaPlayer.h"
#include "Timer.h"
#include "wtf/Noncopyable.h"
#include <gtk/gtk.h>
......@@ -51,7 +51,7 @@ namespace WebCore {
friend gboolean mediaPlayerPrivateStateCallback(GstBus* bus, GstMessage* message, gpointer data);
public:
MediaPlayerPrivate(MediaPlayer* m);
MediaPlayerPrivate(MediaPlayer*);
~MediaPlayerPrivate();
IntSize naturalSize();
......@@ -68,8 +68,8 @@ namespace WebCore {
float duration();
float currentTime() const;
void seek(float time);
void setEndTime(float time);
void seek(float);
void setEndTime(float);
void setRate(float);
void setVolume(float);
......@@ -87,7 +87,7 @@ namespace WebCore {
unsigned totalBytes();
void setVisible(bool);
void setRect(const IntRect& r);
void setRect(const IntRect&);
void loadStateChanged();
void rateChanged();
......@@ -97,8 +97,9 @@ namespace WebCore {
void didEnd();
void loadingFailed();
void paint(GraphicsContext* p, const IntRect& r);
static void getSupportedTypes(HashSet<String>& types);
void repaint();
void paint(GraphicsContext*, const IntRect&);
static void getSupportedTypes(HashSet<String>&);
private:
......@@ -123,6 +124,9 @@ namespace WebCore {
MediaPlayer::ReadyState m_readyState;
bool m_startedPlaying;
bool m_isStreaming;
IntRect m_rect;
bool m_visible;
cairo_surface_t* m_surface;
};
}
......
/*
* Copyright (C) 2007 OpenedHand
* Copyright (C) 2007 Alp Toker <alp@atoker.com>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* SECTION:webkit-video-sink
* @short_description: GStreamer video sink
*
* #WebKitVideoSink is a GStreamer sink element that sends
* data to a #cairo_surface_t.
*/
#include "config.h"
#include "VideoSinkGStreamer.h"
#include <glib.h>
#include <gst/gst.h>
#include <gst/video/video.h>
static GstStaticPadTemplate sinktemplate = GST_STATIC_PAD_TEMPLATE("sink",
GST_PAD_SINK, GST_PAD_ALWAYS,
GST_STATIC_CAPS(GST_VIDEO_CAPS_RGBx ";" GST_VIDEO_CAPS_BGRx));
GST_DEBUG_CATEGORY_STATIC(webkit_video_sink_debug);
#define GST_CAT_DEFAULT webkit_video_sink_debug
static GstElementDetails webkit_video_sink_details =
GST_ELEMENT_DETAILS("WebKit video sink",
"Sink/Video",
"Sends video data from a GStreamer pipeline to a Cairo surface",
"Alp Toker <alp@atoker.com>");
enum {
PROP_0,
PROP_SURFACE
};
struct _WebKitVideoSinkPrivate {
cairo_surface_t* surface;
GAsyncQueue* async_queue;
gboolean rgb_ordering;
int width;
int height;
int fps_n;
int fps_d;
int par_n;
int par_d;
};
#define _do_init(bla) \
GST_DEBUG_CATEGORY_INIT (webkit_video_sink_debug, \
"webkitsink", \
0, \
"webkit video sink")
GST_BOILERPLATE_FULL(WebKitVideoSink,
webkit_video_sink,
GstBaseSink,
GST_TYPE_BASE_SINK,
_do_init);
static void
webkit_video_sink_base_init(gpointer g_class)
{
GstElementClass* element_class = GST_ELEMENT_CLASS(g_class);
gst_element_class_add_pad_template(element_class, gst_static_pad_template_get(&sinktemplate));
gst_element_class_set_details(element_class, &webkit_video_sink_details);
}
static void
webkit_video_sink_init(WebKitVideoSink* sink, WebKitVideoSinkClass* klass)
{
WebKitVideoSinkPrivate* priv;
sink->priv = priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);
priv->async_queue = g_async_queue_new();
}
static gboolean
webkit_video_sink_idle_func(gpointer data)
{
WebKitVideoSinkPrivate* priv;
GstBuffer* buffer;
priv = (WebKitVideoSinkPrivate*)data;
if (!priv->async_queue)
return FALSE;
buffer = (GstBuffer*)g_async_queue_try_pop(priv->async_queue);
if (buffer == NULL || G_UNLIKELY(!GST_IS_BUFFER(buffer)))
return FALSE;
// TODO: consider priv->rgb_ordering?
cairo_surface_t* src = cairo_image_surface_create_for_data(GST_BUFFER_DATA(buffer), CAIRO_FORMAT_RGB24, priv->width, priv->height, (4 * priv->width + 3) & ~ 3);
// TODO: We copy the data twice right now. This could be easily improved.
cairo_t* cr = cairo_create(priv->surface);
cairo_set_operator(cr, CAIRO_OPERATOR_SOURCE);
cairo_set_source_surface(cr, src, 0, 0);
cairo_surface_destroy(src);
cairo_rectangle(cr, 0, 0, priv->width, priv->height);
cairo_fill(cr);
cairo_destroy(cr);
gst_buffer_unref(buffer);
return FALSE;
}
static GstFlowReturn
webkit_video_sink_render(GstBaseSink* bsink, GstBuffer* buffer)
{
WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(bsink);
WebKitVideoSinkPrivate* priv = sink->priv;
g_async_queue_push(priv->async_queue, gst_buffer_ref(buffer));
g_idle_add_full(G_PRIORITY_HIGH_IDLE, webkit_video_sink_idle_func, priv, NULL);
return GST_FLOW_OK;
}
static gboolean
webkit_video_sink_set_caps(GstBaseSink* bsink, GstCaps* caps)
{
WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(bsink);
WebKitVideoSinkPrivate* priv = sink->priv;
GstStructure* structure;
gboolean ret;
const GValue* fps;
const GValue* par;
gint width, height;
int red_mask;
GstCaps* intersection = gst_caps_intersect(gst_static_pad_template_get_caps(&sinktemplate), caps);
if (gst_caps_is_empty(intersection))
return FALSE;
gst_caps_unref(intersection);
structure = gst_caps_get_structure(caps, 0);
ret = gst_structure_get_int(structure, "width", &width);
ret &= gst_structure_get_int(structure, "height", &height);
fps = gst_structure_get_value(structure, "framerate");
ret &= (fps != NULL);
par = gst_structure_get_value(structure, "pixel-aspect-ratio");
if (!ret)
return FALSE;
priv->width = width;
priv->height = height;
/* We dont yet use fps or pixel aspect into but handy to have */
priv->fps_n = gst_value_get_fraction_numerator(fps);
priv->fps_d = gst_value_get_fraction_denominator(fps);
if (par) {
priv->par_n = gst_value_get_fraction_numerator(par);
priv->par_d = gst_value_get_fraction_denominator(par);
} else
priv->par_n = priv->par_d = 1;
gst_structure_get_int(structure, "red_mask", &red_mask);
priv->rgb_ordering = (red_mask == 0xff000000);
return TRUE;
}
static void
webkit_video_sink_dispose(GObject* object)
{
WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
WebKitVideoSinkPrivate* priv = sink->priv;
if (priv->surface) {
cairo_surface_destroy(priv->surface);
priv->surface = NULL;
}
if (priv->async_queue) {
g_async_queue_unref(priv->async_queue);
priv->async_queue = NULL;
}
G_OBJECT_CLASS(parent_class)->dispose(object);
}
static void
webkit_video_sink_finalize(GObject* object)
{
WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
G_OBJECT_CLASS(parent_class)->finalize(object);
}
static void
webkit_video_sink_set_property(GObject* object, guint prop_id, const GValue* value, GParamSpec* pspec)
{
WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
WebKitVideoSinkPrivate* priv = sink->priv;
switch (prop_id) {
case PROP_SURFACE:
if (priv->surface)
cairo_surface_destroy(priv->surface);
priv->surface = cairo_surface_reference((cairo_surface_t*)g_value_get_pointer(value));
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
}
}
static void
webkit_video_sink_get_property(GObject* object, guint prop_id, GValue* value, GParamSpec* pspec)
{
WebKitVideoSink* sink = WEBKIT_VIDEO_SINK(object);
switch (prop_id) {
case PROP_SURFACE:
g_value_set_pointer(value, sink->priv->surface);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec);
break;
}
}
static gboolean
webkit_video_sink_stop(GstBaseSink* base_sink)
{
WebKitVideoSinkPrivate* priv = WEBKIT_VIDEO_SINK(base_sink)->priv;
g_async_queue_lock(priv->async_queue);
/* Remove all remaining objects from the queue */
while(GstBuffer* buffer = (GstBuffer*)g_async_queue_try_pop_unlocked(priv->async_queue))
gst_buffer_unref(buffer);
g_async_queue_unlock(priv->async_queue);
return TRUE;
}
static void
webkit_video_sink_class_init(WebKitVideoSinkClass* klass)
{
GObjectClass* gobject_class = G_OBJECT_CLASS(klass);
GstBaseSinkClass* gstbase_sink_class = GST_BASE_SINK_CLASS(klass);
g_type_class_add_private(klass, sizeof(WebKitVideoSinkPrivate));
gobject_class->set_property = webkit_video_sink_set_property;
gobject_class->get_property = webkit_video_sink_get_property;
gobject_class->dispose = webkit_video_sink_dispose;
gobject_class->finalize = webkit_video_sink_finalize;
gstbase_sink_class->render = webkit_video_sink_render;
gstbase_sink_class->preroll = webkit_video_sink_render;
gstbase_sink_class->stop = webkit_video_sink_stop;
gstbase_sink_class->set_caps = webkit_video_sink_set_caps;
g_object_class_install_property(
gobject_class, PROP_SURFACE,
g_param_spec_pointer("surface", "surface", "Target cairo_surface_t*",
(GParamFlags)(G_PARAM_READWRITE)));
}
/**
* webkit_video_sink_new:
* @surface: a #cairo_surface_t
*
* Creates a new GStreamer video sink which uses @surface as the target
* for sinking a video stream from GStreamer.
*
* Return value: a #GstElement for the newly created video sink
*/
GstElement*
webkit_video_sink_new(cairo_surface_t* surface)
{
return (GstElement*)g_object_new(WEBKIT_TYPE_VIDEO_SINK, "surface", surface, NULL);
}
void
webkit_video_sink_set_surface(WebKitVideoSink* sink, cairo_surface_t* surface)
{
WebKitVideoSinkPrivate* priv;
sink->priv = priv = G_TYPE_INSTANCE_GET_PRIVATE(sink, WEBKIT_TYPE_VIDEO_SINK, WebKitVideoSinkPrivate);