javascript 以编程方式与ipywidget交互Jupyter笔记本中的视频小部件

ss2ws0br  于 2023-02-07  发布在  Java
关注(0)|答案(1)|浏览(151)

我试图构建一个Jupyter笔记本解决方案,用于视频数据集的离群值分析。我想使用Video小部件来实现此目的,但我在文档中没有找到如何通过调用一些小部件的方法来获取当前视频帧和/或滚动到所需位置。我的问题与这些未回答的问题onetwo非常相似(几乎相同)。
我通过将视频帧保存到numpy数组并使用matplotlib的imshow函数来实现这个想法,但视频播放非常抖动。我使用了位块传输技术来获得一些额外的fps,但没有多大帮助,相比之下,Video小部件产生了更流畅的体验。看起来Video小部件本质上是浏览器内置视频播放器的 Package 器。

    • 问题**:如何通过编程来控制widget的播放,以便同步多个widget?一些%%javascript魔术可以帮助实现IPython.display交互吗?

下面是一个Python代码片段,仅用于说明目的,让您了解我想要实现的目标。

%matplotlib widget

from videoreader import VideoReader # nice pithonic wrapper for video reading with opencv
import numpy as np
import matplotlib.pyplot as plt
from ipywidgets import IntSlider, Play, link, HBox, VBox 

# prepare buffered video frames
vr = VideoReader('Big.Buck.Bunny.mp4')

fps = vr.frame_rate
frames = []
for frame in vr[0:300:1]:
    frames.append(frame[:,:,::-1]) # brg2rgb 
del vr

vi_buff = np.stack(frames, axis=0) # dimensions (T, H, W, C)  

# simulate random signal for demonstration purposes
t = np.linspace(0.0, vi_buff.shape[0], num=vi_buff.shape[0]*10)
s = np.sin(2*np.pi*t)*np.random.randn(vi_buff.shape[0]*10)

plt.ioff()
fig = plt.figure(figsize=(11, 8))
ax1 = plt.subplot2grid((6, 6), (0, 0), rowspan=2, colspan=3)
ax2 = plt.subplot2grid((6, 6), (0, 3), colspan=3)
ax3 = plt.subplot2grid((6, 6), (1, 3), colspan=3)
plt.ion()

# initial plots 
img = ax1.imshow(vi_buff[0,...])
l0 = ax2.plot(t, s)
l1 = ax3.plot(t, -s)

# initial scene
lo_y, hi_y = ax2.get_ybound()
ax2.set_xbound(lower=-12., upper=2.)
ax3.set_xbound(lower=-12., upper=2.)

def update_plot(change):
    val = change.new
    img.set_data(vi_buff[val,...])
    ax2.axis([val - 12, val + 2, lo_y, hi_y])
    ax3.axis([val - 12, val + 2, lo_y, hi_y])
    fig.canvas.draw_idle()

player = Play(
    value=0, #intial frame index
    min=0,
    max=vi_buff.shape[0]-1,
    step=1,
    interval=int(1/round(fps)*1000) #referesh interval in ms
)
fr_slider = IntSlider(
    value=0,
    min=0,
    max=vi_buff.shape[0]-1
)
fr_slider.observe(update_plot, names='value')

link((player,"value"), (fr_slider,"value"))

VBox([HBox([player, fr_slider]), fig.canvas])

uklbhaso

uklbhaso1#

我不得不用艰苦的方式学习它,并编写我自己的自定义视频小部件。感谢ipywidgets模块的作者,它不是从头开始的。我还发现,我想做的事情,可能可以用PyViz面板视频以更简单的方式完成。如果有人感兴趣,我会分享我的解决方案,它可以节省你学习前端和Backbone.js的时间。

    • 注意事项**:它不能在JupyterLab中运行,您需要切换到Jupyter经典模式。请在评论中分享解决方案,如果您知道如何修复:"Javascript错误:未定义require "

Python端的小部件模型

from traitlets import Unicode, Float, Bool
from ipywidgets import Video, register

@register
class VideoE(Video):
    _view_name   = Unicode('VideoEView').tag(sync=True)
    _view_module = Unicode('video_enhanced').tag(sync=True)
    _view_module_version = Unicode('0.1.1').tag(sync=True)
    
    playing = Bool(False, help="Sync point for play/pause operations").tag(sync=True)
    rate    = Float(1.0, help="Sync point for changing playback rate").tag(sync=True)
    time    = Float(0.0, help="Sync point for seek operation").tag(sync=True)
    
    @classmethod
    def from_file(cls, filename, **kwargs):
        return super(VideoE, cls).from_file(filename, **kwargs)

前端的小部件视图

%%javascript
require.undef('video_enhanced');

define('video_enhanced', ["@jupyter-widgets/controls"], function(widgets) {

    var VideoEView = widgets.VideoView.extend({
        
        events: {
            // update Model when event is generated on View side
            'pause'      : 'onPause',
            'play'       : 'onPlay',
            'ratechange' : 'onRatechange',
            'seeked'     : 'onSeeked',
        },
    
        initialize: function() {
            // propagate changes from Model to View
            this.listenTo(this.model, 'change:playing', this.onPlayingChanged); // play/pause
            this.listenTo(this.model, 'change:rate',    this.onRateChanged);    // playbackRate
            this.listenTo(this.model, 'change:time',    this.onTimeChanged);    // currentTime
        },
        
        // View -> Model
        onPause: function() {
            this.model.set('playing', false, {silent: true}); 
            this.model.save_changes();
        },
        // View -> Model
        onPlay: function() {
            this.model.set('playing', true, {silent: true}); 
            this.model.save_changes();
        },
        // Model -> View    
        onPlayingChanged: function() {
            if (this.model.get('playing')) {
                this.el.play();
            } else {
                this.el.pause();
            }
        },
        // View -> Model
        onRatechange: function() {
            this.model.set('rate', this.el.playbackRate, {silent: true}); 
            this.model.save_changes();
        },
        // Model -> View
        onRateChanged: function() {
            this.el.playbackRate = this.model.get('rate'); 
        },
        // View -> Model        
        onSeeked: function() {
            this.model.set('time', this.el.currentTime, {silent: true}); 
            this.model.save_changes();
        },
        // Model -> View
        onTimeChanged: function() {                  
            this.el.currentTime = this.model.get('time'); 
        },
    });

    return {
        VideoEView : VideoEView,
    }
});

我真正要做的是

from ipywidgets import link, Checkbox, VBox, HBox

class SyncManager():
    '''
    Syncing videos needs an explicit setting of "time" property or clicking on the progress bar, 
    since the property is not updated continuously, but on "seeked" event generated
    '''
    def __init__(self, video1, video2):
        self._links = []
        self._video1 = video1
        self._video2 = video2
        
        self.box = Checkbox(False, description='Synchronize videos')
        self.box.observe(self.handle_sync_changed, names=['value'])
        
    def handle_sync_changed(self, value):
        if value.new:
            for prop in ['playing', 'rate', 'time']:
                l = link((self._video1, prop), (self._video2, prop))
                self._links.append(l)
        else:
            for _link in self._links:
                _link.unlink()
            self._links.clear()

video1 = VideoE.from_file("images/Big.Buck.Bunny.mp4")
video1.autoplay = False
video1.loop = False
video1.width = 480

video2 = VideoE.from_file("images/Big.Buck.Bunny.mp4")
video2.autoplay = False
video2.loop = False
video2.width = 480

sync_m = SyncManager(video1, video2)

VBox([sync_m.box, HBox([video1, video2])])

相关问题