I'm experimenting with a new feature for Pitivi: content-based auto-alignment. The attached patch is a (very) rough draft of the feature. Unfortunately, it doesn't work, which is why I'm asking for help here.
The code includes Extractor classes that are cargo-culted directly from the Previewer classes of previewer.py. The current problem is that RandomAccessAudioExtractor._startSegment() fails to seek. This is surprising because it is copied directly from RandomAccessAudioPreviewer._startThumbnail(), which works fine. Can anyone guess what step I might be missing that would cause a seek (to time zero!) to fail? Thanks, Ben
diff --git a/pitivi/timeline/align.py b/pitivi/timeline/align.py new file mode 100644 index 0000000..16afccc --- /dev/null +++ b/pitivi/timeline/align.py @@ -0,0 +1,77 @@ +import numpy +from pitivi.timeline.extract import Extractee, RandomAccessAudioExtractor +import pitivi.instance as instance +from pitivi.stream import AudioStream + +def nextpow2(n): + i = 1 + while i < n: + i *= 2 + return i + +class EnvelopeExtractee(Extractee): + + def __init__(self, blocksize, callback, *cbargs): + self._blocksize = blocksize + self._cb = callback + self._cbargs = cbargs + self._chunks = [] + self._leftover = numpy.zeros((0,)) + + def receive(self, a): + if len(self._leftover) > 0: + a = numpy.concatenate((self._leftover, a)) + lol = len(a) % self._blocksize + if lol > 0: + self._leftover = a[-lol:] + a = a[:-lol] + a = numpy.abs(a).reshape((len(a)//self._blocksize, self._blocksize)) + a = numpy.sum(a) + self._chunks.append(a) + + def finalize(self): + self._cb(numpy.concatenate(self._chunks), *cbargs) + +class AutoAligner: + BLOCKRATE = 25 + + @staticmethod + def _getAudioTrack(to): + for track in to.track_objects: + if track.stream_type == AudioStream: + return track + return None + + def __init__(self, tobjects, callback): + self._tobjects = list(tobjects) + self._envelopes = [None] * len(tobjects) + self._callback = callback + + def _envelopeCb(self, array, i): + self._envelopes[i] = array + if not None in self._envelopes: + self._performShifts() + + def start(self): + for i in xrange(len(self._tobjects)): + a = self._getAudioTrack(self._tobjects[i]) + blocksize = a.stream.channels * a.stream.rate // self.BLOCKRATE + e = EnvelopeExtractee(blocksize, self._envelopeCb, i) + r = RandomAccessAudioExtractor(instance.PiTiVi, a.factory, a.stream) + r.extract(e, a.in_point, a.out_point - a.in_point) + + def _performShifts(self): + L = len(self._envelopes[0]) + max(len(e) for e in self._envelopes[1:]) - 1 + L = nextpow2(L) + template = self._envelopes[0] + template -= numpy.mean(template) + template = numpy.fft.rfft(template, L).conj() + for i in xrange(len(self._tobjects)): + e = self._envelopes[i] + e -= numpy.mean(e) + xcorr = numpy.fft.irfft(template*numpy.fft.rfft(e, L)) + p = (L - 1) - numpy.argmax(xcorr) + if p > len(self._envelopes[0]): + p -= L + tshift = (p * 1e9)//self.BLOCKRATE + self._tobjects[i].start = self._tobjects[0].start + tshift diff --git a/pitivi/timeline/extract.py b/pitivi/timeline/extract.py new file mode 100644 index 0000000..fdec908 --- /dev/null +++ b/pitivi/timeline/extract.py @@ -0,0 +1,113 @@ +import gst +from pitivi.elements.singledecodebin import SingleDecodeBin +from pitivi.elements.arraysink import ArraySink +from pitivi.log.loggable import Loggable +import pitivi.utils as utils + + +class Extractee: + def receive(self, array): + raise NotImplementedError + + def finalize(self): + raise NotImplementedError + +class Extractor(Loggable): + + def __init__(self, instance, factory, stream_): + Loggable.__init__(self) + + def extract(self, e, start, duration): + raise NotImplementedError + +class RandomAccessExtractor(Extractor): + + def __init__(self, instance, factory, stream_): + Extractor.__init__(self, instance, factory, stream_) + # FIXME: + # why doesn't this work? + # bin = factory.makeBin(stream_) + uri = factory.uri + caps = stream_.caps + bin = SingleDecodeBin(uri=uri, caps=caps, stream=stream_) + + self._pipelineInit(factory, bin) + + def _pipelineInit(self, factory, bin): + """Create the pipeline for the preview process. Subclasses should + override this method and create a pipeline, connecting to callbacks to + the appropriate signals, and prerolling the pipeline if necessary.""" + raise NotImplementedError + +class RandomAccessAudioExtractor(RandomAccessExtractor): + + def __init__(self, instance, factory, stream_): + self.tdur = 30 * gst.SECOND + self._queue = [] + RandomAccessExtractor.__init__(self, instance, factory, stream_) + + def _pipelineInit(self, factory, sbin): + self.spacing = 0 + + self.audioSink = ArraySink() + conv = gst.element_factory_make("audioconvert") + self.audioPipeline = utils.pipeline({ + sbin : conv, + conv : self.audioSink, + self.audioSink : None}) + bus = self.audioPipeline.get_bus() + bus.add_signal_watch() + bus.connect("message::segment-done", self._busMessageSegmentDoneCb) + bus.connect("message::error", self._busMessageErrorCb) + + self._audio_cur = None + self.audioPipeline.set_state(gst.STATE_PAUSED) + + def _busMessageSegmentDoneCb(self, bus, message): + self.debug("segment done") + self._finishSegment() + + def _busMessageErrorCb(self, bus, message): + error, debug = message.parse_error() + print "Event bus error:", str(error), str(debug) + + return gst.BUS_PASS + + def _startSegment(self, timestamp, duration): + self.debug("processing segment with timestamp=%i and duration=%i" % (timestamp, duration)) + self._audio_cur = timestamp, duration + res = self.audioPipeline.seek(1.0, + gst.FORMAT_TIME, + gst.SEEK_FLAG_FLUSH | gst.SEEK_FLAG_ACCURATE | gst.SEEK_FLAG_SEGMENT, + gst.SEEK_TYPE_SET, timestamp, + gst.SEEK_TYPE_SET, timestamp + duration) + if not res: + self.warning("seek failed %s", timestamp) + self.audioPipeline.set_state(gst.STATE_PLAYING) + + return res + + def _finishSegment(self): + samples = self.audioSink.samples + e, start, duration = self._queue[0] + e.receive(samples) + self.audioSink.reset() + start += self.tdur + duration -= self.tdur + if duration > 0: + self._queue[0] = (e, start, duration) + else: + self._queue.pop(0) + if len(self._queue) > 0: + self._run() + + def extract(self, e, start, duration): + stopped = len(self._queue) == 0 + self._queue.append((e, start, duration)) + if stopped: + self._run() + + def _run(self): + e, start, duration = self._queue[0] + self._startSegment(start, min(duration, self.tdur)) + diff --git a/pitivi/timeline/timeline.py b/pitivi/timeline/timeline.py index 5bd21c5..6fbec2c 100644 --- a/pitivi/timeline/timeline.py +++ b/pitivi/timeline/timeline.py @@ -32,6 +32,7 @@ from pitivi.utils import start_insort_right, infinity, getPreviousObject, \ getNextObject from pitivi.timeline.gap import Gap, SmallestGapsFinder, invalid_gap from pitivi.stream import VideoStream +from pitivi.timeline.align import AutoAligner # Selection modes SELECT = 0 @@ -1940,6 +1941,11 @@ class Timeline(Signallable, Loggable): self.selection.setSelection(new_track_objects, SELECT_ADD) + def alignSelection(self): + a = AutoAligner(self.selection.selected, self.enableUpdates) + self.disableUpdates() + a.start() + def deleteSelection(self): """ Removes all the currently selected L{TimelineObject}s from the Timeline. diff --git a/pitivi/ui/timeline.py b/pitivi/ui/timeline.py index c75ccf0..d4a8ba1 100644 --- a/pitivi/ui/timeline.py +++ b/pitivi/ui/timeline.py @@ -64,6 +64,7 @@ UNLINK = _("Break links between clips") LINK = _("Link together arbitrary clips") UNGROUP = _("Ungroup clips") GROUP = _("Group clips") +ALIGN = _("Align clips based on content") SELECT_BEFORE = ("Select all sources before selected") SELECT_AFTER = ("Select all after selected") @@ -87,6 +88,7 @@ ui = ''' <menuitem action="UnlinkObj" /> <menuitem action="GroupObj" /> <menuitem action="UngroupObj" /> + <menuitem action="AlignObj" /> <separator /> <menuitem action="Prevframe" /> <menuitem action="Nextframe" /> @@ -104,6 +106,7 @@ ui = ''' <toolitem action="LinkObj" /> <toolitem action="GroupObj" /> <toolitem action="UngroupObj" /> + <toolitem action="AlignObj" /> </placeholder> </toolbar> <accelerator action="DeleteObj" /> @@ -325,6 +328,8 @@ class Timeline(gtk.Table, Loggable, Zoomable): self.ungroupSelected), ("GroupObj", "pitivi-group", None, "<Control>G", GROUP, self.groupSelected), + ("AlignObj", "pitivi-group", None, "<Shift><Control>A", ALIGN, + self.alignSelected), ) self.playhead_actions = ( @@ -349,6 +354,7 @@ class Timeline(gtk.Table, Loggable, Zoomable): self.unlink_action = actiongroup.get_action("UnlinkObj") self.group_action = actiongroup.get_action("GroupObj") self.ungroup_action = actiongroup.get_action("UngroupObj") + self.align_action = actiongroup.get_action("AlignObj") self.delete_action = actiongroup.get_action("DeleteObj") self.split_action = actiongroup.get_action("Split") self.keyframe_action = actiongroup.get_action("Keyframe") @@ -716,6 +722,7 @@ class Timeline(gtk.Table, Loggable, Zoomable): unlink = False group = False ungroup = False + align = False split = False keyframe = False if timeline.selection: @@ -723,6 +730,7 @@ class Timeline(gtk.Table, Loggable, Zoomable): if len(timeline.selection) > 1: link = True group = True + align = True start = None duration = None @@ -751,6 +759,7 @@ class Timeline(gtk.Table, Loggable, Zoomable): self.unlink_action.set_sensitive(unlink) self.group_action.set_sensitive(group) self.ungroup_action.set_sensitive(ungroup) + self.align_action.set_sensitive(align) self.split_action.set_sensitive(split) self.keyframe_action.set_sensitive(keyframe) @@ -794,6 +803,10 @@ class Timeline(gtk.Table, Loggable, Zoomable): if self.timeline: self.timeline.groupSelection() + def alignSelected(self, unused_action): + if self.timeline: + self.timeline.alignSelection() + def split(self, action): self.app.action_log.begin("split") self.timeline.disableUpdates()
signature.asc
Description: OpenPGP digital signature
------------------------------------------------------------------------------ EditLive Enterprise is the world's most technically advanced content authoring tool. Experience the power of Track Changes, Inline Image Editing and ensure content is compliant with Accessibility Checking. http://p.sf.net/sfu/ephox-dev2dev
_______________________________________________ Pitivi-pitivi mailing list Pitivi-pitivi@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/pitivi-pitivi