I'm experimenting with a new feature for Pitivi: content-based auto-alignment. The attached patch is a (very) rough draft of the feature. Unfortunately, it doesn't work, which is why I'm asking for help here.
The code includes Extractor classes that are cargo-culted directly from the Previewer classes of previewer.py. The current problem is that RandomAccessAudioExtractor._startSegment() fails to seek. This is surprising because it is copied directly from RandomAccessAudioPreviewer._startThumbnail(), which works fine. Can anyone guess what step I might be missing that would cause a seek (to time zero!) to fail? Thanks, Ben
diff --git a/pitivi/timeline/align.py b/pitivi/timeline/align.py
new file mode 100644
index 0000000..16afccc
--- /dev/null
+++ b/pitivi/timeline/align.py
@@ -0,0 +1,77 @@
+import numpy
+from pitivi.timeline.extract import Extractee, RandomAccessAudioExtractor
+import pitivi.instance as instance
+from pitivi.stream import AudioStream
+
+def nextpow2(n):
+ i = 1
+ while i < n:
+ i *= 2
+ return i
+
+class EnvelopeExtractee(Extractee):
+
+ def __init__(self, blocksize, callback, *cbargs):
+ self._blocksize = blocksize
+ self._cb = callback
+ self._cbargs = cbargs
+ self._chunks = []
+ self._leftover = numpy.zeros((0,))
+
+ def receive(self, a):
+ if len(self._leftover) > 0:
+ a = numpy.concatenate((self._leftover, a))
+ lol = len(a) % self._blocksize
+ if lol > 0:
+ self._leftover = a[-lol:]
+ a = a[:-lol]
+ a = numpy.abs(a).reshape((len(a)//self._blocksize, self._blocksize))
+ a = numpy.sum(a)
+ self._chunks.append(a)
+
+ def finalize(self):
+ self._cb(numpy.concatenate(self._chunks), *cbargs)
+
+class AutoAligner:
+ BLOCKRATE = 25
+
+ @staticmethod
+ def _getAudioTrack(to):
+ for track in to.track_objects:
+ if track.stream_type == AudioStream:
+ return track
+ return None
+
+ def __init__(self, tobjects, callback):
+ self._tobjects = list(tobjects)
+ self._envelopes = [None] * len(tobjects)
+ self._callback = callback
+
+ def _envelopeCb(self, array, i):
+ self._envelopes[i] = array
+ if not None in self._envelopes:
+ self._performShifts()
+
+ def start(self):
+ for i in xrange(len(self._tobjects)):
+ a = self._getAudioTrack(self._tobjects[i])
+ blocksize = a.stream.channels * a.stream.rate // self.BLOCKRATE
+ e = EnvelopeExtractee(blocksize, self._envelopeCb, i)
+ r = RandomAccessAudioExtractor(instance.PiTiVi, a.factory, a.stream)
+ r.extract(e, a.in_point, a.out_point - a.in_point)
+
+ def _performShifts(self):
+ L = len(self._envelopes[0]) + max(len(e) for e in self._envelopes[1:]) - 1
+ L = nextpow2(L)
+ template = self._envelopes[0]
+ template -= numpy.mean(template)
+ template = numpy.fft.rfft(template, L).conj()
+ for i in xrange(len(self._tobjects)):
+ e = self._envelopes[i]
+ e -= numpy.mean(e)
+ xcorr = numpy.fft.irfft(template*numpy.fft.rfft(e, L))
+ p = (L - 1) - numpy.argmax(xcorr)
+ if p > len(self._envelopes[0]):
+ p -= L
+ tshift = (p * 1e9)//self.BLOCKRATE
+ self._tobjects[i].start = self._tobjects[0].start + tshift
diff --git a/pitivi/timeline/extract.py b/pitivi/timeline/extract.py
new file mode 100644
index 0000000..fdec908
--- /dev/null
+++ b/pitivi/timeline/extract.py
@@ -0,0 +1,113 @@
+import gst
+from pitivi.elements.singledecodebin import SingleDecodeBin
+from pitivi.elements.arraysink import ArraySink
+from pitivi.log.loggable import Loggable
+import pitivi.utils as utils
+
+
+class Extractee:
+ def receive(self, array):
+ raise NotImplementedError
+
+ def finalize(self):
+ raise NotImplementedError
+
+class Extractor(Loggable):
+
+ def __init__(self, instance, factory, stream_):
+ Loggable.__init__(self)
+
+ def extract(self, e, start, duration):
+ raise NotImplementedError
+
+class RandomAccessExtractor(Extractor):
+
+ def __init__(self, instance, factory, stream_):
+ Extractor.__init__(self, instance, factory, stream_)
+ # FIXME:
+ # why doesn't this work?
+ # bin = factory.makeBin(stream_)
+ uri = factory.uri
+ caps = stream_.caps
+ bin = SingleDecodeBin(uri=uri, caps=caps, stream=stream_)
+
+ self._pipelineInit(factory, bin)
+
+ def _pipelineInit(self, factory, bin):
+ """Create the pipeline for the preview process. Subclasses should
+ override this method and create a pipeline, connecting to callbacks to
+ the appropriate signals, and prerolling the pipeline if necessary."""
+ raise NotImplementedError
+
+class RandomAccessAudioExtractor(RandomAccessExtractor):
+
+ def __init__(self, instance, factory, stream_):
+ self.tdur = 30 * gst.SECOND
+ self._queue = []
+ RandomAccessExtractor.__init__(self, instance, factory, stream_)
+
+ def _pipelineInit(self, factory, sbin):
+ self.spacing = 0
+
+ self.audioSink = ArraySink()
+ conv = gst.element_factory_make("audioconvert")
+ self.audioPipeline = utils.pipeline({
+ sbin : conv,
+ conv : self.audioSink,
+ self.audioSink : None})
+ bus = self.audioPipeline.get_bus()
+ bus.add_signal_watch()
+ bus.connect("message::segment-done", self._busMessageSegmentDoneCb)
+ bus.connect("message::error", self._busMessageErrorCb)
+
+ self._audio_cur = None
+ self.audioPipeline.set_state(gst.STATE_PAUSED)
+
+ def _busMessageSegmentDoneCb(self, bus, message):
+ self.debug("segment done")
+ self._finishSegment()
+
+ def _busMessageErrorCb(self, bus, message):
+ error, debug = message.parse_error()
+ print "Event bus error:", str(error), str(debug)
+
+ return gst.BUS_PASS
+
+ def _startSegment(self, timestamp, duration):
+ self.debug("processing segment with timestamp=%i and duration=%i" % (timestamp, duration))
+ self._audio_cur = timestamp, duration
+ res = self.audioPipeline.seek(1.0,
+ gst.FORMAT_TIME,
+ gst.SEEK_FLAG_FLUSH | gst.SEEK_FLAG_ACCURATE | gst.SEEK_FLAG_SEGMENT,
+ gst.SEEK_TYPE_SET, timestamp,
+ gst.SEEK_TYPE_SET, timestamp + duration)
+ if not res:
+ self.warning("seek failed %s", timestamp)
+ self.audioPipeline.set_state(gst.STATE_PLAYING)
+
+ return res
+
+ def _finishSegment(self):
+ samples = self.audioSink.samples
+ e, start, duration = self._queue[0]
+ e.receive(samples)
+ self.audioSink.reset()
+ start += self.tdur
+ duration -= self.tdur
+ if duration > 0:
+ self._queue[0] = (e, start, duration)
+ else:
+ self._queue.pop(0)
+ if len(self._queue) > 0:
+ self._run()
+
+ def extract(self, e, start, duration):
+ stopped = len(self._queue) == 0
+ self._queue.append((e, start, duration))
+ if stopped:
+ self._run()
+
+ def _run(self):
+ e, start, duration = self._queue[0]
+ self._startSegment(start, min(duration, self.tdur))
+
diff --git a/pitivi/timeline/timeline.py b/pitivi/timeline/timeline.py
index 5bd21c5..6fbec2c 100644
--- a/pitivi/timeline/timeline.py
+++ b/pitivi/timeline/timeline.py
@@ -32,6 +32,7 @@ from pitivi.utils import start_insort_right, infinity, getPreviousObject, \
getNextObject
from pitivi.timeline.gap import Gap, SmallestGapsFinder, invalid_gap
from pitivi.stream import VideoStream
+from pitivi.timeline.align import AutoAligner
# Selection modes
SELECT = 0
@@ -1940,6 +1941,11 @@ class Timeline(Signallable, Loggable):
self.selection.setSelection(new_track_objects, SELECT_ADD)
+ def alignSelection(self):
+ a = AutoAligner(self.selection.selected, self.enableUpdates)
+ self.disableUpdates()
+ a.start()
+
def deleteSelection(self):
"""
Removes all the currently selected L{TimelineObject}s from the Timeline.
diff --git a/pitivi/ui/timeline.py b/pitivi/ui/timeline.py
index c75ccf0..d4a8ba1 100644
--- a/pitivi/ui/timeline.py
+++ b/pitivi/ui/timeline.py
@@ -64,6 +64,7 @@ UNLINK = _("Break links between clips")
LINK = _("Link together arbitrary clips")
UNGROUP = _("Ungroup clips")
GROUP = _("Group clips")
+ALIGN = _("Align clips based on content")
SELECT_BEFORE = ("Select all sources before selected")
SELECT_AFTER = ("Select all after selected")
@@ -87,6 +88,7 @@ ui = '''
<menuitem action="UnlinkObj" />
<menuitem action="GroupObj" />
<menuitem action="UngroupObj" />
+ <menuitem action="AlignObj" />
<separator />
<menuitem action="Prevframe" />
<menuitem action="Nextframe" />
@@ -104,6 +106,7 @@ ui = '''
<toolitem action="LinkObj" />
<toolitem action="GroupObj" />
<toolitem action="UngroupObj" />
+ <toolitem action="AlignObj" />
</placeholder>
</toolbar>
<accelerator action="DeleteObj" />
@@ -325,6 +328,8 @@ class Timeline(gtk.Table, Loggable, Zoomable):
self.ungroupSelected),
("GroupObj", "pitivi-group", None, "<Control>G", GROUP,
self.groupSelected),
+ ("AlignObj", "pitivi-group", None, "<Shift><Control>A", ALIGN,
+ self.alignSelected),
)
self.playhead_actions = (
@@ -349,6 +354,7 @@ class Timeline(gtk.Table, Loggable, Zoomable):
self.unlink_action = actiongroup.get_action("UnlinkObj")
self.group_action = actiongroup.get_action("GroupObj")
self.ungroup_action = actiongroup.get_action("UngroupObj")
+ self.align_action = actiongroup.get_action("AlignObj")
self.delete_action = actiongroup.get_action("DeleteObj")
self.split_action = actiongroup.get_action("Split")
self.keyframe_action = actiongroup.get_action("Keyframe")
@@ -716,6 +722,7 @@ class Timeline(gtk.Table, Loggable, Zoomable):
unlink = False
group = False
ungroup = False
+ align = False
split = False
keyframe = False
if timeline.selection:
@@ -723,6 +730,7 @@ class Timeline(gtk.Table, Loggable, Zoomable):
if len(timeline.selection) > 1:
link = True
group = True
+ align = True
start = None
duration = None
@@ -751,6 +759,7 @@ class Timeline(gtk.Table, Loggable, Zoomable):
self.unlink_action.set_sensitive(unlink)
self.group_action.set_sensitive(group)
self.ungroup_action.set_sensitive(ungroup)
+ self.align_action.set_sensitive(align)
self.split_action.set_sensitive(split)
self.keyframe_action.set_sensitive(keyframe)
@@ -794,6 +803,10 @@ class Timeline(gtk.Table, Loggable, Zoomable):
if self.timeline:
self.timeline.groupSelection()
+ def alignSelected(self, unused_action):
+ if self.timeline:
+ self.timeline.alignSelection()
+
def split(self, action):
self.app.action_log.begin("split")
self.timeline.disableUpdates()
signature.asc
Description: OpenPGP digital signature
------------------------------------------------------------------------------ EditLive Enterprise is the world's most technically advanced content authoring tool. Experience the power of Track Changes, Inline Image Editing and ensure content is compliant with Accessibility Checking. http://p.sf.net/sfu/ephox-dev2dev
_______________________________________________ Pitivi-pitivi mailing list Pitivi-pitivi@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/pitivi-pitivi
