Hello community, here is the log from the commit of package kdebase4-runtime for openSUSE:12.1 checked in at 2011-10-26 15:26:02 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:12.1/kdebase4-runtime (Old) and /work/SRC/openSUSE:12.1/.kdebase4-runtime.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "kdebase4-runtime", Maintainer is "kde-maintain...@suse.de" Changes: -------- --- /work/SRC/openSUSE:12.1/kdebase4-runtime/kdebase4-runtime.changes 2011-10-24 12:56:35.000000000 +0200 +++ /work/SRC/openSUSE:12.1/.kdebase4-runtime.new/kdebase4-runtime.changes 2011-10-28 15:39:06.000000000 +0200 @@ -1,0 +2,13 @@ +Tue Oct 25 11:03:58 UTC 2011 - wstephen...@suse.com + +- Cherrypicked patches from 4.7 branch for Nepomuk + - React if the Virtuoso server exits unexpectedly, don't spam to + stderr (bko#263730 + - Fix crash in NepomukServiceStub when indexing removable media + (bko#284602) + - Fix queries of files on removable media (bko#284529) + - Improve reliability of Nepomuk start after crash (bko#263730) + - Fix memory leak of dbus events (bko#226676) + - Improve reliability of reindexing changed files + +------------------------------------------------------------------- New: ---- 07438391-kderuntime-nepomuk-47branch-reindexing.diff 0f511184-kderuntime-nepomuk-47branch-memleak.diff 1b66b02e-kderuntime-nepomuk-47branch-partialurls.diff 348d7cbf-kderuntime-nepomuk-47branch-regexp.diff ab58993a-kderuntime-nepomuk-47branch-nullpointercrash.diff cc5cf57c-kderuntime-nepomuk-47branch-robust-restart.diff ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ kdebase4-runtime.spec ++++++ --- /var/tmp/diff_new_pack.sGeHt5/_old 2011-10-28 15:39:09.000000000 +0200 +++ /var/tmp/diff_new_pack.sGeHt5/_new 2011-10-28 15:39:09.000000000 +0200 @@ -42,6 +42,12 @@ Patch15: kdesu-symbol-lookup-workaround.diff Patch16: phonon-always-forget.diff Patch17: desktop-files.diff +Patch18: 348d7cbf-kderuntime-nepomuk-47branch-regexp.diff +Patch19: 1b66b02e-kderuntime-nepomuk-47branch-partialurls.diff +Patch20: ab58993a-kderuntime-nepomuk-47branch-nullpointercrash.diff +Patch21: 0f511184-kderuntime-nepomuk-47branch-memleak.diff +Patch22: 07438391-kderuntime-nepomuk-47branch-reindexing.diff +Patch23: cc5cf57c-kderuntime-nepomuk-47branch-robust-restart.diff BuildRequires: NetworkManager-devel BuildRequires: QtZeitgeist-devel BuildRequires: bluez-devel @@ -167,6 +173,12 @@ %patch15 %patch16 %patch17 +%patch18 -p1 +%patch19 -p1 +%patch20 -p1 +%patch21 -p1 +%patch22 -p1 +%patch23 -p1 %build %cmake_kde4 -d build -- -DKDE4_ENABLE_FPIE=1 ++++++ 07438391-kderuntime-nepomuk-47branch-reindexing.diff ++++++ commit 074383916dfdfb2cd5cf2b5dd269715229465366 Author: Sebastian Trueg <tr...@kde.org> Date: Mon Oct 24 20:48:07 2011 +0200 Always re-index files on close after write events. This is required since mmapped files do not create modification events while being written. diff --git a/nepomuk/services/filewatch/nepomukfilewatch.cpp b/nepomuk/services/filewatch/nepomukfilewatch.cpp index 41d7de7..1ec2db3 100644 --- a/nepomuk/services/filewatch/nepomukfilewatch.cpp +++ b/nepomuk/services/filewatch/nepomukfilewatch.cpp @@ -97,13 +97,9 @@ namespace { //Only watch the strigi index folders for file creation and change. if( Nepomuk::StrigiServiceConfig::self()->shouldFolderBeIndexed( path ) ) { - modes |= KInotify::EventCreate; - modes |= KInotify::EventModify; modes |= KInotify::EventCloseWrite; } else { - modes &= (~KInotify::EventCreate); - modes &= (~KInotify::EventModify); modes &= (~KInotify::EventCloseWrite); } @@ -150,10 +146,6 @@ Nepomuk::FileWatch::FileWatch( QObject* parent, const QList<QVariant>& ) this, SLOT( slotFileMoved( QString, QString ) ) ); connect( m_dirWatch, SIGNAL( deleted( QString, bool ) ), this, SLOT( slotFileDeleted( QString, bool ) ) ); - connect( m_dirWatch, SIGNAL( created( QString ) ), - this, SLOT( slotFileCreated( QString ) ) ); - connect( m_dirWatch, SIGNAL( modified( QString ) ), - this, SLOT( slotFileModified( QString ) ) ); connect( m_dirWatch, SIGNAL( closedWrite( QString ) ), this, SLOT( slotFileClosedAfterWrite( QString ) ) ); connect( m_dirWatch, SIGNAL( watchUserLimitReached() ), @@ -200,7 +192,7 @@ void Nepomuk::FileWatch::watchFolder( const QString& path ) #ifdef BUILD_KINOTIFY if ( m_dirWatch && !m_dirWatch->watchingPath( path ) ) m_dirWatch->addWatch( path, - KInotify::WatchEvents( KInotify::EventMove|KInotify::EventDelete|KInotify::EventDeleteSelf|KInotify::EventCreate|KInotify::EventModify|KInotify::EventCloseWrite ), + KInotify::WatchEvents( KInotify::EventMove|KInotify::EventDelete|KInotify::EventDeleteSelf|KInotify::EventCloseWrite ), KInotify::WatchFlags() ); #endif } @@ -243,32 +235,11 @@ void Nepomuk::FileWatch::slotFileDeleted( const QString& urlString, bool isDir ) } -void Nepomuk::FileWatch::slotFileCreated( const QString& path ) -{ - if( StrigiServiceConfig::self()->shouldBeIndexed(path) ) { - // we only cache the file and wait until it has been closed, ie. the writing has been finished - m_modifiedFilesCache.insert(path); - } -} - - -void Nepomuk::FileWatch::slotFileModified( const QString& path ) -{ - if( StrigiServiceConfig::self()->shouldBeIndexed(path) ) { - // we only cache the file and wait until it has been closed, ie. the writing has been finished - m_modifiedFilesCache.insert(path); - } -} - - void Nepomuk::FileWatch::slotFileClosedAfterWrite( const QString& path ) { - // we only need to update the file if it has actually been modified - QSet<KUrl>::iterator it = m_modifiedFilesCache.find(path); - if(it != m_modifiedFilesCache.end()) { + if(StrigiServiceConfig::self()->shouldBeIndexed(path)) { // we do not tell the file indexer right away but wait a short while in case the file is modified very often (irc logs for example) m_fileModificationQueue->enqueueUrl( path ); - m_modifiedFilesCache.erase(it); } } diff --git a/nepomuk/services/filewatch/nepomukfilewatch.h b/nepomuk/services/filewatch/nepomukfilewatch.h index 7f0f8fb..efd8254 100644 --- a/nepomuk/services/filewatch/nepomukfilewatch.h +++ b/nepomuk/services/filewatch/nepomukfilewatch.h @@ -72,8 +72,6 @@ namespace Nepomuk { void slotFileMoved( const QString& from, const QString& to ); void slotFileDeleted( const QString& urlString, bool isDir ); void slotFilesDeleted( const QStringList& path ); - void slotFileCreated( const QString& ); - void slotFileModified( const QString& ); void slotFileClosedAfterWrite( const QString& ); void slotMovedWithoutData( const QString& ); void connectToKDirWatch(); @@ -119,9 +117,6 @@ namespace Nepomuk { RegExpCache* m_pathExcludeRegExpCache; RemovableMediaCache* m_removableMediaCache; - /// stores all the file URLs that have been modified but not closed yet - QSet<KUrl> m_modifiedFilesCache; - /// queue used to "compress" constant file modifications like downloads ActiveFileQueue* m_fileModificationQueue; }; ++++++ 0f511184-kderuntime-nepomuk-47branch-memleak.diff ++++++ commit 0f511184ae25364618ba244f6afda5570b02c388 Author: Sebastian Trueg <tr...@kde.org> Date: Mon Oct 24 17:47:25 2011 +0200 Run the MetaDataMover with an event loop. It is using the exact same approach as the file indexer does: a new thread is created and started and the MetaDataMover is then QObject::moveToThread'ed to it. This fixes mem leaks caused by DBus events that are not cleaned up. BUG: 226676 diff --git a/nepomuk/services/filewatch/metadatamover.cpp b/nepomuk/services/filewatch/metadatamover.cpp index 36ff533..f247fa5 100644 --- a/nepomuk/services/filewatch/metadatamover.cpp +++ b/nepomuk/services/filewatch/metadatamover.cpp @@ -1,5 +1,5 @@ /* This file is part of the KDE Project - Copyright (c) 2009-2010 Sebastian Trueg <tr...@kde.org> + Copyright (c) 2009-2011 Sebastian Trueg <tr...@kde.org> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -34,42 +34,44 @@ Nepomuk::MetadataMover::MetadataMover( Soprano::Model* model, QObject* parent ) - : QThread( parent ), - m_stopped(false), + : QObject( parent ), + m_queueMutex(QMutex::Recursive), m_model( model ) { - QTimer* timer = new QTimer( this ); - connect( timer, SIGNAL( timeout() ), this, SLOT( slotClearRecentlyFinishedRequests() ) ); - timer->setInterval( 30000 ); - timer->start(); -} - + // setup the main update queue timer + m_queueTimer = new QTimer(this); + connect(m_queueTimer, SIGNAL(timeout()), + this, SLOT(slotWorkUpdateQueue()), + Qt::DirectConnection); -Nepomuk::MetadataMover::~MetadataMover() -{ + // setup the cleanup timer which removes requests that are done + m_recentlyFinishedRequestsTimer = new QTimer(this); + connect( m_recentlyFinishedRequestsTimer, SIGNAL( timeout() ), + this, SLOT( slotClearRecentlyFinishedRequests() ), + Qt::DirectConnection ); + m_recentlyFinishedRequestsTimer->setInterval( 30000 ); } -void Nepomuk::MetadataMover::stop() +Nepomuk::MetadataMover::~MetadataMover() { - QMutexLocker lock( &m_queueMutex ); - m_stopped = true; - m_queueWaiter.wakeAll(); } void Nepomuk::MetadataMover::moveFileMetadata( const KUrl& from, const KUrl& to ) { - kDebug() << from << to; +// kDebug() << from << to; Q_ASSERT( !from.path().isEmpty() && from.path() != "/" ); Q_ASSERT( !to.path().isEmpty() && to.path() != "/" ); - m_queueMutex.lock(); + + QMutexLocker lock(&m_queueMutex); + UpdateRequest req( from, to ); if ( !m_updateQueue.contains( req ) && !m_recentlyFinishedRequests.contains( req ) ) m_updateQueue.enqueue( req ); - m_queueMutex.unlock(); - m_queueWaiter.wakeAll(); + + QTimer::singleShot(0, this, SLOT(slotStartUpdateTimer())); } @@ -83,79 +85,68 @@ void Nepomuk::MetadataMover::removeFileMetadata( const KUrl& file ) void Nepomuk::MetadataMover::removeFileMetadata( const KUrl::List& files ) { kDebug() << files; - m_queueMutex.lock(); + QMutexLocker lock(&m_queueMutex); + foreach( const KUrl& file, files ) { UpdateRequest req( file ); if ( !m_updateQueue.contains( req ) && !m_recentlyFinishedRequests.contains( req ) ) m_updateQueue.enqueue( req ); } - m_queueMutex.unlock(); - m_queueWaiter.wakeAll(); + + QTimer::singleShot(0, this, SLOT(slotStartUpdateTimer())); } -// FIXME: somehow detect when the nepomuk storage service is down and wait for it to come up again (how about having methods for that in Nepomuk::Service?) -void Nepomuk::MetadataMover::run() +void Nepomuk::MetadataMover::slotWorkUpdateQueue() { - m_stopped = false; - while( !m_stopped ) { - - // lock for initial iteration - m_queueMutex.lock(); - - // work the queue - while( !m_updateQueue.isEmpty() ) { - UpdateRequest updateRequest = m_updateQueue.dequeue(); - m_recentlyFinishedRequests.insert( updateRequest ); - - // unlock after queue utilization - m_queueMutex.unlock(); + // lock for initial iteration + QMutexLocker lock(&m_queueMutex); - kDebug() << "========================= handling" << updateRequest.source() << updateRequest.target(); + // work the queue + if( !m_updateQueue.isEmpty() ) { + UpdateRequest updateRequest = m_updateQueue.dequeue(); + m_recentlyFinishedRequests.insert( updateRequest ); - // an empty second url means deletion - if( updateRequest.target().isEmpty() ) { - KUrl url = updateRequest.source(); - removeMetadata( url ); - } - else { - KUrl from = updateRequest.source(); - KUrl to = updateRequest.target(); + // unlock after queue utilization + lock.unlock(); - // We do NOT get deleted messages for overwritten files! Thus, we - // have to remove all metadata for overwritten files first. - removeMetadata( to ); +// kDebug() << "========================= handling" << updateRequest.source() << updateRequest.target(); - // and finally update the old statements - updateMetadata( from, to ); - } + // an empty second url means deletion + if( updateRequest.target().isEmpty() ) { + removeMetadata( updateRequest.source() ); + } + else { + const KUrl from = updateRequest.source(); + const KUrl to = updateRequest.target(); - kDebug() << "========================= done with" << updateRequest.source() << updateRequest.target(); + // We do NOT get deleted messages for overwritten files! Thus, we + // have to remove all metadata for overwritten files first. + removeMetadata( to ); - // lock for next iteration - m_queueMutex.lock(); + // and finally update the old statements + updateMetadata( from, to ); } - // wait for more input - kDebug() << "Waiting..."; - m_queueWaiter.wait( &m_queueMutex ); - m_queueMutex.unlock(); - kDebug() << "Woke up."; +// kDebug() << "========================= done with" << updateRequest.source() << updateRequest.target(); + } + else { + kDebug() << "All update requests handled. Stopping timer."; + m_queueTimer->stop(); } } void Nepomuk::MetadataMover::removeMetadata( const KUrl& url ) { - kDebug() << url; +// kDebug() << url; if ( url.isEmpty() ) { kDebug() << "empty path. Looks like a bug somewhere..."; } else { const bool isFolder = url.url().endsWith('/'); - kDebug() << "removing metadata for file" << url; Nepomuk::removeResources(QList<QUrl>() << url); if( isFolder ) { @@ -173,7 +164,6 @@ void Nepomuk::MetadataMover::removeMetadata( const KUrl& url ) "}" ) .arg( Soprano::Node::resourceToN3( Nepomuk::Vocabulary::NIE::url() ), url.url(KUrl::AddTrailingSlash) ); - kDebug() << query; // // We cannot use one big loop since our updateMetadata calls below can change the iterator @@ -232,6 +222,23 @@ void Nepomuk::MetadataMover::slotClearRecentlyFinishedRequests() ++it; } } + + if(m_recentlyFinishedRequests.isEmpty()) { + kDebug() << "No more old requests. Stopping timer."; + m_recentlyFinishedRequestsTimer->stop(); + } +} + + +// start the timer in the update thread +void Nepomuk::MetadataMover::slotStartUpdateTimer() +{ + if(!m_queueTimer->isActive()) { + m_queueTimer->start(); + } + if(!m_recentlyFinishedRequestsTimer->isActive()) { + m_recentlyFinishedRequestsTimer->start(); + } } #include "metadatamover.moc" diff --git a/nepomuk/services/filewatch/metadatamover.h b/nepomuk/services/filewatch/metadatamover.h index 9697abe..1e96dac 100644 --- a/nepomuk/services/filewatch/metadatamover.h +++ b/nepomuk/services/filewatch/metadatamover.h @@ -1,5 +1,5 @@ /* This file is part of the KDE Project - Copyright (c) 2009 Sebastian Trueg <tr...@kde.org> + Copyright (c) 2009-2011 Sebastian Trueg <tr...@kde.org> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -19,9 +19,7 @@ #ifndef _NEPOMUK_METADATA_MOVER_H_ #define _NEPOMUK_METADATA_MOVER_H_ -#include <QtCore/QThread> #include <QtCore/QMutex> -#include <QtCore/QWaitCondition> #include <QtCore/QQueue> #include <QtCore/QSet> #include <QtCore/QDateTime> @@ -30,12 +28,14 @@ #include "updaterequest.h" +class QTimer; + namespace Soprano { class Model; } namespace Nepomuk { - class MetadataMover : public QThread + class MetadataMover : public QObject { Q_OBJECT @@ -48,8 +48,6 @@ namespace Nepomuk { void removeFileMetadata( const KUrl& file ); void removeFileMetadata( const KUrl::List& files ); - void stop(); - Q_SIGNALS: /** * Emitted for files (and folders) that have been moved but @@ -62,10 +60,14 @@ namespace Nepomuk { private Q_SLOTS: void slotClearRecentlyFinishedRequests(); + void slotWorkUpdateQueue(); - private: - void run(); + /** + * Start the update queue from the main thread. + */ + void slotStartUpdateTimer(); + private: /** * Remove the metadata for file \p url */ @@ -89,8 +91,9 @@ namespace Nepomuk { QSet<UpdateRequest> m_recentlyFinishedRequests; QMutex m_queueMutex; - QWaitCondition m_queueWaiter; - bool m_stopped; + + QTimer* m_queueTimer; + QTimer* m_recentlyFinishedRequestsTimer; Soprano::Model* m_model; }; diff --git a/nepomuk/services/filewatch/nepomukfilewatch.cpp b/nepomuk/services/filewatch/nepomukfilewatch.cpp index 3f038db..6d4e56e 100644 --- a/nepomuk/services/filewatch/nepomukfilewatch.cpp +++ b/nepomuk/services/filewatch/nepomukfilewatch.cpp @@ -1,5 +1,5 @@ /* This file is part of the KDE Project - Copyright (c) 2007-2010 Sebastian Trueg <tr...@kde.org> + Copyright (c) 2007-2011 Sebastian Trueg <tr...@kde.org> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -130,11 +130,13 @@ Nepomuk::FileWatch::FileWatch( QObject* parent, const QList<QVariant>& ) m_pathExcludeRegExpCache->rebuildCacheFromFilterList( defaultExcludeFilterList() ); // start the mover thread - m_metadataMover = new MetadataMover( mainModel(), this ); + m_metadataMoverThread = new QThread(this); + m_metadataMoverThread->start(); + m_metadataMover = new MetadataMover( mainModel() ); connect( m_metadataMover, SIGNAL(movedWithoutData(QString)), this, SLOT(slotMovedWithoutData(QString)), Qt::QueuedConnection ); - m_metadataMover->start(); + m_metadataMover->moveToThread(m_metadataMoverThread); m_fileModificationQueue = new ActiveFileQueue(this); connect(m_fileModificationQueue, SIGNAL(urlTimeout(KUrl)), @@ -187,8 +189,8 @@ Nepomuk::FileWatch::FileWatch( QObject* parent, const QList<QVariant>& ) Nepomuk::FileWatch::~FileWatch() { kDebug(); - m_metadataMover->stop(); - m_metadataMover->wait(); + m_metadataMoverThread->quit(); + m_metadataMoverThread->wait(); } @@ -207,11 +209,8 @@ void Nepomuk::FileWatch::watchFolder( const QString& path ) void Nepomuk::FileWatch::slotFileMoved( const QString& urlFrom, const QString& urlTo ) { if( !ignorePath( urlFrom ) || !ignorePath( urlTo ) ) { - KUrl from( urlFrom ); - KUrl to( urlTo ); - - kDebug() << from << to; - + const KUrl from( urlFrom ); + const KUrl to( urlTo ); m_metadataMover->moveFileMetadata( from, to ); } } diff --git a/nepomuk/services/filewatch/nepomukfilewatch.h b/nepomuk/services/filewatch/nepomukfilewatch.h index a6f4e2c..fbebfb8 100644 --- a/nepomuk/services/filewatch/nepomukfilewatch.h +++ b/nepomuk/services/filewatch/nepomukfilewatch.h @@ -1,5 +1,5 @@ /* This file is part of the KDE Project - Copyright (c) 2007-2010 Sebastian Trueg <tr...@kde.org> + Copyright (c) 2007-2011 Sebastian Trueg <tr...@kde.org> This library is free software; you can redistribute it and/or modify it under the terms of the GNU Library General Public @@ -38,6 +38,7 @@ class KInotify; class KUrl; class RegExpCache; class ActiveFileQueue; +class QThread; namespace Nepomuk { @@ -108,6 +109,7 @@ namespace Nepomuk { */ bool ignorePath( const QString& path ); + QThread* m_metadataMoverThread; MetadataMover* m_metadataMover; #ifdef BUILD_KINOTIFY ++++++ 1b66b02e-kderuntime-nepomuk-47branch-partialurls.diff ++++++ commit 1b66b02ecf6a055159290849c8bf708a7043bd0d Author: Sebastian Trueg <tr...@kde.org> Date: Thu Oct 20 21:32:14 2011 +0200 Handle parent folders or convertable URLs. So far we converted URLs which actually exist. Now we also handle partial URLs like /media/foo if a storage is mounted on /media/foobar. This is handled by adding additional regex filters to the already existing one. BUG: 284529 diff --git a/nepomuk/common/removablemediacache.cpp b/nepomuk/common/removablemediacache.cpp index dbb31c8..c6cde9b 100644 --- a/nepomuk/common/removablemediacache.cpp +++ b/nepomuk/common/removablemediacache.cpp @@ -165,6 +165,21 @@ const Nepomuk::RemovableMediaCache::Entry* Nepomuk::RemovableMediaCache::findEnt } +QList<const Nepomuk::RemovableMediaCache::Entry*> Nepomuk::RemovableMediaCache::findEntriesByMountPath(const QString &path) const +{ + QList<const Entry*> entries; + for( QHash<QString, Entry>::const_iterator it = m_metadataCache.constBegin(); + it != m_metadataCache.constEnd(); ++it ) { + const Entry& entry = *it; + if(entry.isMounted() && + entry.mountPath().startsWith(path)) { + entries.append(&entry); + } + } + return entries; +} + + bool Nepomuk::RemovableMediaCache::hasRemovableSchema(const KUrl &url) const { return m_usedSchemas.contains(url.scheme()); diff --git a/nepomuk/common/removablemediacache.h b/nepomuk/common/removablemediacache.h index 25982f0..8060765 100644 --- a/nepomuk/common/removablemediacache.h +++ b/nepomuk/common/removablemediacache.h @@ -74,6 +74,14 @@ public: const Entry* findEntryByFilePath( const QString& path ) const; const Entry* findEntryByUrl(const KUrl& url) const; + /** + * Searches for entries which are mounted at a path which starts with + * the given one. Example: a \p path \p /media will result in all + * entries which are mounted under \p /media like \p /media/disk1 or + * \p /media/cdrom. + */ + QList<const Entry*> findEntriesByMountPath(const QString& path) const; + QList<const Entry*> allMedia() const; /** diff --git a/nepomuk/services/storage/removablemediamodel.cpp b/nepomuk/services/storage/removablemediamodel.cpp index 836668e..aa3c49c 100644 --- a/nepomuk/services/storage/removablemediamodel.cpp +++ b/nepomuk/services/storage/removablemediamodel.cpp @@ -193,14 +193,15 @@ Soprano::QueryResultIterator Nepomuk::RemovableMediaModel::executeQuery(const QS return new QueryResultIteratorBackend(this, FilterModel::executeQuery(convertFileUrls(query), language, userQueryLanguage)); } -Soprano::Node Nepomuk::RemovableMediaModel::convertFileUrl(const Soprano::Node &node) const +Soprano::Node Nepomuk::RemovableMediaModel::convertFileUrl(const Soprano::Node &node, bool forRegEx) const { if(node.isResource()) { const QUrl url = node.uri(); if(url.scheme() == QLatin1String("file")) { const QString localFilePath = url.toLocalFile(); if(const RemovableMediaCache::Entry* entry = m_removableMediaCache->findEntryByFilePath(localFilePath)) { - if(entry->isMounted()) { + if(entry->isMounted() && + (!forRegEx || entry->mountPath().length() < localFilePath.length())) { return entry->constructRelativeUrl(localFilePath); } } @@ -270,11 +271,28 @@ QString Nepomuk::RemovableMediaModel::convertFileUrls(const QString &query) cons // is 0, 1, or 3 - nothing else int quoteCnt = 0; + + // if quoteCnt > 0, ie. we are in a literal this is the first char of it int literalStartPos = 0; + + // true if we are in a regex filter bool inRegEx = false; + + // if inRegEx is true this is the position where the regex starts in the new query + int newQueryRegExStart = 0; + + // if inRegEx is true this is the variable used in the regex (including any functions) + QString variable; + + // true if we are in a resource URI like: <....> bool inRes = false; + + // the char used for quote ' or " QChar quote; + + // the new query we construct QString newQuery; + for(int i = 0; i < query.length(); ++i) { const QChar c = query[i]; @@ -359,7 +377,21 @@ QString Nepomuk::RemovableMediaModel::convertFileUrls(const QString &query) cons query[i+2].toLower() == 'g' && query[i+3].toLower() == 'e' && query[i+4].toLower() == 'x') { - inRegEx = true; + // determine the variable + int pos = query.indexOf('(', i+4); + if(pos > i+4) { + int endPos = query.indexOf(',', pos+1); + if(endPos > pos+1) { + inRegEx = true; + variable = query.mid(pos+1, endPos-pos-1).trimmed(); + newQueryRegExStart = newQuery.length(); + // we already copy the entire start of the REGEX since our REGEX end check is way too simple + // It would not handle function calls like REGEX(STR(?var)),... + newQuery.append(query.mid(i, endPos-i+1)); + i = endPos; + continue; + } + } } } @@ -401,10 +433,44 @@ QString Nepomuk::RemovableMediaModel::convertFileUrls(const QString &query) cons if(pos > 0) { // convert the file URL into a filex URL (if necessary) const KUrl fileUrl = query.mid(i, pos-i); - newQuery += KUrl(convertFileUrl(fileUrl).uri()).url(); - // set i to last char we handled and let the loop continue with the end of the quote - i = pos-1; - continue; + KUrl convertedUrl = KUrl(convertFileUrl(fileUrl, true).uri()); + + // 1. Case: we have an exact match with one of the removable media (this always includes a trailing slash) + if(fileUrl != convertedUrl) { + newQuery += convertedUrl.url(); + // set i to last char we handled and let the loop continue with the end of the quote + i = pos-1; + continue; + } + + // 2. Case The query tries to match a super-folder of one of the removable media. In that case we need + // to add additional regex filters which match all candidates + else { + // create regex filters for all of them + // We need to append a slash since we do not want to include a possibly unmounted medium "foobar" if only "foo" is mounted. + QStringList filters; + foreach(const RemovableMediaCache::Entry* entry, m_removableMediaCache->findEntriesByMountPath(fileUrl.toLocalFile())) { + filters << QString::fromLatin1("REGEX(%1, '^%2/')").arg(variable, entry->constructRelativeUrl(QString()).url()); + } + if(!filters.isEmpty()) { + // 1. copy the original regex + filters.prepend(QString::fromLatin1("REGEX(%1, '^%2')").arg(variable, query.mid(i, pos-i))); + + // 2. Strip away the previsouly copied REGEX term + newQuery.truncate(newQueryRegExStart); + + // 3. append the new ones and add new parethesis + newQuery.append('('); + newQuery.append(filters.join(QLatin1String(" || "))); + newQuery.append(')'); + + // 4. update i: + // if we find a closing parenthesis, that is the last char we handled + // otherwise it is just pos+quoteCnt-1 since the last quote is what we handled + i = qMax(pos+quoteCnt-1, query.indexOf(')', pos+1)); + continue; + } + } } } } diff --git a/nepomuk/services/storage/removablemediamodel.h b/nepomuk/services/storage/removablemediamodel.h index 52e4eda..4926439 100644 --- a/nepomuk/services/storage/removablemediamodel.h +++ b/nepomuk/services/storage/removablemediamodel.h @@ -93,7 +93,13 @@ private: */ Soprano::Statement convertFileUrls(const Soprano::Statement& s) const; - Soprano::Node convertFileUrl(const Soprano::Node& node) const; + /** + * Convert a local file URL into its internal counterpart or return the + * given URL if it does not need to be converted. + * \param forRegEx If true the base path of the storage medium will not + * be converted. + */ + Soprano::Node convertFileUrl(const Soprano::Node& node, bool forRegEx = false) const; /** * Converts file:/ URLs into their filex:/ counterpart if necessary. diff --git a/nepomuk/services/storage/test/removablemediamodeltest.cpp b/nepomuk/services/storage/test/removablemediamodeltest.cpp index c47c26d..eeae40d 100644 --- a/nepomuk/services/storage/test/removablemediamodeltest.cpp +++ b/nepomuk/services/storage/test/removablemediamodeltest.cpp @@ -153,6 +153,10 @@ void RemovableMediaModelTest::testConvertFileUrlsInQuery_data() << QString::fromLatin1("select ?r where { ?r ?p <file:///media/XO-Y4/test.txt> . }") << QString::fromLatin1("select ?r where { ?r ?p <filex://xyz-123/test.txt> . }"); + QTest::newRow("queryWithConvertableFileUrl1WeirdFormatting") + << QString::fromLatin1("select ?r where { ?r ?p <file:///media/XO-Y4/test.txt> . }") + << QString::fromLatin1("select ?r where { ?r ?p <filex://xyz-123/test.txt> . }"); + QTest::newRow("queryWithConvertableFileUrl2") << QString::fromLatin1("select ?r where { ?r ?p <file:///media/nfs/test.txt> . }") << QString::fromLatin1("select ?r where { ?r ?p <nfs://thehost/solid-path/test.txt> . }"); @@ -161,14 +165,48 @@ void RemovableMediaModelTest::testConvertFileUrlsInQuery_data() << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^file:///media/XO-Y4/test')) . }") << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^filex://xyz-123/test')) . }"); + QTest::newRow("queryWithConvertableRegex1WithWeirdFormatting") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . filter(reGEx( STR(?u) , 'file:///media/XO-Y4/test' ) ) . }") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . filter(reGEx( STR(?u) , 'filex://xyz-123/test' ) ) . }"); + QTest::newRow("queryWithConvertableRegex2") << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^file:///media/nfs/')) . }") << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^nfs://thehost/solid-path/')) . }"); + QTest::newRow("queryWithConvertableRegex3") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '''^file:///media/nfs/''')) . }") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '''^nfs://thehost/solid-path/''')) . }"); + + // looking for anything in /media includes files from any storage mounted somewhere under /media + QTest::newRow("queryWithConvertableRegex4") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^file:///media')) . }") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . " + "FILTER((" + "REGEX(?u, '^file:///media') || " + "REGEX(?u, '^optical://solidman_begins/') || " + "REGEX(?u, '^filex://whatever/') || " + "REGEX(?u, '^nfs://thehost/solid-path/') || " + "REGEX(?u, '^filex://xyz-123/'))" + ") . }"); + + QTest::newRow("queryWithConvertableRegex4WithWeirdFormatting") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . filter ( reGEX( str( ?u) , '^file:///media' ) ) . }") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . " + "filter ( (" + "REGEX(str( ?u), '^file:///media') || " + "REGEX(str( ?u), '^optical://solidman_begins/') || " + "REGEX(str( ?u), '^filex://whatever/') || " + "REGEX(str( ?u), '^nfs://thehost/solid-path/') || " + "REGEX(str( ?u), '^filex://xyz-123/'))" + " ) . }"); + + QTest::newRow("queryWithConvertableRegex4") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^file:///media/nfs')) . }") + << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER((REGEX(?u, '^file:///media/nfs') || REGEX(?u, '^nfs://thehost/solid-path/'))) . }"); + QTest::newRow("queryWithNotReallyAFileUrl") << QString::fromLatin1("select ?r where { ?r ?p ?u . FILTER(REGEX(?u, 'ffile:///media/nfs/')) . }") << QString::fromLatin1("select ?r where { ?r ?p ?u . FILTER(REGEX(?u, 'ffile:///media/nfs/')) . }"); - // TODO: add queries that should NOT be converted } void RemovableMediaModelTest::testConvertFileUrlsInQuery() ++++++ 348d7cbf-kderuntime-nepomuk-47branch-regexp.diff ++++++ commit 348d7cbf37e3e0cc638a8c6844f66efee131f6cf Author: Sebastian Trueg <tr...@kde.org> Date: Thu Oct 20 21:31:52 2011 +0200 Only convert file URLs in REGEX filters that appear at the beginning. diff --git a/nepomuk/services/storage/removablemediamodel.cpp b/nepomuk/services/storage/removablemediamodel.cpp index ebd83f1..836668e 100644 --- a/nepomuk/services/storage/removablemediamodel.cpp +++ b/nepomuk/services/storage/removablemediamodel.cpp @@ -270,6 +270,7 @@ QString Nepomuk::RemovableMediaModel::convertFileUrls(const QString &query) cons // is 0, 1, or 3 - nothing else int quoteCnt = 0; + int literalStartPos = 0; bool inRegEx = false; bool inRes = false; QChar quote; @@ -293,6 +294,7 @@ QString Nepomuk::RemovableMediaModel::convertFileUrls(const QString &query) cons newQuery.append(c); newQuery.append(c); } + literalStartPos = i+1; continue; } else if(c == quote) { @@ -379,12 +381,16 @@ QString Nepomuk::RemovableMediaModel::convertFileUrls(const QString &query) cons // else if(inRegEx && quoteCnt && c == 'f') { // peek forward to see if its a file URL - if(i+5 < query.length() && - query[i+1] == 'i' && - query[i+2] == 'l' && - query[i+3] == 'e' && - query[i+4] == ':' && - query[i+5] == '/') { + // file URLs in regexps only make sense if they appear at the beginning or following a '^' + if((i == literalStartPos || + (query[literalStartPos] == '^' && + i == literalStartPos+1)) && + i+5 < query.length() && + query[i+1] == 'i' && + query[i+2] == 'l' && + query[i+3] == 'e' && + query[i+4] == ':' && + query[i+5] == '/') { // find end of regex QString quoteEnd = quote; if(quoteCnt == 3) { diff --git a/nepomuk/services/storage/test/removablemediamodeltest.cpp b/nepomuk/services/storage/test/removablemediamodeltest.cpp index 3a4abee..c47c26d 100644 --- a/nepomuk/services/storage/test/removablemediamodeltest.cpp +++ b/nepomuk/services/storage/test/removablemediamodeltest.cpp @@ -164,6 +164,11 @@ void RemovableMediaModelTest::testConvertFileUrlsInQuery_data() QTest::newRow("queryWithConvertableRegex2") << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^file:///media/nfs/')) . }") << QString::fromLatin1("select ?r where { ?r nie:url ?u . FILTER(REGEX(?u, '^nfs://thehost/solid-path/')) . }"); + + QTest::newRow("queryWithNotReallyAFileUrl") + << QString::fromLatin1("select ?r where { ?r ?p ?u . FILTER(REGEX(?u, 'ffile:///media/nfs/')) . }") + << QString::fromLatin1("select ?r where { ?r ?p ?u . FILTER(REGEX(?u, 'ffile:///media/nfs/')) . }"); + // TODO: add queries that should NOT be converted } void RemovableMediaModelTest::testConvertFileUrlsInQuery() ++++++ ab58993a-kderuntime-nepomuk-47branch-nullpointercrash.diff ++++++ commit ab58993ab29520345bbee4b5b75e88734401c96c Author: Sebastian Trueg <tr...@kde.org> Date: Fri Oct 21 12:22:00 2011 +0200 Check StorageAccess for 0 before using it. This avoids crashes. BUG: 284602 diff --git a/nepomuk/common/removablemediacache.cpp b/nepomuk/common/removablemediacache.cpp index c6cde9b..00a1156 100644 --- a/nepomuk/common/removablemediacache.cpp +++ b/nepomuk/common/removablemediacache.cpp @@ -139,8 +139,10 @@ const Nepomuk::RemovableMediaCache::Entry* Nepomuk::RemovableMediaCache::findEnt for( QHash<QString, Entry>::const_iterator it = m_metadataCache.begin(); it != m_metadataCache.end(); ++it ) { const Entry& entry = *it; - if ( entry.device().as<Solid::StorageAccess>()->isAccessible() && - path.startsWith( entry.device().as<Solid::StorageAccess>()->filePath() ) ) + const Solid::StorageAccess* storage = entry.device().as<Solid::StorageAccess>(); + if ( storage && + storage->isAccessible() && + path.startsWith( storage->filePath() ) ) return &entry; } ++++++ cc5cf57c-kderuntime-nepomuk-47branch-robust-restart.diff ++++++ commit cc5cf57c7e5be45fbf757414994290dd58ea1231 Author: Sebastian Trueg <tr...@kde.org> Date: Tue Oct 25 10:27:59 2011 +0200 Re-create the repository in case of an un-scheduled shutdown of the DB. This is a hacky way of ensuring that Nepomuk even works after a crash or a third-party kill of the Virtuoso instance. "Hacky" because it does not care about clients which will get some query errors during the restart. BUG: 263730 FIXED-IN: 4.7.3 diff --git a/nepomuk/services/storage/repository.cpp b/nepomuk/services/storage/repository.cpp index b9b2051..784cbe9 100644 --- a/nepomuk/services/storage/repository.cpp +++ b/nepomuk/services/storage/repository.cpp @@ -89,9 +89,13 @@ void Nepomuk::Repository::close() delete m_dataManagementAdaptor; m_dataManagementAdaptor = 0; + setParentModel(0); delete m_dataManagementModel; m_dataManagementModel = 0; + delete m_classAndPropertyTree; + m_classAndPropertyTree = 0; + delete m_inferencer; m_inferencer = 0; @@ -188,6 +192,10 @@ void Nepomuk::Repository::open() return; } +#if SOPRANO_IS_VERSION(2, 7, 3) + connect(m_model, SIGNAL(virtuosoStopped(bool)), this, SLOT(slotVirtuosoStopped(bool))); +#endif + kDebug() << "Successfully created new model for repository" << name(); // Fire up the graph maintainer on the pure data model. @@ -211,12 +219,12 @@ void Nepomuk::Repository::open() // create a SignalCacheModel to make sure no client slows us down by listening to the stupid signals // ================================= Soprano::Util::SignalCacheModel* scm = new Soprano::Util::SignalCacheModel( m_removableStorageModel ); - scm->setParent(this); // memory management + scm->setParent(m_removableStorageModel); // memory management // Create the NRLModel which is required by the DMM below // ================================= m_nrlModel = new Soprano::NRLModel(scm); - m_nrlModel->setParent(this); // memory management + m_nrlModel->setParent(scm); // memory management // create the DataManagementModel on top of everything // ================================= @@ -423,4 +431,15 @@ void Nepomuk::Repository::updateInference() m_inferencer->updateAllResources(); } +void Nepomuk::Repository::slotVirtuosoStopped(bool normalExit) +{ + if(!normalExit) { + kDebug() << "Virtuoso was killed or crashed. Restarting the repository."; + // restart the dumb way for now + // Ideally we would inform the other services so they can be restarted or something. + close(); + open(); + } +} + #include "repository.moc" diff --git a/nepomuk/services/storage/repository.h b/nepomuk/services/storage/repository.h index a52f92f..c573b67 100644 --- a/nepomuk/services/storage/repository.h +++ b/nepomuk/services/storage/repository.h @@ -99,6 +99,7 @@ namespace Nepomuk { private Q_SLOTS: void copyFinished( KJob* job ); + void slotVirtuosoStopped( bool normalExit ); private: Soprano::BackendSettings readVirtuosoSettings() const; -- To unsubscribe, e-mail: opensuse-commit+unsubscr...@opensuse.org For additional commands, e-mail: opensuse-commit+h...@opensuse.org