common/Util.cpp | 21 ++++++++++++++++++++
common/Util.hpp | 3 ++
loleaflet/src/core/Socket.js | 4 +--
loolwsd.xml.in | 1
wsd/Admin.cpp | 44 +++++++++++++++++++++++++++++++++++++++++++
wsd/Admin.hpp | 8 ++++++-
wsd/AdminModel.cpp | 19 ++++++++++++++++++
wsd/AdminModel.hpp | 21 ++++++++++++++++++++
wsd/DocumentBroker.cpp | 12 +++++------
wsd/DocumentBroker.hpp | 4 +++
wsd/LOOLWSD.cpp | 14 +++++++++++++
wsd/LOOLWSD.hpp | 4 +++
12 files changed, 146 insertions(+), 9 deletions(-)
New commits:
commit 3754972df41bc4f7a0638ed5f1ab1b61bb47d915
Author: Pranav Kant <pran...@collabora.co.uk>
Date: Fri Jul 7 17:12:19 2017 +0530
Introduce memproportion of memory to use before shutting down processes.
Start killing documents when memory usage goes above threshold.
Also make it possible to close documents from admin instance.
In DocumentBroker::closeDocument, just set the _stop flag and wake
up the polling thread which will terminate the children, instead of
manually terminating the children.
Change-Id: Ie70e05b3fb6ea816a87b6dcfaed92cdddb94aa90
(cherry picked from commit fde57adbbf9ab2fba80c6b8e0d877c797b55bea5)
(cherry picked from commit 99c65d66434e6c623b0a2d7e5fdc5b46a45ade19)
Reviewed-on: https://gerrit.libreoffice.org/39768
Reviewed-by: Jan Holesovsky <ke...@collabora.com>
Tested-by: Jan Holesovsky <ke...@collabora.com>
diff --git a/common/Util.cpp b/common/Util.cpp
index b3bcb354..bd7209f9 100644
--- a/common/Util.cpp
+++ b/common/Util.cpp
@@ -171,6 +171,27 @@ namespace Util
return nullptr;
}
+ size_t getTotalSystemMemory()
+ {
+ size_t totalMemKb = 0;
+ FILE* file = fopen("/proc/meminfo", "r");
+ if (file != nullptr)
+ {
+ char line[4096] = { 0 };
+ while (fgets(line, sizeof(line), file))
+ {
+ const char* value;
+ if ((value = startsWith(line, "MemTotal:")))
+ {
+ totalMemKb = atoi(value);
+ break;
+ }
+ }
+ }
+
+ return totalMemKb;
+ }
+
std::pair<size_t, size_t> getPssAndDirtyFromSMaps(FILE* file)
{
size_t numPSSKb = 0;
diff --git a/common/Util.hpp b/common/Util.hpp
index 93f25cf0..5480d790 100644
--- a/common/Util.hpp
+++ b/common/Util.hpp
@@ -95,6 +95,9 @@ namespace Util
#endif
}
+ /// Returns the total physical memory (in kB) available in the system
+ size_t getTotalSystemMemory();
+
/// Returns the process PSS in KB (works only when we have perms for
/proc/pid/smaps).
size_t getMemoryUsagePSS(const Poco::Process::PID pid);
diff --git a/loleaflet/src/core/Socket.js b/loleaflet/src/core/Socket.js
index 444abe62..88d3fd4d 100644
--- a/loleaflet/src/core/Socket.js
+++ b/loleaflet/src/core/Socket.js
@@ -243,7 +243,7 @@ L.Socket = L.Class.extend({
if (textMsg === 'ownertermination') {
msg = _('Session terminated by document owner');
}
- else if (textMsg === 'idle') {
+ else if (textMsg === 'idle' || textMsg === 'oom') {
msg = _('Session was terminated due to idleness
- please click to reload');
this._map._documentIdle = true;
}
@@ -335,7 +335,7 @@ L.Socket = L.Class.extend({
});
options.$vex.append(options.$vexContent);
- if (textMsg === 'idle') {
+ if (textMsg === 'idle' || textMsg === 'oom') {
var map = this._map;
options.$vex.bind('click.vex', function(e) {
console.debug('idleness: reactivating');
diff --git a/loolwsd.xml.in b/loolwsd.xml.in
index eed51283..6f8d6d35 100644
--- a/loolwsd.xml.in
+++ b/loolwsd.xml.in
@@ -11,6 +11,7 @@
<server_name desc="Hostname:port of the server running loolwsd. If empty,
it's derived from the request." type="string" default=""></server_name>
<file_server_root_path desc="Path to the directory that should be
considered root for the file server. This should be the directory containing
loleaflet." type="path" relative="true"
default="loleaflet/../"></file_server_root_path>
+ <memproportion desc="The maximum percentage of system memory consumed by
all of the LibreOffice Online, after which we start cleaning up idle documents"
type="double" default="80.0"></memproportion>
<num_prespawn_children desc="Number of child processes to keep started in
advance and waiting for new clients." type="uint"
default="1">1</num_prespawn_children>
<per_document desc="Document-specific settings, including LO Core
settings.">
<max_concurrency desc="The maximum number of threads to use while
processing a document." type="uint" default="4">4</max_concurrency>
diff --git a/wsd/Admin.cpp b/wsd/Admin.cpp
index eb80bce3..cb85edbc 100644
--- a/wsd/Admin.cpp
+++ b/wsd/Admin.cpp
@@ -291,6 +291,9 @@ Admin::Admin() :
{
LOG_INF("Admin ctor.");
+ _totalSysMem = Util::getTotalSystemMemory();
+ LOG_TRC("Total system memory : " << _totalSysMem);
+
const auto totalMem = getTotalMemoryUsage();
LOG_TRC("Total memory used: " << totalMem);
_model.addMemStats(totalMem);
@@ -336,6 +339,9 @@ void Admin::pollingThread()
lastMem = now;
memWait += _memStatsTaskIntervalMs;
+
+ // If our total memory consumption is above limit, cleanup
+ triggerMemoryCleanup(totalMem);
}
// Handle websockets & other work.
@@ -423,6 +429,44 @@ void Admin::updateMemoryDirty(const std::string& docKey,
int dirty)
{ _model.updateMemoryDirty(docKey, dirty); });
}
+
+void Admin::triggerMemoryCleanup(size_t totalMem)
+{
+ LOG_TRC("Total memory we are consuming (in kB): " << totalMem);
+ // Trigger mem cleanup when we are consuming too much memory (as
configured by sysadmin)
+ const auto memLimit = LOOLWSD::getConfigValue<double>("memproportion",
static_cast<double>(80.0));
+ LOG_TRC("Mem proportion for LOOL configured : " << memLimit);
+ float memToFreePercentage = 0;
+ if ( (memToFreePercentage = (totalMem/static_cast<double>(_totalSysMem)) -
memLimit/100.) > 0.0 )
+ {
+ int memToFree = memToFreePercentage * totalMem;
+ LOG_TRC("Memory to be freed (in kB) : " << memToFree);
+ // prepare document list sorted by most idle times
+ std::list<DocBasicInfo> docList = _model.getDocumentsSortedByIdle();
+
+ LOG_TRC("Checking saved documents in document list, length: " <<
docList.size());
+ // Kill the saved documents first
+ std::list<DocBasicInfo>::iterator docIt = docList.begin();
+ while (docIt != docList.end() && memToFree > 0)
+ {
+ LOG_TRC("Document: DocKey[" << docIt->_docKey << "], Idletime[" <<
docIt->_idleTime << "],"
+ << " Saved: [" << docIt->_saved << "], Mem: [" <<
docIt->_mem << "].");
+ if (docIt->_saved)
+ {
+ // Kill and remove from list
+ LOG_DBG("OOM: Killing saved document with DocKey " <<
docIt->_docKey);
+ LOOLWSD::closeDocument(docIt->_docKey, "oom");
+ memToFree -= docIt->_mem;
+ docList.erase(docIt);
+ }
+ else
+ ++docIt;
+ }
+ }
+
+ LOG_TRC("OOM: Memory to free percentage : " << memToFreePercentage);
+}
+
void Admin::dumpState(std::ostream& os)
{
// FIXME: be more helpful ...
diff --git a/wsd/Admin.hpp b/wsd/Admin.hpp
index e003d619..864c6c82 100644
--- a/wsd/Admin.hpp
+++ b/wsd/Admin.hpp
@@ -103,11 +103,17 @@ public:
void dumpState(std::ostream& os) override;
private:
+ /// Memory consumption has increased, start killing kits etc. till memory
consumption gets back
+ /// under @hardModeLimit
+ void triggerMemoryCleanup(size_t hardModeLimit);
+
+private:
/// The model is accessed only during startup & in
/// the Admin Poll thread.
AdminModel _model;
int _forKitPid;
- long _lastTotalMemory;
+ size_t _lastTotalMemory;
+ size_t _totalSysMem;
std::atomic<int> _memStatsTaskIntervalMs;
std::atomic<int> _cpuStatsTaskIntervalMs;
diff --git a/wsd/AdminModel.cpp b/wsd/AdminModel.cpp
index 5e1bef3a..f0f65b5b 100644
--- a/wsd/AdminModel.cpp
+++ b/wsd/AdminModel.cpp
@@ -429,6 +429,25 @@ unsigned AdminModel::getTotalActiveViews()
return numTotalViews;
}
+std::list<DocBasicInfo> AdminModel::getDocumentsSortedByIdle() const
+{
+ std::list<DocBasicInfo> docList;
+ for (const auto& it: _documents)
+ {
+ docList.emplace_back(it.second.getDocKey(),
+ it.second.getIdleTime(),
+ !it.second.getModifiedStatus(),
+ it.second.getMemoryDirty());
+ }
+
+ // Sort the list by idle times;
+ docList.sort([](const DocBasicInfo& a, const DocBasicInfo& b) {
+ return a._idleTime > b._idleTime;
+ });
+
+ return docList;
+}
+
std::string AdminModel::getDocuments() const
{
assertCorrectThread();
diff --git a/wsd/AdminModel.hpp b/wsd/AdminModel.hpp
index cdca2243..f963a9da 100644
--- a/wsd/AdminModel.hpp
+++ b/wsd/AdminModel.hpp
@@ -46,6 +46,22 @@ private:
std::time_t _end = 0;
};
+/// Containing basic information about document
+struct DocBasicInfo
+{
+ std::string _docKey;
+ std::time_t _idleTime;
+ bool _saved;
+ int _mem;
+
+ DocBasicInfo(const std::string& docKey, std::time_t idleTime, bool saved,
int mem)
+ : _docKey(docKey),
+ _idleTime(idleTime),
+ _saved(saved),
+ _mem(mem)
+ { }
+};
+
/// A document in Admin controller.
class Document
{
@@ -62,6 +78,8 @@ public:
{
}
+ const std::string getDocKey() const { return _docKey; }
+
Poco::Process::PID getPid() const { return _pid; }
std::string getFilename() const { return _filename; }
@@ -201,6 +219,9 @@ public:
void updateLastActivityTime(const std::string& docKey);
void updateMemoryDirty(const std::string& docKey, int dirty);
+ /// Document basic info list sorted by most idle time
+ std::list<DocBasicInfo> getDocumentsSortedByIdle() const;
+
private:
std::string getMemStats();
diff --git a/wsd/DocumentBroker.cpp b/wsd/DocumentBroker.cpp
index 5b4d8d47..29fd8c76 100644
--- a/wsd/DocumentBroker.cpp
+++ b/wsd/DocumentBroker.cpp
@@ -158,6 +158,7 @@ DocumentBroker::DocumentBroker(const std::string& uri,
_cursorHeight(0),
_poll(new DocumentBrokerPoll("docbroker_" + _docId, *this)),
_stop(false),
+ _closeReason("stopped"),
_tileVersion(0),
_debugRenderedTileCount(0)
{
@@ -227,8 +228,6 @@ void DocumentBroker::pollThread()
static const bool AutoSaveEnabled = !std::getenv("LOOL_NO_AUTOSAVE");
static const size_t IdleDocTimeoutSecs = LOOLWSD::getConfigValue<int>(
"per_document.idle_timeout_secs", 3600);
- std::string closeReason = "stopped";
-
// Main polling loop goodness.
while (!_stop && _poll->continuePolling() && !TerminationFlag)
{
@@ -245,7 +244,7 @@ void DocumentBroker::pollThread()
if (ShutdownRequestFlag)
{
- closeReason = "recycling";
+ _closeReason = "recycling";
_stop = true;
}
else if (AutoSaveEnabled && !_stop &&
@@ -264,7 +263,7 @@ void DocumentBroker::pollThread()
{
LOG_INF("Terminating " << (idle ? "idle" : "dead") <<
" DocumentBroker for docKey [" << getDocKey() << "].");
- closeReason = (idle ? "idle" : "dead");
+ _closeReason = (idle ? "idle" : "dead");
_stop = true;
}
}
@@ -287,7 +286,7 @@ void DocumentBroker::pollThread()
}
// Terminate properly while we can.
- terminateChild(closeReason);
+ terminateChild(_closeReason);
// Stop to mark it done and cleanup.
_poll->stop();
@@ -1422,7 +1421,8 @@ void DocumentBroker::closeDocument(const std::string&
reason)
assertCorrectThread();
LOG_DBG("Closing DocumentBroker for docKey [" << _docKey << "] with
reason: " << reason);
- terminateChild(reason);
+ _closeReason = reason; // used later in the polling loop
+ stop();
}
void DocumentBroker::broadcastMessage(const std::string& message)
diff --git a/wsd/DocumentBroker.hpp b/wsd/DocumentBroker.hpp
index 27878c69..79d81bd0 100644
--- a/wsd/DocumentBroker.hpp
+++ b/wsd/DocumentBroker.hpp
@@ -341,6 +341,9 @@ public:
/// Sends a message to all sessions
void broadcastMessage(const std::string& message);
+
+ /// Sets the reason for closing document;
+ void setCloseReason(const std::string& closeReason) { _closeReason =
closeReason; }
private:
@@ -412,6 +415,7 @@ private:
mutable std::mutex _mutex;
std::unique_ptr<DocumentBrokerPoll> _poll;
std::atomic<bool> _stop;
+ std::string _closeReason;
/// Versioning is used to prevent races between
/// painting and invalidation.
diff --git a/wsd/LOOLWSD.cpp b/wsd/LOOLWSD.cpp
index 2d91d49b..6307acb5 100644
--- a/wsd/LOOLWSD.cpp
+++ b/wsd/LOOLWSD.cpp
@@ -617,6 +617,7 @@ void LOOLWSD::initialize(Application& self)
{ "lo_jail_subpath", "lo" },
{ "server_name", "" },
{ "file_server_root_path", "loleaflet/.." },
+ { "memproportion", "80.0" },
{ "num_prespawn_children", "1" },
{ "per_document.max_concurrency", "4" },
{ "per_document.idle_timeout_secs", "3600" },
@@ -1096,6 +1097,19 @@ void LOOLWSD::doHousekeeping()
PrisonerPoll.wakeup();
}
+void LOOLWSD::closeDocument(const std::string& docKey, const std::string&
message)
+{
+ std::unique_lock<std::mutex> docBrokersLock(DocBrokersMutex);
+ auto docBrokerIt = DocBrokers.find(docKey);
+ if (docBrokerIt != DocBrokers.end())
+ {
+ std::shared_ptr<DocumentBroker> docBroker = docBrokerIt->second;
+ docBroker->addCallback([docBroker, message]() {
+ docBroker->closeDocument(message);
+ });
+ }
+}
+
/// Really do the house-keeping
void PrisonerPoll::wakeupHook()
{
diff --git a/wsd/LOOLWSD.hpp b/wsd/LOOLWSD.hpp
index f9c3d409..b3d0a75b 100644
--- a/wsd/LOOLWSD.hpp
+++ b/wsd/LOOLWSD.hpp
@@ -121,6 +121,9 @@ public:
/// child kit processes and cleans up DocBrokers.
static void doHousekeeping();
+ /// Close document with @docKey and a @message
+ static void closeDocument(const std::string& docKey, const std::string&
message);
+
protected:
void initialize(Poco::Util::Application& self) override;
void defineOptions(Poco::Util::OptionSet& options) override;
@@ -155,6 +158,7 @@ private:
void operator()(unsigned int& value) { value = _config.getUInt(_name);
}
void operator()(bool& value) { value = _config.getBool(_name); }
void operator()(std::string& value) { value =
_config.getString(_name); }
+ void operator()(double& value) { value = _config.getDouble(_name); }
};
template <typename T>
_______________________________________________
Libreoffice-commits mailing list
libreoffice-comm...@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
