[MediaWiki-commits] [Gerrit] Fixed memory issues - change (operations...incremental)
Petr Onderka has uploaded a new change for review.
https://gerrit.wikimedia.org/r/83783
Change subject: Fixed memory issues
..
Fixed memory issues
- clearing index cache periodically, so that it doesn't grow too much
- fixed memory leaks
- less copying
Change-Id: I1483a2f4855a7faa751dd8ed7ee050b17a0c7204
---
M DumpObjects/DumpObject.h
M Indexes/Index.h
M Indexes/Index.tpp
M Indexes/IndexInnerNode.h
M Indexes/IndexInnerNode.tpp
M Indexes/IndexLeafNode.h
M Indexes/IndexLeafNode.tpp
M Indexes/IndexNode.h
M Indexes/Iterators/IndexNodeIterator.h
M XmlUtils.cpp
10 files changed, 47 insertions(+), 10 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/dumps/incremental
refs/changes/83/83783/1
diff --git a/DumpObjects/DumpObject.h b/DumpObjects/DumpObject.h
index ef21805..3efc910 100644
--- a/DumpObjects/DumpObject.h
+++ b/DumpObjects/DumpObject.h
@@ -16,18 +16,20 @@
virtual void WriteInternal() = 0;
templatetypename T
-void WriteValue(const T value);
+void WriteValue(const T value);
public:
templatetypename T
-static void WriteValue(std::ostream stream, const T value);
+static void WriteValue(std::ostream stream, const T value);
templatetypename T
static void ReadValue(std::istream stream, T value);
templatetypename T
-static std::uint32_t ValueSize(const T value);
+static std::uint32_t ValueSize(const T value);
virtual std::uint32_t NewLength() = 0;
+
+virtual ~DumpObjectBase() {}
};
class DumpObject : public DumpObjectBase
@@ -47,13 +49,13 @@
};
templatetypename T
-void DumpObjectBase::WriteValue(const T value)
+void DumpObjectBase::WriteValue(const T value)
{
WriteValue(*stream, value);
}
templatetypename T
-void DumpObjectBase::WriteValue(std::ostream stream, const T value)
+void DumpObjectBase::WriteValue(std::ostream stream, const T value)
{
DumpTraitsT::Write(stream, value);
}
@@ -65,7 +67,7 @@
}
templatetypename T
-uint32_t DumpObjectBase::ValueSize(const T value)
+uint32_t DumpObjectBase::ValueSize(const T value)
{
return DumpTraitsT::DumpSize(value);
}
\ No newline at end of file
diff --git a/Indexes/Index.h b/Indexes/Index.h
index 24e822c..ac77ddb 100644
--- a/Indexes/Index.h
+++ b/Indexes/Index.h
@@ -17,6 +17,8 @@
std::weak_ptrWritableDump dump;
std::weak_ptrOffset fileHeaderOffset;
+int recentChanges;
+
void AfterAdd();
public:
Index(std::weak_ptrWritableDump dump, std::weak_ptrOffset
fileHeaderOffset, bool delaySave = false);
diff --git a/Indexes/Index.tpp b/Indexes/Index.tpp
index 294d755..7ba38d5 100644
--- a/Indexes/Index.tpp
+++ b/Indexes/Index.tpp
@@ -7,7 +7,7 @@
templatetypename TKey, typename TValue
IndexTKey, TValue::Index(std::weak_ptrWritableDump dump,
std::weak_ptrOffset fileHeaderOffset, bool delaySave)
-: dump(dump), fileHeaderOffset(fileHeaderOffset)
+: dump(dump), fileHeaderOffset(fileHeaderOffset), recentChanges(0)
{
auto offset = fileHeaderOffset.lock();
@@ -63,6 +63,14 @@
rootNodeUnsaved = false;
}
+
+recentChanges++;
+
+if (recentChanges = 10)
+{
+rootNode-ClearCached();
+recentChanges = 0;
+}
}
templatetypename TKey, typename TValue
diff --git a/Indexes/IndexInnerNode.h b/Indexes/IndexInnerNode.h
index 3b039bc..5ccc1e1 100644
--- a/Indexes/IndexInnerNode.h
+++ b/Indexes/IndexInnerNode.h
@@ -41,6 +41,8 @@
virtual std::uint32_t RealLength() override;
virtual SplitResult Split() override;
+virtual void ClearCached() override;
+
virtual std::unique_ptrIndexNodeIteratorTKey, TValue begin() override;
virtual std::unique_ptrIndexNodeIteratorTKey, TValue end() override;
};
diff --git a/Indexes/IndexInnerNode.tpp b/Indexes/IndexInnerNode.tpp
index b6607c1..0890b2a 100644
--- a/Indexes/IndexInnerNode.tpp
+++ b/Indexes/IndexInnerNode.tpp
@@ -152,6 +152,8 @@
templatetypename TKey, typename TValue
void IndexInnerNodeTKey, TValue::Write()
{
+// TODO: don't do anything when there are no changes
+
IndexNodeTKey, TValue::Write();
for (auto cachedChild : cachedChildren)
@@ -217,6 +219,17 @@
}
templatetypename TKey, typename TValue
+void IndexInnerNodeTKey, TValue::ClearCached()
+{
+Write();
+
+for (unsigned i = 0; i cachedChildren.size(); i++)
+{
+cachedChildren.at(i) = nullptr;
+}
+}
+
+templatetypename TKey, typename TValue
std::unique_ptrIndexNodeIteratorTKey, TValue IndexInnerNodeTKey,
TValue::begin()
{
return std::unique_ptrIndexNodeIteratorTKey, TValue(new
IndexInnerIteratorTKey, TValue(this, true));
diff --git a/Indexes/IndexLeafNode.h b/Indexes/IndexLeafNode.h
index 1598488..ad069ac 100644
--- a/Indexes/IndexLeafNode.h
+++ b/Indexes/IndexLeafNode.h
@@ -32,6 +32,8 @@
virtual std::uint32_t RealLength() override;
virtual SplitResult Split() override;
+virtual void ClearCached() override;
+
[MediaWiki-commits] [Gerrit] Fixed memory issues - change (operations...incremental)
Petr Onderka has submitted this change and it was merged.
Change subject: Fixed memory issues
..
Fixed memory issues
- clearing index cache periodically, so that it doesn't grow too much
- fixed memory leaks
- less copying
Change-Id: I1483a2f4855a7faa751dd8ed7ee050b17a0c7204
---
M DumpObjects/DumpObject.h
M Indexes/Index.h
M Indexes/Index.tpp
M Indexes/IndexInnerNode.h
M Indexes/IndexInnerNode.tpp
M Indexes/IndexLeafNode.h
M Indexes/IndexLeafNode.tpp
M Indexes/IndexNode.h
M Indexes/Iterators/IndexNodeIterator.h
M XmlUtils.cpp
10 files changed, 47 insertions(+), 10 deletions(-)
Approvals:
Petr Onderka: Verified; Looks good to me, approved
diff --git a/DumpObjects/DumpObject.h b/DumpObjects/DumpObject.h
index ef21805..3efc910 100644
--- a/DumpObjects/DumpObject.h
+++ b/DumpObjects/DumpObject.h
@@ -16,18 +16,20 @@
virtual void WriteInternal() = 0;
templatetypename T
-void WriteValue(const T value);
+void WriteValue(const T value);
public:
templatetypename T
-static void WriteValue(std::ostream stream, const T value);
+static void WriteValue(std::ostream stream, const T value);
templatetypename T
static void ReadValue(std::istream stream, T value);
templatetypename T
-static std::uint32_t ValueSize(const T value);
+static std::uint32_t ValueSize(const T value);
virtual std::uint32_t NewLength() = 0;
+
+virtual ~DumpObjectBase() {}
};
class DumpObject : public DumpObjectBase
@@ -47,13 +49,13 @@
};
templatetypename T
-void DumpObjectBase::WriteValue(const T value)
+void DumpObjectBase::WriteValue(const T value)
{
WriteValue(*stream, value);
}
templatetypename T
-void DumpObjectBase::WriteValue(std::ostream stream, const T value)
+void DumpObjectBase::WriteValue(std::ostream stream, const T value)
{
DumpTraitsT::Write(stream, value);
}
@@ -65,7 +67,7 @@
}
templatetypename T
-uint32_t DumpObjectBase::ValueSize(const T value)
+uint32_t DumpObjectBase::ValueSize(const T value)
{
return DumpTraitsT::DumpSize(value);
}
\ No newline at end of file
diff --git a/Indexes/Index.h b/Indexes/Index.h
index 24e822c..ac77ddb 100644
--- a/Indexes/Index.h
+++ b/Indexes/Index.h
@@ -17,6 +17,8 @@
std::weak_ptrWritableDump dump;
std::weak_ptrOffset fileHeaderOffset;
+int recentChanges;
+
void AfterAdd();
public:
Index(std::weak_ptrWritableDump dump, std::weak_ptrOffset
fileHeaderOffset, bool delaySave = false);
diff --git a/Indexes/Index.tpp b/Indexes/Index.tpp
index 294d755..7ba38d5 100644
--- a/Indexes/Index.tpp
+++ b/Indexes/Index.tpp
@@ -7,7 +7,7 @@
templatetypename TKey, typename TValue
IndexTKey, TValue::Index(std::weak_ptrWritableDump dump,
std::weak_ptrOffset fileHeaderOffset, bool delaySave)
-: dump(dump), fileHeaderOffset(fileHeaderOffset)
+: dump(dump), fileHeaderOffset(fileHeaderOffset), recentChanges(0)
{
auto offset = fileHeaderOffset.lock();
@@ -63,6 +63,14 @@
rootNodeUnsaved = false;
}
+
+recentChanges++;
+
+if (recentChanges = 10)
+{
+rootNode-ClearCached();
+recentChanges = 0;
+}
}
templatetypename TKey, typename TValue
diff --git a/Indexes/IndexInnerNode.h b/Indexes/IndexInnerNode.h
index 3b039bc..5ccc1e1 100644
--- a/Indexes/IndexInnerNode.h
+++ b/Indexes/IndexInnerNode.h
@@ -41,6 +41,8 @@
virtual std::uint32_t RealLength() override;
virtual SplitResult Split() override;
+virtual void ClearCached() override;
+
virtual std::unique_ptrIndexNodeIteratorTKey, TValue begin() override;
virtual std::unique_ptrIndexNodeIteratorTKey, TValue end() override;
};
diff --git a/Indexes/IndexInnerNode.tpp b/Indexes/IndexInnerNode.tpp
index b6607c1..0890b2a 100644
--- a/Indexes/IndexInnerNode.tpp
+++ b/Indexes/IndexInnerNode.tpp
@@ -152,6 +152,8 @@
templatetypename TKey, typename TValue
void IndexInnerNodeTKey, TValue::Write()
{
+// TODO: don't do anything when there are no changes
+
IndexNodeTKey, TValue::Write();
for (auto cachedChild : cachedChildren)
@@ -217,6 +219,17 @@
}
templatetypename TKey, typename TValue
+void IndexInnerNodeTKey, TValue::ClearCached()
+{
+Write();
+
+for (unsigned i = 0; i cachedChildren.size(); i++)
+{
+cachedChildren.at(i) = nullptr;
+}
+}
+
+templatetypename TKey, typename TValue
std::unique_ptrIndexNodeIteratorTKey, TValue IndexInnerNodeTKey,
TValue::begin()
{
return std::unique_ptrIndexNodeIteratorTKey, TValue(new
IndexInnerIteratorTKey, TValue(this, true));
diff --git a/Indexes/IndexLeafNode.h b/Indexes/IndexLeafNode.h
index 1598488..ad069ac 100644
--- a/Indexes/IndexLeafNode.h
+++ b/Indexes/IndexLeafNode.h
@@ -32,6 +32,8 @@
virtual std::uint32_t RealLength() override;
virtual SplitResult Split() override;
+virtual void ClearCached() override;
+
virtual unique_ptrIndexNodeIteratorTKey, TValue begin()
