changeset b9a742cdd75a in /z/repo/gem5
details: http://repo.gem5.org/gem5?cmd=changeset;node=b9a742cdd75a
description:
kvm: Don't handle IO and execute in the same tick
We currently execute instructions in the guest and then handle any IO
request right after we break out of the virtualized environment. This
has the effect of executing IO requests in the exact same tick as the
first instruction in the sequence that was just run. There seem to be
cases where this simplification upsets some timing-sensitive devices.
This changeset splits execute and IO (and other services) across
multiple ticks. This is implemented by adding a separate
RunningService state to the CPU state machine. When a VM requires
service, it enters into this state and pending IO is then serviced in
the future instead of immediately. The delay between getting the
request and servicing it depends on the number of cycles executed in
the guest, which allows other components to catch up with the CPU.
diffstat:
src/cpu/kvm/arm_cpu.cc | 6 +-
src/cpu/kvm/arm_cpu.hh | 2 +-
src/cpu/kvm/base.cc | 395 ++++++++++++++++++++++++++++++++++++++----------
src/cpu/kvm/base.hh | 147 +++++++++++++++++-
4 files changed, 452 insertions(+), 98 deletions(-)
diffs (truncated from 781 to 300 lines):
diff -r a152d7f114b8 -r b9a742cdd75a src/cpu/kvm/arm_cpu.cc
--- a/src/cpu/kvm/arm_cpu.cc Tue Jun 11 09:24:40 2013 +0200
+++ b/src/cpu/kvm/arm_cpu.cc Tue Jun 11 09:24:51 2013 +0200
@@ -266,8 +266,8 @@
kvmArmVCpuInit(KVM_ARM_TARGET_CORTEX_A15);
}
-void
-ArmKvmCPU::tick()
+Tick
+ArmKvmCPU::kvmRun(Tick ticks)
{
bool simFIQ(interrupts->checkRaw(INT_FIQ));
bool simIRQ(interrupts->checkRaw(INT_IRQ));
@@ -283,7 +283,7 @@
vm.setIRQLine(INTERRUPT_VCPU_IRQ(vcpuID), simIRQ);
}
- BaseKvmCPU::tick();
+ return BaseKvmCPU::kvmRun(ticks);
}
void
diff -r a152d7f114b8 -r b9a742cdd75a src/cpu/kvm/arm_cpu.hh
--- a/src/cpu/kvm/arm_cpu.hh Tue Jun 11 09:24:40 2013 +0200
+++ b/src/cpu/kvm/arm_cpu.hh Tue Jun 11 09:24:51 2013 +0200
@@ -89,7 +89,7 @@
typedef std::vector<uint64_t> RegIndexVector;
- void tick();
+ Tick kvmRun(Tick ticks);
void updateKvmState();
void updateThreadContext();
diff -r a152d7f114b8 -r b9a742cdd75a src/cpu/kvm/base.cc
--- a/src/cpu/kvm/base.cc Tue Jun 11 09:24:40 2013 +0200
+++ b/src/cpu/kvm/base.cc Tue Jun 11 09:24:51 2013 +0200
@@ -49,6 +49,7 @@
#include "arch/utility.hh"
#include "cpu/kvm/base.hh"
#include "debug/Checkpoint.hh"
+#include "debug/Drain.hh"
#include "debug/Kvm.hh"
#include "debug/KvmIO.hh"
#include "debug/KvmRun.hh"
@@ -56,6 +57,8 @@
#include "sim/process.hh"
#include "sim/system.hh"
+#include <signal.h>
+
/* Used by some KVM macros */
#define PAGE_SIZE pageSize
@@ -81,6 +84,7 @@
tickEvent(*this),
perfControlledByTimer(params->usePerfOverflow),
hostFactor(params->hostFactor),
+ drainManager(NULL),
ctrInsts(0)
{
if (pageSize == -1)
@@ -94,7 +98,6 @@
threadContexts.push_back(tc);
setupCounters();
- setupSignalHandler();
if (params->usePerfOverflow)
runTimer.reset(new PerfKvmTimer(hwCycles,
@@ -151,6 +154,10 @@
// point. Initialize virtual CPUs here instead.
vcpuFD = vm.createVCPU(vcpuID);
+ // Setup signal handlers. This has to be done after the vCPU is
+ // created since it manipulates the vCPU signal mask.
+ setupSignalHandler();
+
// Map the KVM run structure */
vcpuMMapSize = kvm.getVCPUMMapSize();
_kvmRun = (struct kvm_run *)mmap(0, vcpuMMapSize,
@@ -232,9 +239,6 @@
dump();
}
- // Update the thread context so we have something to serialize.
- syncThreadContext();
-
assert(tid == 0);
assert(_status == Idle);
thread->serialize(os);
@@ -258,15 +262,62 @@
if (switchedOut())
return 0;
- DPRINTF(Kvm, "drain\n");
+ DPRINTF(Drain, "BaseKvmCPU::drain\n");
+ switch (_status) {
+ case Running:
+ // The base KVM code is normally ready when it is in the
+ // Running state, but the architecture specific code might be
+ // of a different opinion. This may happen when the CPU been
+ // notified of an event that hasn't been accepted by the vCPU
+ // yet.
+ if (!archIsDrained()) {
+ drainManager = dm;
+ return 1;
+ }
- // De-schedule the tick event so we don't insert any more MMIOs
- // into the system while it is draining.
- if (tickEvent.scheduled())
- deschedule(tickEvent);
+ // The state of the CPU is consistent, so we don't need to do
+ // anything special to drain it. We simply de-schedule the
+ // tick event and enter the Idle state to prevent nasty things
+ // like MMIOs from happening.
+ if (tickEvent.scheduled())
+ deschedule(tickEvent);
+ _status = Idle;
- _status = Idle;
- return 0;
+ /** FALLTHROUGH */
+ case Idle:
+ // Idle, no need to drain
+ assert(!tickEvent.scheduled());
+
+ // Sync the thread context here since we'll need it when we
+ // switch CPUs or checkpoint the CPU.
+ syncThreadContext();
+
+ return 0;
+
+ case RunningServiceCompletion:
+ // The CPU has just requested a service that was handled in
+ // the RunningService state, but the results have still not
+ // been reported to the CPU. Now, we /could/ probably just
+ // update the register state ourselves instead of letting KVM
+ // handle it, but that would be tricky. Instead, we enter KVM
+ // and let it do its stuff.
+ drainManager = dm;
+
+ DPRINTF(Drain, "KVM CPU is waiting for service completion, "
+ "requesting drain.\n");
+ return 1;
+
+ case RunningService:
+ // We need to drain since the CPU is waiting for service (e.g., MMIOs)
+ drainManager = dm;
+
+ DPRINTF(Drain, "KVM CPU is waiting for service, requesting drain.\n");
+ return 1;
+
+ default:
+ panic("KVM: Unhandled CPU state in drain()\n");
+ return 0;
+ }
}
void
@@ -297,10 +348,6 @@
{
DPRINTF(Kvm, "switchOut\n");
- // Make sure to update the thread context in case, the new CPU
- // will need to access it.
- syncThreadContext();
-
BaseCPU::switchOut();
// We should have drained prior to executing a switchOut, which
@@ -324,9 +371,12 @@
assert(_status == Idle);
assert(threadContexts.size() == 1);
- // The BaseCPU updated the thread context, make sure that we
- // synchronize next time we enter start the CPU.
- threadContextDirty = true;
+ // Force an update of the KVM state here instead of flagging the
+ // TC as dirty. This is not ideal from a performance point of
+ // view, but it makes debugging easier as it allows meaningful KVM
+ // state to be dumped before and after a takeover.
+ updateKvmState();
+ threadContextDirty = false;
}
void
@@ -436,25 +486,73 @@
void
BaseKvmCPU::tick()
{
- assert(_status == Running);
-
- DPRINTF(KvmRun, "Entering KVM...\n");
-
- Tick ticksToExecute(mainEventQueue.nextTick() - curTick());
- Tick ticksExecuted(kvmRun(ticksToExecute));
-
- Tick delay(ticksExecuted + handleKvmExit());
+ Tick delay(0);
+ assert(_status != Idle);
switch (_status) {
- case Running:
- schedule(tickEvent, clockEdge(ticksToCycles(delay)));
+ case RunningService:
+ // handleKvmExit() will determine the next state of the CPU
+ delay = handleKvmExit();
+
+ if (tryDrain())
+ _status = Idle;
break;
+ case RunningServiceCompletion:
+ case Running: {
+ Tick ticksToExecute(mainEventQueue.nextTick() - curTick());
+
+ // We might need to update the KVM state.
+ syncKvmState();
+
+ DPRINTF(KvmRun, "Entering KVM...\n");
+ if (drainManager) {
+ // Force an immediate exit from KVM after completing
+ // pending operations. The architecture-specific code
+ // takes care to run until it is in a state where it can
+ // safely be drained.
+ delay = kvmRunDrain();
+ } else {
+ delay = kvmRun(ticksToExecute);
+ }
+
+ // Entering into KVM implies that we'll have to reload the thread
+ // context from KVM if we want to access it. Flag the KVM state as
+ // dirty with respect to the cached thread context.
+ kvmStateDirty = true;
+
+ // Enter into the RunningService state unless the
+ // simulation was stopped by a timer.
+ if (_kvmRun->exit_reason != KVM_EXIT_INTR)
+ _status = RunningService;
+ else
+ _status = Running;
+
+ if (tryDrain())
+ _status = Idle;
+ } break;
+
default:
- /* The CPU is halted or waiting for an interrupt from a
- * device. Don't start it. */
- break;
+ panic("BaseKvmCPU entered tick() in an illegal state (%i)\n",
+ _status);
}
+
+ // Schedule a new tick if we are still running
+ if (_status != Idle)
+ schedule(tickEvent, clockEdge(ticksToCycles(delay)));
+}
+
+Tick
+BaseKvmCPU::kvmRunDrain()
+{
+ // By default, the only thing we need to drain is a pending IO
+ // operation which assumes that we are in the
+ // RunningServiceCompletion state.
+ assert(_status == RunningServiceCompletion);
+
+ // Deliver the data from the pending IO operation and immediately
+ // exit.
+ return kvmRun(0);
}
uint64_t
@@ -466,68 +564,91 @@
Tick
BaseKvmCPU::kvmRun(Tick ticks)
{
- // We might need to update the KVM state.
- syncKvmState();
- // Entering into KVM implies that we'll have to reload the thread
- // context from KVM if we want to access it. Flag the KVM state as
- // dirty with respect to the cached thread context.
- kvmStateDirty = true;
-
- if (ticks < runTimer->resolution()) {
- DPRINTF(KvmRun, "KVM: Adjusting tick count (%i -> %i)\n",
- ticks, runTimer->resolution());
- ticks = runTimer->resolution();
- }
-
+ Tick ticksExecuted;
DPRINTF(KvmRun, "KVM: Executing for %i ticks\n", ticks);
timerOverflowed = false;
- // Get hardware statistics after synchronizing contexts. The KVM
- // state update might affect guest cycle counters.
- uint64_t baseCycles(getHostCycles());
_______________________________________________
gem5-dev mailing list
[email protected]
http://m5sim.org/mailman/listinfo/gem5-dev
