Re: [KVM][Kemari]: Build error fix
On 2012/06/28 5:34, Sterling Windmill wrote:
Is Kemari still in active development?
No, it's not. Currently we have no intention to add new features into
Kemari.
Thanks,
Kei
Best regards,
Sterling Windmill
On Sun, Dec 4, 2011 at 9:45 PM, OHMURA Kei ohmura@lab.ntt.co.jp
mailto:ohmura@lab.ntt.co.jp wrote:
On 2011/12/02 21:51, Pradeep Kumar wrote:
It fixes build failure.
I hit this error, after succsfull migration and sync.
(qemu) qemu-system-x86_64: fill buffer failed, Interrupted system call
qemu-system-x86_64: recv header failed
qemu-system-x86_64: recv ack failed
qemu_transaction_begin failed
Did you use master branch?
It is not latest version. next branch is latest and fixed error.
git://kemari.git.sourceforge.net/gitroot/kemari/kemari
http://kemari.git.sourceforge.net/gitroot/kemari/kemari next
Thanks,
Kei
Any one working on this now?
From 827c04da6574be80d8352acd7c40b0b4524af5f4 Mon Sep 17 00:00:00 2001
Date: Fri, 2 Dec 2011 18:11:40 +0530
Subject: [PATCH] [Qemu][Kemari]: Build Failure
Signed-off-by: pradeep psuri...@linux.vnet.ibm.com
mailto:psuri...@linux.vnet.ibm.com
modified: ft_trans_file.c
---
ft_trans_file.c |2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/ft_trans_file.c b/ft_trans_file.c
index 4e33034..dc36757 100644
--- a/ft_trans_file.c
+++ b/ft_trans_file.c
@@ -174,7 +174,7 @@ static int ft_trans_send_header(QEMUFileFtTrans *s,
static int ft_trans_put_buffer(void *opaque, const uint8_t *buf,
int64_t pos, int size)
{
QEMUFileFtTrans *s = opaque;
-ssize_t ret;
+ssize_t ret = 0;
trace_ft_trans_put_buffer(size, pos);
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
mailto:majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
--
大村 圭
NTTソフトウェアイノベーションセンタ
分散処理基盤技術プロジェクト
E-mail: ohmura@lab.ntt.co.jp
TEL: 0422-59-2823
FAX: 0422-59-2965
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [KVM][Kemari]: Build error fix
On 2011/12/02 21:51, Pradeep Kumar wrote:
It fixes build failure.
I hit this error, after succsfull migration and sync.
(qemu) qemu-system-x86_64: fill buffer failed, Interrupted system call
qemu-system-x86_64: recv header failed
qemu-system-x86_64: recv ack failed
qemu_transaction_begin failed
Did you use master branch?
It is not latest version. next branch is latest and fixed error.
git://kemari.git.sourceforge.net/gitroot/kemari/kemari next
Thanks,
Kei
Any one working on this now?
From 827c04da6574be80d8352acd7c40b0b4524af5f4 Mon Sep 17 00:00:00 2001
Date: Fri, 2 Dec 2011 18:11:40 +0530
Subject: [PATCH] [Qemu][Kemari]: Build Failure
Signed-off-by: pradeep psuri...@linux.vnet.ibm.com
modified: ft_trans_file.c
---
ft_trans_file.c |2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/ft_trans_file.c b/ft_trans_file.c
index 4e33034..dc36757 100644
--- a/ft_trans_file.c
+++ b/ft_trans_file.c
@@ -174,7 +174,7 @@ static int ft_trans_send_header(QEMUFileFtTrans *s,
static int ft_trans_put_buffer(void *opaque, const uint8_t *buf, int64_t
pos, int size)
{
QEMUFileFtTrans *s = opaque;
-ssize_t ret;
+ssize_t ret = 0;
trace_ft_trans_put_buffer(size, pos);
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 13/18] net: insert event-tap to qemu_send_packet() and qemu_sendv_packet_async().
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
event-tap function is called only when it is on.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
net.c |9 +
1 files changed, 9 insertions(+), 0 deletions(-)
diff --git a/net.c b/net.c
index 4f777c3..8bcc504 100644
--- a/net.c
+++ b/net.c
@@ -36,6 +36,7 @@
#include qemu_socket.h
#include hw/qdev.h
#include iov.h
+#include event-tap.h
static QTAILQ_HEAD(, VLANState) vlans;
static QTAILQ_HEAD(, VLANClientState) non_vlan_clients;
@@ -518,6 +519,10 @@ ssize_t qemu_send_packet_async(VLANClientState *sender,
void qemu_send_packet(VLANClientState *vc, const uint8_t *buf, int size)
{
+if (event_tap_is_on()) {
+return event_tap_send_packet(vc, buf, size);
+}
+
qemu_send_packet_async(vc, buf, size, NULL);
}
@@ -599,6 +604,10 @@ ssize_t qemu_sendv_packet_async(VLANClientState *sender,
{
NetQueue *queue;
+if (event_tap_is_on()) {
+return event_tap_sendv_packet_async(sender, iov, iovcnt, sent_cb);
+}
+
if (sender-link_down || (!sender-peer !sender-vlan)) {
return iov_size(iov, iovcnt);
}
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 04/18] qemu-char: export socket_set_nodelay().
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
qemu-char.c |2 +-
qemu_socket.h |1 +
2 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/qemu-char.c b/qemu-char.c
index 03858d4..fef33b0 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -2115,7 +2115,7 @@ static void tcp_chr_telnet_init(int fd)
send(fd, (char *)buf, 3, 0);
}
-static void socket_set_nodelay(int fd)
+void socket_set_nodelay(int fd)
{
int val = 1;
setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *)val, sizeof(val));
diff --git a/qemu_socket.h b/qemu_socket.h
index 180e4db..a05e1e5 100644
--- a/qemu_socket.h
+++ b/qemu_socket.h
@@ -36,6 +36,7 @@ int inet_aton(const char *cp, struct in_addr *ia);
int qemu_socket(int domain, int type, int protocol);
int qemu_accept(int s, struct sockaddr *addr, socklen_t *addrlen);
void socket_set_nonblock(int fd);
+void socket_set_nodelay(int fd);
int send_all(int fd, const void *buf, int len1);
/* New, ipv6-ready socket helper functions, see qemu-sockets.c */
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 02/18] Introduce read() to FdMigrationState.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Currently FdMigrationState doesn't support read(), and this patch
introduces it to get response from the other side. Note that this
won't change the existing migration protocol to be bi-directional.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
migration-tcp.c | 15 +++
migration.c | 13 +
migration.h |3 +++
3 files changed, 31 insertions(+), 0 deletions(-)
diff --git a/migration-tcp.c b/migration-tcp.c
index d3d80c9..bb67d53 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -38,6 +38,20 @@ static int socket_write(FdMigrationState *s, const void *
buf, size_t size)
return send(s-fd, buf, size, 0);
}
+static int socket_read(FdMigrationState *s, const void * buf, size_t size)
+{
+ssize_t len;
+
+do {
+len = recv(s-fd, (void *)buf, size, 0);
+} while (len == -1 socket_error() == EINTR);
+if (len == -1) {
+len = -socket_error();
+}
+
+return len;
+}
+
static int tcp_close(FdMigrationState *s)
{
DPRINTF(tcp_close\n);
@@ -93,6 +107,7 @@ MigrationState *tcp_start_outgoing_migration(Monitor *mon,
s-get_error = socket_errno;
s-write = socket_write;
+s-read = socket_read;
s-close = tcp_close;
s-mig_state.cancel = migrate_fd_cancel;
s-mig_state.get_status = migrate_fd_get_status;
diff --git a/migration.c b/migration.c
index af3a1f2..302b8fe 100644
--- a/migration.c
+++ b/migration.c
@@ -340,6 +340,19 @@ ssize_t migrate_fd_put_buffer(void *opaque, const void
*data, size_t size)
return ret;
}
+int migrate_fd_get_buffer(void *opaque, uint8_t *data, int64_t pos, size_t
size)
+{
+FdMigrationState *s = opaque;
+int ret;
+
+ret = s-read(s, data, size);
+if (ret == -1) {
+ret = -(s-get_error(s));
+}
+
+return ret;
+}
+
void migrate_fd_connect(FdMigrationState *s)
{
int ret;
diff --git a/migration.h b/migration.h
index 050c56c..6a76f77 100644
--- a/migration.h
+++ b/migration.h
@@ -48,6 +48,7 @@ struct FdMigrationState
int (*get_error)(struct FdMigrationState*);
int (*close)(struct FdMigrationState*);
int (*write)(struct FdMigrationState*, const void *, size_t);
+int (*read)(struct FdMigrationState *, const void *, size_t);
void *opaque;
};
@@ -116,6 +117,8 @@ void migrate_fd_put_notify(void *opaque);
ssize_t migrate_fd_put_buffer(void *opaque, const void *data, size_t size);
+int migrate_fd_get_buffer(void *opaque, uint8_t *data, int64_t pos, size_t
size);
+
void migrate_fd_connect(FdMigrationState *s);
void migrate_fd_put_ready(void *opaque);
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 08/18] savevm: introduce util functions to control ft_trans_file from savevm layer.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
To utilize ft_trans_file function, savevm needs interfaces to be
exported.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
hw/hw.h |5 ++
savevm.c | 150 ++
2 files changed, 155 insertions(+), 0 deletions(-)
diff --git a/hw/hw.h b/hw/hw.h
index f90ff15..2d4d595 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -51,6 +51,7 @@ QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc
*put_buffer,
QEMUFile *qemu_fopen(const char *filename, const char *mode);
QEMUFile *qemu_fdopen(int fd, const char *mode);
QEMUFile *qemu_fopen_socket(int fd);
+QEMUFile *qemu_fopen_ft_trans(int s_fd, int c_fd);
QEMUFile *qemu_popen(FILE *popen_file, const char *mode);
QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
int qemu_stdio_fd(QEMUFile *f);
@@ -60,6 +61,9 @@ void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int
size);
void qemu_put_byte(QEMUFile *f, int v);
void *qemu_realloc_buffer(QEMUFile *f, int size);
void qemu_clear_buffer(QEMUFile *f);
+int qemu_ft_trans_begin(QEMUFile *f);
+int qemu_ft_trans_commit(QEMUFile *f);
+int qemu_ft_trans_cancel(QEMUFile *f);
static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
{
@@ -94,6 +98,7 @@ void qemu_file_set_error(QEMUFile *f);
* halted due to rate limiting or EAGAIN errors occur as it can be used to
* resume output. */
void qemu_file_put_notify(QEMUFile *f);
+void qemu_file_get_notify(void *opaque);
static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
{
diff --git a/savevm.c b/savevm.c
index d017760..5b57e94 100644
--- a/savevm.c
+++ b/savevm.c
@@ -83,6 +83,7 @@
#include qemu_socket.h
#include qemu-queue.h
#include cpus.h
+#include ft_trans_file.h
#define SELF_ANNOUNCE_ROUNDS 5
@@ -190,6 +191,13 @@ typedef struct QEMUFileSocket
QEMUFile *file;
} QEMUFileSocket;
+typedef struct QEMUFileSocketTrans
+{
+int fd;
+QEMUFileSocket *s;
+VMChangeStateEntry *e;
+} QEMUFileSocketTrans;
+
static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
{
QEMUFileSocket *s = opaque;
@@ -205,6 +213,22 @@ static int socket_get_buffer(void *opaque, uint8_t *buf,
int64_t pos, int size)
return len;
}
+static ssize_t socket_put_buffer(void *opaque, const void *buf, size_t size)
+{
+QEMUFileSocket *s = opaque;
+ssize_t len;
+
+do {
+len = send(s-fd, (void *)buf, size, 0);
+} while (len == -1 socket_error() == EINTR);
+
+if (len == -1) {
+len = -socket_error();
+}
+
+return len;
+}
+
static int socket_close(void *opaque)
{
QEMUFileSocket *s = opaque;
@@ -212,6 +236,71 @@ static int socket_close(void *opaque)
return 0;
}
+static int socket_trans_get_buffer(void *opaque, uint8_t *buf, int64_t pos,
size_t size)
+{
+QEMUFileSocketTrans *t = opaque;
+QEMUFileSocket *s = t-s;
+ssize_t len;
+
+len = socket_get_buffer(s, buf, pos, size);
+
+return len;
+}
+
+static ssize_t socket_trans_put_buffer(void *opaque, const void *buf, size_t
size)
+{
+QEMUFileSocketTrans *t = opaque;
+
+return socket_put_buffer(t-s, buf, size);
+}
+
+static int qemu_loadvm_state_no_header(QEMUFile *f);
+
+static int socket_trans_get_ready(void *opaque)
+{
+QEMUFileSocketTrans *t = opaque;
+QEMUFileSocket *s = t-s;
+QEMUFile *f = s-file;
+int ret = 0;
+
+ret = qemu_loadvm_state_no_header(f);
+if (ret 0) {
+fprintf(stderr,
+socket_trans_get_ready: error while loading vmstate\n);
+}
+
+return ret;
+}
+
+static int socket_trans_close(void *opaque)
+{
+QEMUFileSocketTrans *t = opaque;
+QEMUFileSocket *s = t-s;
+
+qemu_set_fd_handler2(s-fd, NULL, NULL, NULL, NULL);
+qemu_set_fd_handler2(t-fd, NULL, NULL, NULL, NULL);
+qemu_del_vm_change_state_handler(t-e);
+close(s-fd);
+close(t-fd);
+qemu_free(s);
+qemu_free(t);
+
+return 0;
+}
+
+static void socket_trans_resume(void *opaque, int running, int reason)
+{
+QEMUFileSocketTrans *t = opaque;
+QEMUFileSocket *s = t-s;
+
+if (!running) {
+return;
+}
+
+qemu_announce_self();
+qemu_fclose(s-file);
+}
+
static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int
size)
{
QEMUFileStdio *s = opaque;
@@ -334,6 +423,26 @@ QEMUFile *qemu_fopen_socket(int fd)
return s-file;
}
+QEMUFile *qemu_fopen_ft_trans(int s_fd, int c_fd)
+{
+QEMUFileSocketTrans *t = qemu_mallocz(sizeof(QEMUFileSocketTrans));
+QEMUFileSocket *s = qemu_mallocz(sizeof(QEMUFileSocket));
+
+t-s = s;
+t-fd = s_fd;
+t-e = qemu_add_vm_change_state_handler(socket_trans_resume, t);
+
+s-fd = c_fd;
+s-file = qemu_fopen_ops_ft_trans(t, socket_trans_put_buffer,
+ socket_trans_get_buffer, NULL
[PATCH 01/18] Make QEMUFile buf expandable, and introduce qemu_realloc_buffer() and qemu_clear_buffer().
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Currently buf size is fixed at 32KB. It would be useful if it could
be flexible.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
hw/hw.h |2 ++
savevm.c | 20 +++-
2 files changed, 21 insertions(+), 1 deletions(-)
diff --git a/hw/hw.h b/hw/hw.h
index 1b09039..f90ff15 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -58,6 +58,8 @@ void qemu_fflush(QEMUFile *f);
int qemu_fclose(QEMUFile *f);
void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
void qemu_put_byte(QEMUFile *f, int v);
+void *qemu_realloc_buffer(QEMUFile *f, int size);
+void qemu_clear_buffer(QEMUFile *f);
static inline void qemu_put_ubyte(QEMUFile *f, unsigned int v)
{
diff --git a/savevm.c b/savevm.c
index f4ff1a1..9cf0258 100644
--- a/savevm.c
+++ b/savevm.c
@@ -172,7 +172,8 @@ struct QEMUFile {
when reading */
int buf_index;
int buf_size; /* 0 when writing */
-uint8_t buf[IO_BUF_SIZE];
+int buf_max_size;
+uint8_t *buf;
int has_error;
};
@@ -423,6 +424,9 @@ QEMUFile *qemu_fopen_ops(void *opaque,
QEMUFilePutBufferFunc *put_buffer,
f-get_rate_limit = get_rate_limit;
f-is_write = 0;
+f-buf_max_size = IO_BUF_SIZE;
+f-buf = qemu_malloc(sizeof(uint8_t) * f-buf_max_size);
+
return f;
}
@@ -453,6 +457,19 @@ void qemu_fflush(QEMUFile *f)
}
}
+void *qemu_realloc_buffer(QEMUFile *f, int size)
+{
+f-buf_max_size = size;
+f-buf = qemu_realloc(f-buf, f-buf_max_size);
+
+return f-buf;
+}
+
+void qemu_clear_buffer(QEMUFile *f)
+{
+f-buf_size = f-buf_index = f-buf_offset = 0;
+}
+
static void qemu_fill_buffer(QEMUFile *f)
{
int len;
@@ -478,6 +495,7 @@ int qemu_fclose(QEMUFile *f)
qemu_fflush(f);
if (f-close)
ret = f-close(f-opaque);
+qemu_free(f-buf);
qemu_free(f);
return ret;
}
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 16/18] migration: introduce migrate_ft_trans_{put,get}_ready(), and modify migrate_fd_put_ready() when ft_mode is on.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Introduce migrate_ft_trans_put_ready() which kicks the FT transaction
cycle. When ft_mode is on, migrate_fd_put_ready() would open
ft_trans_file and turn on event_tap. To end or cancel FT transaction,
ft_mode and event_tap is turned off. migrate_ft_trans_get_ready() is
called to receive ack from the receiver.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
migration.c | 266 ++-
1 files changed, 265 insertions(+), 1 deletions(-)
diff --git a/migration.c b/migration.c
index 1c2d956..d536df0 100644
--- a/migration.c
+++ b/migration.c
@@ -21,6 +21,7 @@
#include qemu_socket.h
#include block-migration.h
#include qemu-objects.h
+#include event-tap.h
//#define DEBUG_MIGRATION
@@ -283,6 +284,17 @@ void migrate_fd_error(FdMigrationState *s)
migrate_fd_cleanup(s);
}
+static void migrate_ft_trans_error(FdMigrationState *s)
+{
+ft_mode = FT_ERROR;
+qemu_savevm_state_cancel(s-mon, s-file);
+migrate_fd_error(s);
+/* we need to set vm running to avoid assert in virtio-net */
+vm_start();
+event_tap_unregister();
+vm_stop(0);
+}
+
int migrate_fd_cleanup(FdMigrationState *s)
{
int ret = 0;
@@ -318,6 +330,17 @@ void migrate_fd_put_notify(void *opaque)
qemu_file_put_notify(s-file);
}
+static void migrate_fd_get_notify(void *opaque)
+{
+FdMigrationState *s = opaque;
+
+qemu_set_fd_handler2(s-fd, NULL, NULL, NULL, NULL);
+qemu_file_get_notify(s-file);
+if (qemu_file_has_error(s-file)) {
+migrate_ft_trans_error(s);
+}
+}
+
ssize_t migrate_fd_put_buffer(void *opaque, const void *data, size_t size)
{
FdMigrationState *s = opaque;
@@ -353,6 +376,10 @@ int migrate_fd_get_buffer(void *opaque, uint8_t *data,
int64_t pos, size_t size)
ret = -(s-get_error(s));
}
+if (ret == -EAGAIN) {
+qemu_set_fd_handler2(s-fd, NULL, migrate_fd_get_notify, NULL, s);
+}
+
return ret;
}
@@ -379,6 +406,230 @@ void migrate_fd_connect(FdMigrationState *s)
migrate_fd_put_ready(s);
}
+static int migrate_ft_trans_commit(void *opaque)
+{
+FdMigrationState *s = opaque;
+int ret = -1;
+
+if (ft_mode != FT_TRANSACTION_COMMIT ft_mode != FT_TRANSACTION_ATOMIC) {
+fprintf(stderr,
+migrate_ft_trans_commit: invalid ft_mode %d\n, ft_mode);
+goto out;
+}
+
+do {
+if (ft_mode == FT_TRANSACTION_ATOMIC) {
+if (qemu_ft_trans_begin(s-file) 0) {
+fprintf(stderr, qemu_ft_trans_begin failed\n);
+goto out;
+}
+
+ret = qemu_savevm_trans_begin(s-mon, s-file, 0);
+if (ret 0) {
+fprintf(stderr, qemu_savevm_trans_begin failed\n);
+goto out;
+}
+
+ft_mode = FT_TRANSACTION_COMMIT;
+if (ret) {
+/* don't proceed until if fd isn't ready */
+goto out;
+}
+}
+
+/* make the VM state consistent by flushing outstanding events */
+vm_stop(0);
+
+/* send at full speed */
+qemu_file_set_rate_limit(s-file, 0);
+
+ret = qemu_savevm_trans_complete(s-mon, s-file);
+if (ret 0) {
+fprintf(stderr, qemu_savevm_trans_complete failed\n);
+goto out;
+}
+
+ret = qemu_ft_trans_commit(s-file);
+if (ret 0) {
+fprintf(stderr, qemu_ft_trans_commit failed\n);
+goto out;
+}
+
+if (ret) {
+ft_mode = FT_TRANSACTION_RECV;
+ret = 1;
+goto out;
+}
+
+/* flush and check if events are remaining */
+vm_start();
+ret = event_tap_flush_one();
+if (ret 0) {
+fprintf(stderr, event_tap_flush_one failed\n);
+goto out;
+}
+
+ft_mode = ret ? FT_TRANSACTION_BEGIN : FT_TRANSACTION_ATOMIC;
+} while (ft_mode != FT_TRANSACTION_BEGIN);
+
+vm_start();
+ret = 0;
+
+out:
+return ret;
+}
+
+static int migrate_ft_trans_get_ready(void *opaque)
+{
+FdMigrationState *s = opaque;
+int ret = -1;
+
+if (ft_mode != FT_TRANSACTION_RECV) {
+fprintf(stderr,
+migrate_ft_trans_get_ready: invalid ft_mode %d\n, ft_mode);
+goto error_out;
+}
+
+/* flush and check if events are remaining */
+vm_start();
+ret = event_tap_flush_one();
+if (ret 0) {
+fprintf(stderr, event_tap_flush_one failed\n);
+goto error_out;
+}
+
+if (ret) {
+ft_mode = FT_TRANSACTION_BEGIN;
+} else {
+ft_mode = FT_TRANSACTION_ATOMIC;
+
+ret = migrate_ft_trans_commit(s);
+if (ret 0) {
+goto error_out;
+}
+if (ret) {
+goto out;
+}
+}
+
+vm_start();
+ret = 0
[PATCH 03/18] Introduce qemu_loadvm_state_no_header() and make qemu_loadvm_state() a wrapper.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Introduce qemu_loadvm_state_no_header() so that it can be called
iteratively without reading the header, and qemu_loadvm_state()
becomes a wrapper of it.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
savevm.c | 45 +++--
1 files changed, 27 insertions(+), 18 deletions(-)
diff --git a/savevm.c b/savevm.c
index 9cf0258..d017760 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1744,30 +1744,14 @@ typedef struct LoadStateEntry {
int version_id;
} LoadStateEntry;
-int qemu_loadvm_state(QEMUFile *f)
+static int qemu_loadvm_state_no_header(QEMUFile *f)
{
QLIST_HEAD(, LoadStateEntry) loadvm_handlers =
QLIST_HEAD_INITIALIZER(loadvm_handlers);
LoadStateEntry *le, *new_le;
uint8_t section_type;
-unsigned int v;
-int ret;
-
-if (qemu_savevm_state_blocked(default_mon)) {
-return -EINVAL;
-}
-
-v = qemu_get_be32(f);
-if (v != QEMU_VM_FILE_MAGIC)
-return -EINVAL;
-v = qemu_get_be32(f);
-if (v == QEMU_VM_FILE_VERSION_COMPAT) {
-fprintf(stderr, SaveVM v2 format is obsolete and don't work
anymore\n);
-return -ENOTSUP;
-}
-if (v != QEMU_VM_FILE_VERSION)
-return -ENOTSUP;
+int ret;
while ((section_type = qemu_get_byte(f)) != QEMU_VM_EOF) {
uint32_t instance_id, version_id, section_id;
@@ -1862,6 +1846,31 @@ out:
return ret;
}
+int qemu_loadvm_state(QEMUFile *f)
+{
+unsigned int v;
+
+if (qemu_savevm_state_blocked(default_mon)) {
+return -EINVAL;
+}
+
+v = qemu_get_be32(f);
+if (v != QEMU_VM_FILE_MAGIC) {
+return -EINVAL;
+}
+
+v = qemu_get_be32(f);
+if (v == QEMU_VM_FILE_VERSION_COMPAT) {
+fprintf(stderr, SaveVM v2 format is obsolete and don't work
anymore\n);
+return -ENOTSUP;
+}
+if (v != QEMU_VM_FILE_VERSION) {
+return -ENOTSUP;
+}
+
+return qemu_loadvm_state_no_header(f);
+}
+
static int bdrv_snapshot_find(BlockDriverState *bs, QEMUSnapshotInfo *sn_info,
const char *name)
{
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 06/18] virtio: decrement last_avail_idx with inuse before saving.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
For regular migration inuse == 0 always as requests are flushed before
save. However, event-tap log when enabled introduces an extra queue
for requests which is not being flushed, thus the last inuse requests
are left in the event-tap queue. Move the last_avail_idx value sent
to the remote back to make it repeat the last inuse requests.
Signed-off-by: Michael S. Tsirkin m...@redhat.com
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
---
hw/virtio.c | 10 +-
1 files changed, 9 insertions(+), 1 deletions(-)
diff --git a/hw/virtio.c b/hw/virtio.c
index 6e8814c..d342e25 100644
--- a/hw/virtio.c
+++ b/hw/virtio.c
@@ -672,12 +672,20 @@ void virtio_save(VirtIODevice *vdev, QEMUFile *f)
qemu_put_be32(f, i);
for (i = 0; i VIRTIO_PCI_QUEUE_MAX; i++) {
+/* For regular migration inuse == 0 always as
+ * requests are flushed before save. However,
+ * event-tap log when enabled introduces an extra
+ * queue for requests which is not being flushed,
+ * thus the last inuse requests are left in the event-tap queue.
+ * Move the last_avail_idx value sent to the remote back
+ * to make it repeat the last inuse requests. */
+uint16_t last_avail = vdev-vq[i].last_avail_idx - vdev-vq[i].inuse;
if (vdev-vq[i].vring.num == 0)
break;
qemu_put_be32(f, vdev-vq[i].vring.num);
qemu_put_be64(f, vdev-vq[i].pa);
-qemu_put_be16s(f, vdev-vq[i].last_avail_idx);
+qemu_put_be16s(f, last_avail);
if (vdev-binding-save_queue)
vdev-binding-save_queue(vdev-binding_opaque, i, f);
}
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 05/18] vl.c: add deleted flag for deleting the handler.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Make deleting handlers robust against deletion of any elements in a
handler by using a deleted flag like in file descriptors.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
vl.c | 15 ++-
1 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/vl.c b/vl.c
index 68c3b53..a18792d 100644
--- a/vl.c
+++ b/vl.c
@@ -1096,6 +1096,7 @@ static void nographic_update(void *opaque)
struct vm_change_state_entry {
VMChangeStateHandler *cb;
void *opaque;
+int deleted;
QLIST_ENTRY (vm_change_state_entry) entries;
};
@@ -1116,18 +1117,22 @@ VMChangeStateEntry
*qemu_add_vm_change_state_handler(VMChangeStateHandler *cb,
void qemu_del_vm_change_state_handler(VMChangeStateEntry *e)
{
-QLIST_REMOVE (e, entries);
-qemu_free (e);
+e-deleted = 1;
}
void vm_state_notify(int running, int reason)
{
-VMChangeStateEntry *e;
+VMChangeStateEntry *e, *ne;
trace_vm_state_notify(running, reason);
-for (e = vm_change_state_head.lh_first; e; e = e-entries.le_next) {
-e-cb(e-opaque, running, reason);
+QLIST_FOREACH_SAFE(e, vm_change_state_head, entries, ne) {
+if (e-deleted) {
+QLIST_REMOVE(e, entries);
+qemu_free(e);
+} else {
+e-cb(e-opaque, running, reason);
+}
}
}
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 18/18] Introduce kemari: to enable FT migration mode (Kemari).
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
When kemari: is set in front of URI of migrate command, it will turn
on ft_mode to start FT migration mode (Kemari). On the receiver side,
the option looks like, -incoming kemari:protocol:address:port
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Acked-by: Paolo Bonzini pbonz...@redhat.com
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
hmp-commands.hx |4 +++-
migration.c | 12
qmp-commands.hx |4 +++-
3 files changed, 18 insertions(+), 2 deletions(-)
diff --git a/hmp-commands.hx b/hmp-commands.hx
index 834e6a8..4cd7bfa 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -760,7 +760,9 @@ ETEXI
\n\t\t\t -b for migration without shared storage with
full copy of disk\n\t\t\t -i for migration without
shared storage with incremental copy of disk
- (base image shared between src and destination),
+ (base image shared between src and destination)
+ \n\t\t\t put \kemari:\ in front of URI to enable
+ Fault Tolerance mode (Kemari protocol),
.user_print = monitor_user_noop,
.mhandler.cmd_new = do_migrate,
},
diff --git a/migration.c b/migration.c
index d536df0..5017dea 100644
--- a/migration.c
+++ b/migration.c
@@ -48,6 +48,12 @@ int qemu_start_incoming_migration(const char *uri)
const char *p;
int ret;
+/* check ft_mode (Kemari protocol) */
+if (strstart(uri, kemari:, p)) {
+ft_mode = FT_INIT;
+uri = p;
+}
+
if (strstart(uri, tcp:, p))
ret = tcp_start_incoming_migration(p);
#if !defined(WIN32)
@@ -99,6 +105,12 @@ int do_migrate(Monitor *mon, const QDict *qdict, QObject
**ret_data)
return -1;
}
+/* check ft_mode (Kemari protocol) */
+if (strstart(uri, kemari:, p)) {
+ft_mode = FT_INIT;
+uri = p;
+}
+
if (strstart(uri, tcp:, p)) {
s = tcp_start_outgoing_migration(mon, p, max_throttle, detach,
blk, inc);
diff --git a/qmp-commands.hx b/qmp-commands.hx
index fbd98ee..71e4f0e 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -437,7 +437,9 @@ EQMP
\n\t\t\t -b for migration without shared storage with
full copy of disk\n\t\t\t -i for migration without
shared storage with incremental copy of disk
- (base image shared between src and destination),
+ (base image shared between src and destination)
+ \n\t\t\t put \kemari:\ in front of URI to enable
+ Fault Tolerance mode (Kemari protocol),
.user_print = monitor_user_noop,
.mhandler.cmd_new = do_migrate,
},
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 15/18] savevm: introduce qemu_savevm_trans_{begin,commit}.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Introduce qemu_savevm_trans_{begin,commit} to send the memory and
device info together, while avoiding cancelling memory state tracking.
This patch also abstracts common code between
qemu_savevm_state_{begin,iterate,commit}.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
savevm.c | 157 +++---
sysemu.h |2 +
2 files changed, 101 insertions(+), 58 deletions(-)
diff --git a/savevm.c b/savevm.c
index 5b57e94..dfbdc6c 100644
--- a/savevm.c
+++ b/savevm.c
@@ -1630,29 +1630,68 @@ bool qemu_savevm_state_blocked(Monitor *mon)
return false;
}
-int qemu_savevm_state_begin(Monitor *mon, QEMUFile *f, int blk_enable,
-int shared)
+/*
+ * section: header to write
+ * inc: if true, forces to pass SECTION_PART instead of SECTION_START
+ * pause: if true, breaks the loop when live handler returned 0
+ */
+static int qemu_savevm_state_live(Monitor *mon, QEMUFile *f, int section,
+ bool inc, bool pause)
{
SaveStateEntry *se;
+int skip = 0, ret;
QTAILQ_FOREACH(se, savevm_handlers, entry) {
-if(se-set_params == NULL) {
+int len, stage;
+
+if (se-save_live_state == NULL) {
continue;
- }
- se-set_params(blk_enable, shared, se-opaque);
+}
+
+/* Section type */
+qemu_put_byte(f, section);
+qemu_put_be32(f, se-section_id);
+
+if (section == QEMU_VM_SECTION_START) {
+/* ID string */
+len = strlen(se-idstr);
+qemu_put_byte(f, len);
+qemu_put_buffer(f, (uint8_t *)se-idstr, len);
+
+qemu_put_be32(f, se-instance_id);
+qemu_put_be32(f, se-version_id);
+
+stage = inc ? QEMU_VM_SECTION_PART : QEMU_VM_SECTION_START;
+} else {
+assert(inc);
+stage = section;
+}
+
+ret = se-save_live_state(mon, f, stage, se-opaque);
+if (!ret) {
+skip++;
+if (pause) {
+break;
+}
+}
}
-
-qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
-qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+
+return skip;
+}
+
+static void qemu_savevm_state_full(QEMUFile *f)
+{
+SaveStateEntry *se;
QTAILQ_FOREACH(se, savevm_handlers, entry) {
int len;
-if (se-save_live_state == NULL)
+if (se-save_state == NULL se-vmsd == NULL) {
continue;
+}
/* Section type */
-qemu_put_byte(f, QEMU_VM_SECTION_START);
+qemu_put_byte(f, QEMU_VM_SECTION_FULL);
qemu_put_be32(f, se-section_id);
/* ID string */
@@ -1663,9 +1702,29 @@ int qemu_savevm_state_begin(Monitor *mon, QEMUFile *f,
int blk_enable,
qemu_put_be32(f, se-instance_id);
qemu_put_be32(f, se-version_id);
-se-save_live_state(mon, f, QEMU_VM_SECTION_START, se-opaque);
+vmstate_save(f, se);
+}
+
+qemu_put_byte(f, QEMU_VM_EOF);
+}
+
+int qemu_savevm_state_begin(Monitor *mon, QEMUFile *f, int blk_enable,
+int shared)
+{
+SaveStateEntry *se;
+
+QTAILQ_FOREACH(se, savevm_handlers, entry) {
+if (se-set_params == NULL) {
+continue;
+}
+se-set_params(blk_enable, shared, se-opaque);
}
+qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+
+qemu_savevm_state_live(mon, f, QEMU_VM_SECTION_START, 0, 0);
+
if (qemu_file_has_error(f)) {
qemu_savevm_state_cancel(mon, f);
return -EIO;
@@ -1676,29 +1735,16 @@ int qemu_savevm_state_begin(Monitor *mon, QEMUFile *f,
int blk_enable,
int qemu_savevm_state_iterate(Monitor *mon, QEMUFile *f)
{
-SaveStateEntry *se;
int ret = 1;
-QTAILQ_FOREACH(se, savevm_handlers, entry) {
-if (se-save_live_state == NULL)
-continue;
-
-/* Section type */
-qemu_put_byte(f, QEMU_VM_SECTION_PART);
-qemu_put_be32(f, se-section_id);
-
-ret = se-save_live_state(mon, f, QEMU_VM_SECTION_PART, se-opaque);
-if (!ret) {
-/* Do not proceed to the next vmstate before this one reported
- completion of the current stage. This serializes the migration
- and reduces the probability that a faster changing state is
- synchronized over and over again. */
-break;
-}
-}
-
-if (ret)
+/* Do not proceed to the next vmstate before this one reported
+ completion of the current stage. This serializes the migration
+ and reduces the probability that a faster changing state is
+ synchronized over and over again. */
+ret = qemu_savevm_state_live(mon, f, QEMU_VM_SECTION_PART, 1, 1);
+if (!ret) {
return 1
[PATCH 10/18] Call init handler of event-tap at main() in vl.c.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp --- vl.c |3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/vl.c b/vl.c index a18792d..2dbda4d 100644 --- a/vl.c +++ b/vl.c @@ -160,6 +160,7 @@ int main(int argc, char **argv) #include qemu-queue.h #include cpus.h #include arch_init.h +#include event-tap.h #include ui/qemu-spice.h @@ -2974,6 +2975,8 @@ int main(int argc, char **argv, char **envp) blk_mig_init(); +event_tap_init(); + /* open the virtual block devices */ if (snapshot) qemu_opts_foreach(qemu_find_opts(drive), drive_enable_snapshot, NULL, 0); -- 1.7.0.2 -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 11/18] ioport: insert event_tap_ioport() to ioport_write().
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp Record ioport event to replay it upon failover. Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp --- ioport.c |2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/ioport.c b/ioport.c index 2e971fa..f485bab 100644 --- a/ioport.c +++ b/ioport.c @@ -27,6 +27,7 @@ #include ioport.h #include trace.h +#include event-tap.h /***/ /* IO Port */ @@ -76,6 +77,7 @@ static void ioport_write(int index, uint32_t address, uint32_t data) default_ioport_writel }; IOPortWriteFunc *func = ioport_write_table[index][address]; +event_tap_ioport(index, address, data); if (!func) func = default_func[index]; func(ioport_opaque[address], address, data); -- 1.7.0.2 -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 14/18] block: insert event-tap to bdrv_aio_writev(), bdrv_aio_flush() and bdrv_flush().
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
event-tap function is called only when it is on, and requests were
sent from device emulators.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Acked-by: Kevin Wolf kw...@redhat.com
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
block.c | 15 +++
1 files changed, 15 insertions(+), 0 deletions(-)
diff --git a/block.c b/block.c
index f731c7a..9e6b610 100644
--- a/block.c
+++ b/block.c
@@ -28,6 +28,7 @@
#include block_int.h
#include module.h
#include qemu-objects.h
+#include event-tap.h
#ifdef CONFIG_BSD
#include sys/types.h
@@ -1591,6 +1592,10 @@ int bdrv_flush(BlockDriverState *bs)
}
if (bs-drv bs-drv-bdrv_flush) {
+if (*bs-device_name event_tap_is_on()) {
+event_tap_bdrv_flush();
+}
+
return bs-drv-bdrv_flush(bs);
}
@@ -2226,6 +2231,11 @@ BlockDriverAIOCB *bdrv_aio_writev(BlockDriverState *bs,
int64_t sector_num,
if (bdrv_check_request(bs, sector_num, nb_sectors))
return NULL;
+if (*bs-device_name event_tap_is_on()) {
+return event_tap_bdrv_aio_writev(bs, sector_num, qiov, nb_sectors,
+ cb, opaque);
+}
+
if (bs-dirty_bitmap) {
blk_cb_data = blk_dirty_cb_alloc(bs, sector_num, nb_sectors, cb,
opaque);
@@ -2499,6 +2509,11 @@ BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
if (!drv)
return NULL;
+
+if (*bs-device_name event_tap_is_on()) {
+return event_tap_bdrv_aio_flush(bs, cb, opaque);
+}
+
return drv-bdrv_aio_flush(bs, cb, opaque);
}
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 09/18] Introduce event-tap.
event-tap controls when to start FT transaction, and provides proxy
functions to called from net/block devices. While FT transaction, it
queues up net/block requests, and flush them when the transaction gets
completed.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
---
Makefile.target |1 +
event-tap.c | 940 +++
event-tap.h | 44 +++
qemu-tool.c | 28 ++
trace-events| 10 +
5 files changed, 1023 insertions(+), 0 deletions(-)
create mode 100644 event-tap.c
create mode 100644 event-tap.h
diff --git a/Makefile.target b/Makefile.target
index 0e0ef36..e489df4 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -199,6 +199,7 @@ obj-y += rwhandler.o
obj-$(CONFIG_KVM) += kvm.o kvm-all.o
obj-$(CONFIG_NO_KVM) += kvm-stub.o
LIBS+=-lz
+obj-y += event-tap.o
QEMU_CFLAGS += $(VNC_TLS_CFLAGS)
QEMU_CFLAGS += $(VNC_SASL_CFLAGS)
diff --git a/event-tap.c b/event-tap.c
new file mode 100644
index 000..95c147a
--- /dev/null
+++ b/event-tap.c
@@ -0,0 +1,940 @@
+/*
+ * Event Tap functions for QEMU
+ *
+ * Copyright (c) 2010 Nippon Telegraph and Telephone Corporation.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ */
+
+#include qemu-common.h
+#include qemu-error.h
+#include block.h
+#include block_int.h
+#include ioport.h
+#include osdep.h
+#include sysemu.h
+#include hw/hw.h
+#include net.h
+#include event-tap.h
+#include trace.h
+
+enum EVENT_TAP_STATE {
+EVENT_TAP_OFF,
+EVENT_TAP_ON,
+EVENT_TAP_SUSPEND,
+EVENT_TAP_FLUSH,
+EVENT_TAP_LOAD,
+EVENT_TAP_REPLAY,
+};
+
+static enum EVENT_TAP_STATE event_tap_state = EVENT_TAP_OFF;
+
+typedef struct EventTapIOport {
+uint32_t address;
+uint32_t data;
+int index;
+} EventTapIOport;
+
+#define MMIO_BUF_SIZE 8
+
+typedef struct EventTapMMIO {
+uint64_t address;
+uint8_t buf[MMIO_BUF_SIZE];
+int len;
+} EventTapMMIO;
+
+typedef struct EventTapNetReq {
+char *device_name;
+int iovcnt;
+int vlan_id;
+bool vlan_needed;
+bool async;
+struct iovec *iov;
+NetPacketSent *sent_cb;
+} EventTapNetReq;
+
+#define MAX_BLOCK_REQUEST 32
+
+typedef struct EventTapAIOCB EventTapAIOCB;
+
+typedef struct EventTapBlkReq {
+char *device_name;
+int num_reqs;
+int num_cbs;
+bool is_flush;
+BlockRequest reqs[MAX_BLOCK_REQUEST];
+EventTapAIOCB *acb[MAX_BLOCK_REQUEST];
+} EventTapBlkReq;
+
+#define EVENT_TAP_IOPORT (1 0)
+#define EVENT_TAP_MMIO (1 1)
+#define EVENT_TAP_NET(1 2)
+#define EVENT_TAP_BLK(1 3)
+
+#define EVENT_TAP_TYPE_MASK (EVENT_TAP_NET - 1)
+
+typedef struct EventTapLog {
+int mode;
+union {
+EventTapIOport ioport;
+EventTapMMIO mmio;
+};
+union {
+EventTapNetReq net_req;
+EventTapBlkReq blk_req;
+};
+QTAILQ_ENTRY(EventTapLog) node;
+} EventTapLog;
+
+struct EventTapAIOCB {
+BlockDriverAIOCB common;
+BlockDriverAIOCB *acb;
+bool is_canceled;
+};
+
+static EventTapLog *last_event_tap;
+
+static QTAILQ_HEAD(, EventTapLog) event_list;
+static QTAILQ_HEAD(, EventTapLog) event_pool;
+
+static int (*event_tap_cb)(void);
+static QEMUBH *event_tap_bh;
+static VMChangeStateEntry *vmstate;
+
+static void event_tap_bh_cb(void *p)
+{
+if (event_tap_cb) {
+event_tap_cb();
+}
+
+qemu_bh_delete(event_tap_bh);
+event_tap_bh = NULL;
+}
+
+static void event_tap_schedule_bh(void)
+{
+trace_event_tap_ignore_bh(!!event_tap_bh);
+
+/* if bh is already set, we ignore it for now */
+if (event_tap_bh) {
+return;
+}
+
+event_tap_bh = qemu_bh_new(event_tap_bh_cb, NULL);
+qemu_bh_schedule(event_tap_bh);
+
+return;
+}
+
+static void *event_tap_alloc_log(void)
+{
+EventTapLog *log;
+
+if (QTAILQ_EMPTY(event_pool)) {
+log = qemu_mallocz(sizeof(EventTapLog));
+} else {
+log = QTAILQ_FIRST(event_pool);
+QTAILQ_REMOVE(event_pool, log, node);
+}
+
+return log;
+}
+
+static void event_tap_free_net_req(EventTapNetReq *net_req);
+static void event_tap_free_blk_req(EventTapBlkReq *blk_req);
+
+static void event_tap_free_log(EventTapLog *log)
+{
+int mode = log-mode ~EVENT_TAP_TYPE_MASK;
+
+if (mode == EVENT_TAP_NET) {
+event_tap_free_net_req(log-net_req);
+} else if (mode == EVENT_TAP_BLK) {
+event_tap_free_blk_req(log-blk_req);
+}
+
+log-mode = 0;
+
+/* return the log to event_pool */
+QTAILQ_INSERT_HEAD(event_pool, log, node);
+}
+
+static void event_tap_free_pool(void)
+{
+EventTapLog *log, *next;
+
+QTAILQ_FOREACH_SAFE(log, event_pool, node, next) {
+QTAILQ_REMOVE(event_pool, log, node);
+qemu_free(log);
+}
+}
+
+static void event_tap_free_net_req(EventTapNetReq *net_req)
+{
+int i;
+
+if (!net_req-async
[PATCH 12/18] Insert event_tap_mmio() to cpu_physical_memory_rw() in exec.c.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Record mmio write event to replay it upon failover.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
exec.c |4
1 files changed, 4 insertions(+), 0 deletions(-)
diff --git a/exec.c b/exec.c
index c3dc68a..3c3cece 100644
--- a/exec.c
+++ b/exec.c
@@ -33,6 +33,7 @@
#include osdep.h
#include kvm.h
#include qemu-timer.h
+#include event-tap.h
#if defined(CONFIG_USER_ONLY)
#include qemu.h
#include signal.h
@@ -3736,6 +3737,9 @@ void cpu_physical_memory_rw(target_phys_addr_t addr,
uint8_t *buf,
io_index = (pd IO_MEM_SHIFT) (IO_MEM_NB_ENTRIES - 1);
if (p)
addr1 = (addr ~TARGET_PAGE_MASK) + p-region_offset;
+
+event_tap_mmio(addr, buf, len);
+
/* XXX: could force cpu_single_env to NULL to avoid
potential bugs */
if (l = 4 ((addr1 3) == 0)) {
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 00/18] Kemari for KVM v0.2.14
and receiver side. The sender side respects the
max_downtime to decide when to switch from async to sync mode.
The repository contains all patches I'm sending with this message.
For those who want to try, please pull the following repository. It
also includes dirty bitmap optimization which aren't ready for posting
yet. To remove the dirty bitmap optimization, please look at HEAD~5
of the tree.
git://kemari.git.sourceforge.net/gitroot/kemari/kemari next
Thanks,
Kei
OHMURA Kei (2):
Introduce fault tolerant VM transaction QEMUFile and ft_mode.
Introduce event-tap.
Yoshiaki Tamura (16):
Make QEMUFile buf expandable, and introduce qemu_realloc_buffer() and
qemu_clear_buffer().
Introduce read() to FdMigrationState.
Introduce qemu_loadvm_state_no_header() and make qemu_loadvm_state()
a wrapper.
qemu-char: export socket_set_nodelay().
vl.c: add deleted flag for deleting the handler.
virtio: decrement last_avail_idx with inuse before saving.
savevm: introduce util functions to control ft_trans_file from savevm
layer.
Call init handler of event-tap at main() in vl.c.
ioport: insert event_tap_ioport() to ioport_write().
Insert event_tap_mmio() to cpu_physical_memory_rw() in exec.c.
net: insert event-tap to qemu_send_packet() and
qemu_sendv_packet_async().
block: insert event-tap to bdrv_aio_writev(), bdrv_aio_flush() and
bdrv_flush().
savevm: introduce qemu_savevm_trans_{begin,commit}.
migration: introduce migrate_ft_trans_{put,get}_ready(), and modify
migrate_fd_put_ready() when ft_mode is on.
migration-tcp: modify tcp_accept_incoming_migration() to handle
ft_mode, and add a hack not to close fd when ft_mode is enabled.
Introduce kemari: to enable FT migration mode (Kemari).
Makefile.objs |1 +
Makefile.target |1 +
block.c | 15 +
event-tap.c | 940 +++
event-tap.h | 44 +++
exec.c |4 +
ft_trans_file.c | 624
ft_trans_file.h | 72 +
hmp-commands.hx |4 +-
hw/hw.h |7 +
hw/virtio.c | 10 +-
ioport.c|2 +
migration-tcp.c | 83 +-
migration.c | 294 +-
migration.h |3 +
net.c |9 +
qemu-char.c |2 +-
qemu-tool.c | 28 ++
qemu_socket.h |1 +
qmp-commands.hx |4 +-
savevm.c| 372 +-
sysemu.h|2 +
trace-events| 25 ++
vl.c| 18 +-
24 files changed, 2477 insertions(+), 88 deletions(-)
create mode 100644 event-tap.c
create mode 100644 event-tap.h
create mode 100644 ft_trans_file.c
create mode 100644 ft_trans_file.h
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 07/18] Introduce fault tolerant VM transaction QEMUFile and ft_mode.
This code implements VM transaction protocol. Like buffered_file, it
sits between savevm and migration layer. With this architecture, VM
transaction protocol is implemented mostly independent from other
existing code.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
---
Makefile.objs |1 +
ft_trans_file.c | 624 +++
ft_trans_file.h | 72 +++
migration.c |3 +
trace-events| 15 ++
5 files changed, 715 insertions(+), 0 deletions(-)
create mode 100644 ft_trans_file.c
create mode 100644 ft_trans_file.h
diff --git a/Makefile.objs b/Makefile.objs
index 44ce368..75e7c79 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -101,6 +101,7 @@ common-obj-y += qdev.o qdev-properties.o
common-obj-y += block-migration.o iohandler.o
common-obj-y += pflib.o
common-obj-y += bitmap.o bitops.o
+common-obj-y += ft_trans_file.o
common-obj-$(CONFIG_BRLAPI) += baum.o
common-obj-$(CONFIG_POSIX) += migration-exec.o migration-unix.o migration-fd.o
diff --git a/ft_trans_file.c b/ft_trans_file.c
new file mode 100644
index 000..2b42b95
--- /dev/null
+++ b/ft_trans_file.c
@@ -0,0 +1,624 @@
+/*
+ * Fault tolerant VM transaction QEMUFile
+ *
+ * Copyright (c) 2010 Nippon Telegraph and Telephone Corporation.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2. See
+ * the COPYING file in the top-level directory.
+ *
+ * This source code is based on buffered_file.c.
+ * Copyright IBM, Corp. 2008
+ * Authors:
+ * Anthony Liguorialigu...@us.ibm.com
+ */
+
+#include qemu-common.h
+#include qemu-error.h
+#include hw/hw.h
+#include qemu-timer.h
+#include sysemu.h
+#include qemu-char.h
+#include trace.h
+#include ft_trans_file.h
+
+typedef struct FtTransHdr
+{
+uint16_t cmd;
+uint16_t id;
+uint32_t seq;
+uint32_t payload_len;
+} FtTransHdr;
+
+typedef struct QEMUFileFtTrans
+{
+FtTransPutBufferFunc *put_buffer;
+FtTransGetBufferFunc *get_buffer;
+FtTransPutReadyFunc *put_ready;
+FtTransGetReadyFunc *get_ready;
+FtTransWaitForUnfreezeFunc *wait_for_unfreeze;
+FtTransCloseFunc *close;
+void *opaque;
+QEMUFile *file;
+
+enum QEMU_VM_TRANSACTION_STATE state;
+uint32_t seq;
+uint16_t id;
+
+int has_error;
+
+bool freeze_output;
+bool freeze_input;
+bool rate_limit;
+bool is_sender;
+bool is_payload;
+
+uint8_t *buf;
+size_t buf_max_size;
+size_t put_offset;
+size_t get_offset;
+
+FtTransHdr header;
+size_t header_offset;
+} QEMUFileFtTrans;
+
+#define IO_BUF_SIZE 32768
+
+static void ft_trans_append(QEMUFileFtTrans *s,
+const uint8_t *buf, size_t size)
+{
+if (size (s-buf_max_size - s-put_offset)) {
+trace_ft_trans_realloc(s-buf_max_size, size + 1024);
+s-buf_max_size += size + 1024;
+s-buf = qemu_realloc(s-buf, s-buf_max_size);
+}
+
+trace_ft_trans_append(size);
+memcpy(s-buf + s-put_offset, buf, size);
+s-put_offset += size;
+}
+
+static void ft_trans_flush(QEMUFileFtTrans *s)
+{
+size_t offset = 0;
+
+if (s-has_error) {
+error_report(flush when error %d, bailing, s-has_error);
+return;
+}
+
+while (offset s-put_offset) {
+ssize_t ret;
+
+ret = s-put_buffer(s-opaque, s-buf + offset, s-put_offset -
offset);
+if (ret == -EAGAIN) {
+break;
+}
+
+if (ret = 0) {
+error_report(error flushing data, %s, strerror(errno));
+s-has_error = FT_TRANS_ERR_FLUSH;
+break;
+} else {
+offset += ret;
+}
+}
+
+trace_ft_trans_flush(offset, s-put_offset);
+memmove(s-buf, s-buf + offset, s-put_offset - offset);
+s-put_offset -= offset;
+s-freeze_output = !!s-put_offset;
+}
+
+static ssize_t ft_trans_put(void *opaque, void *buf, int size)
+{
+QEMUFileFtTrans *s = opaque;
+size_t offset = 0;
+ssize_t len;
+
+/* flush buffered data before putting next */
+if (s-put_offset) {
+ft_trans_flush(s);
+}
+
+while (!s-freeze_output offset size) {
+len = s-put_buffer(s-opaque, (uint8_t *)buf + offset, size - offset);
+
+if (len == -EAGAIN) {
+trace_ft_trans_freeze_output();
+s-freeze_output = 1;
+break;
+}
+
+if (len = 0) {
+error_report(putting data failed, %s, strerror(errno));
+s-has_error = 1;
+offset = -EINVAL;
+break;
+}
+
+offset += len;
+}
+
+if (s-freeze_output) {
+ft_trans_append(s, buf + offset, size - offset);
+offset = size;
+}
+
+return offset;
+}
+
+static int ft_trans_send_header(QEMUFileFtTrans *s,
+enum QEMU_VM_TRANSACTION_STATE state,
+uint32_t payload_len)
+{
+int ret
[PATCH 17/18] migration-tcp: modify tcp_accept_incoming_migration() to handle ft_mode, and add a hack not to close fd when ft_mode is enabled.
From: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
When ft_mode is set in the header, tcp_accept_incoming_migration()
sets ft_trans_incoming() as a callback, and call
qemu_file_get_notify() to receive FT transaction iteratively. We also
need a hack no to close fd before moving to ft_transaction mode, so
that we can reuse the fd for it. vm_change_state_handler is added to
turn off ft_mode when cont is pressed.
Signed-off-by: Yoshiaki Tamura tamura.yoshi...@lab.ntt.co.jp
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
migration-tcp.c | 68 ++-
1 files changed, 67 insertions(+), 1 deletions(-)
diff --git a/migration-tcp.c b/migration-tcp.c
index bb67d53..1eeac2b 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -17,6 +17,9 @@
#include qemu-char.h
#include buffered_file.h
#include block.h
+#include sysemu.h
+#include ft_trans_file.h
+#include event-tap.h
//#define DEBUG_MIGRATION_TCP
@@ -28,6 +31,8 @@
do { } while (0)
#endif
+static VMChangeStateEntry *vmstate;
+
static int socket_errno(FdMigrationState *s)
{
return socket_error();
@@ -55,7 +60,8 @@ static int socket_read(FdMigrationState *s, const void * buf,
size_t size)
static int tcp_close(FdMigrationState *s)
{
DPRINTF(tcp_close\n);
-if (s-fd != -1) {
+/* FIX ME: accessing ft_mode here isn't clean */
+if (s-fd != -1 ft_mode != FT_INIT) {
close(s-fd);
s-fd = -1;
}
@@ -149,6 +155,36 @@ MigrationState *tcp_start_outgoing_migration(Monitor *mon,
return s-mig_state;
}
+static void ft_trans_incoming(void *opaque)
+{
+QEMUFile *f = opaque;
+
+qemu_file_get_notify(f);
+if (qemu_file_has_error(f)) {
+ft_mode = FT_ERROR;
+qemu_fclose(f);
+}
+}
+
+static void ft_trans_reset(void *opaque, int running, int reason)
+{
+QEMUFile *f = opaque;
+
+if (running) {
+if (ft_mode != FT_ERROR) {
+qemu_fclose(f);
+}
+ft_mode = FT_OFF;
+qemu_del_vm_change_state_handler(vmstate);
+}
+}
+
+static void ft_trans_schedule_replay(QEMUFile *f)
+{
+event_tap_schedule_replay();
+vmstate = qemu_add_vm_change_state_handler(ft_trans_reset, f);
+}
+
static void tcp_accept_incoming_migration(void *opaque)
{
struct sockaddr_in addr;
@@ -174,8 +210,38 @@ static void tcp_accept_incoming_migration(void *opaque)
goto out;
}
+if (ft_mode == FT_INIT) {
+autostart = 0;
+}
+
process_incoming_migration(f);
+
+if (ft_mode == FT_INIT) {
+int ret;
+
+socket_set_nodelay(c);
+
+f = qemu_fopen_ft_trans(s, c);
+if (f == NULL) {
+fprintf(stderr, could not qemu_fopen_ft_trans\n);
+goto out;
+}
+
+/* need to wait sender to setup */
+ret = qemu_ft_trans_begin(f);
+if (ret 0) {
+goto out;
+}
+
+qemu_set_fd_handler2(c, NULL, ft_trans_incoming, NULL, f);
+ft_trans_schedule_replay(f);
+ft_mode = FT_TRANSACTION_RECV;
+
+return;
+}
+
qemu_fclose(f);
+
out:
close(c);
out2:
--
1.7.0.2
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: virtio_blk_load() question
Thanks for your reply. When I ported virtio to vmstate, I was unable to get that list not empty for more than I tried. It should be not empty in the case of one error or similar, but I was not able to reproduce it. Actually, I wasn't able to get that condition either. We're having problem in loading continuously sent VM image, and were looking deeper into the device models. We were doubting the virtio_blk_load() first, but seems to be different. I agree this change is ok/needed. Notice that my series ( [PATCH 0/9] Virtio cleanups) that changes it to a QLIST and fixes it. I guess you're mentioning the following patch, and it's good to know that. http://www.mail-archive.com/qemu-de...@nongnu.org/msg27324.html However, although QLIST_INSERT_HEAD is used, virtio_blk_save() is adding requests to the tail of the list, and if we need to keep the order of outstading requests, shouldn't we put incoming requests to the tail in virtio_blk_load()? -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: virtio_blk_load() question
On 2010/03/18 21:07, Juan Quintela wrote:
Really, ordering doesn't matter (in this case):
see virtio-blk.c:virtio_blk_dma_restart_bh()
QLIST_FOREACH_SAFE(req, rq_copy, next, next_req) {
QLIST_REMOVE(req, next);
virtio_blk_handle_request(req, mrb);
}
This mean that we are just removing from the beggining and addin from
the beginnig (i.e. reversing). Adding by the beggining made it easier,
but I can change if you mean.
Thanks, I understood.
However, since it's difficult to understand this at first glance,
it would be great if you could modify, but I'm OK if you could just add
comments on this to the exiting patch. It's kind for a newbie like me.
Notice that except if there are any errors (I was not able to trigger
it, but didnt' try too hard), that list is going to be syncked in the
qemu_aio_flush();
bdrv_flush_all();
in migrate_fd_put_ready(), so it is not trivial to hit it and probably
the difference is just theoretical.
Thank you for your information.
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [Qemu-devel] Re: [PATCH v2] qemu-kvm: Speed up of the dirty-bitmap-traveling
We think? I mean - yes, I think so too. But have you actually measured it? How much improvement are we talking here? Is it still faster when a bswap is involved? Thanks for pointing out. I will post the data for x86 later. However, I don't have a test environment to check the impact of bswap. Would you please measure the run time between the following section if possible? It'd make more sense to have a real stand alone test program, no? I can try to write one today, but I have some really nasty important bugs to fix first. OK. I will prepare a test code with sample data. Since I found a ppc machine around, I will run the code and post the results of x86 and ppc. By the way, the following data is a result of x86 measured in QEMU/KVM. This data shows, how many times the function is called (#called), runtime of original function(orig.), runtime of this patch(patch), speedup ratio (ratio). Test1: Guest OS read 3GB file, which is bigger than memory. #called orig.(msec) patch(msec) ratio 108 1.1 0.1 7.6 102 1.0 0.1 6.8 132 1.6 0.2 7.1 Test2: Guest OS read/write 3GB file, which is bigger than memory. #called orig.(msec) patch(msec) ratio 239433 7.7 4.3 210029 7.1 4.1 283240 9.9 4.0 -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [Qemu-devel] Re: [PATCH v2] qemu-kvm: Speed up of the dirty-bitmap-traveling
We think? I mean - yes, I think so too. But have you actually measured it? How much improvement are we talking here? Is it still faster when a bswap is involved? Thanks for pointing out. I will post the data for x86 later. However, I don't have a test environment to check the impact of bswap. Would you please measure the run time between the following section if possible? It'd make more sense to have a real stand alone test program, no? I can try to write one today, but I have some really nasty important bugs to fix first. OK. I will prepare a test code with sample data. Since I found a ppc machine around, I will run the code and post the results of x86 and ppc. By the way, the following data is a result of x86 measured in QEMU/KVM. This data shows, how many times the function is called (#called), runtime of original function(orig.), runtime of this patch(patch), speedup ratio (ratio). That does indeed look promising! Thanks for doing this micro-benchmark. I just want to be 100% sure that it doesn't affect performance for big endian badly. I measured runtime of the test code with sample data. My test environment and results are described below. x86 Test Environment: CPU: 4x Intel Xeon Quad Core 2.66GHz Mem size: 6GB ppc Test Environment: CPU: 2x Dual Core PPC970MP Mem size: 2GB The sample data of dirty bitmap was produced by QEMU/KVM while the guest OS was live migrating. To measure the runtime I copied cpu_get_real_ticks() of QEMU to my test program. Experimental results: Test1: Guest OS read 3GB file, which is bigger than memory. orig.(msec)patch(msec)ratio x860.30.16.4 ppc7.92.73.0 Test2: Guest OS read/write 3GB file, which is bigger than memory. orig.(msec)patch(msec)ratio x8612.0 3.23.7 ppc251.1 1232.0 I also measured the runtime of bswap itself on ppc, and I found it was only just 0.3% ~ 0.7 % of the runtime described above. -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [Qemu-devel] Re: [PATCH v2] qemu-kvm: Speed up of the dirty-bitmap-traveling
We think? I mean - yes, I think so too. But have you actually measured it?
How much improvement are we talking here?
Is it still faster when a bswap is involved?
Thanks for pointing out.
I will post the data for x86 later.
However, I don't have a test environment to check the impact of bswap.
Would you please measure the run time between the following section if possible?
start -
qemu-kvm.c:
static int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
void *bitmap, void *opaque)
{
/* warm up each function */
kvm_get_dirty_pages_log_range(start, bitmap, start, len);
kvm_get_dirty_pages_log_range_new(start, bitmap, start, len);
/* measurement */
int64_t t1, t2;
t1 = cpu_get_real_ticks();
kvm_get_dirty_pages_log_range(start, bitmap, start, len);
t1 = cpu_get_real_ticks() - t1;
t2 = cpu_get_real_ticks();
kvm_get_dirty_pages_log_range_new(start, bitmap, start, len);
t2 = cpu_get_real_ticks() - t2;
printf(## %zd, %zd\n, t1, t2); fflush(stdout);
return kvm_get_dirty_pages_log_range_new(start, bitmap, start, len);
}
end -
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v2] qemu-kvm: Speed up of the dirty-bitmap-traveling
dirty-bitmap-traveling is carried out by byte size in qemu-kvm.c.
But We think that dirty-bitmap-traveling by long size is faster than by byte
size especially when most of memory is not dirty.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
bswap.h|2 ++
qemu-kvm.c | 31 ---
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/bswap.h b/bswap.h
index 4558704..1f87e6d 100644
--- a/bswap.h
+++ b/bswap.h
@@ -205,8 +205,10 @@ static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
#ifdef HOST_WORDS_BIGENDIAN
#define cpu_to_32wu cpu_to_be32wu
+#define leul_to_cpu(v) le ## HOST_LONG_BITS ## _to_cpu(v)
#else
#define cpu_to_32wu cpu_to_le32wu
+#define leul_to_cpu(v) (v)
#endif
#undef le_bswap
diff --git a/qemu-kvm.c b/qemu-kvm.c
index a305907..6952aa5 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -2434,31 +2434,32 @@ int kvm_physical_memory_set_dirty_tracking(int enable)
/* get kvm's dirty pages bitmap and update qemu's */
static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
- unsigned char *bitmap,
+ unsigned long *bitmap,
unsigned long offset,
unsigned long mem_size)
{
-unsigned int i, j, n = 0;
-unsigned char c;
-unsigned long page_number, addr, addr1;
+unsigned int i, j;
+unsigned long page_number, addr, addr1, c;
ram_addr_t ram_addr;
-unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
+HOST_LONG_BITS;
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
* especially when most of the memory is not dirty.
*/
for (i = 0; i len; i++) {
-c = bitmap[i];
-while (c 0) {
-j = ffsl(c) - 1;
-c = ~(1u j);
-page_number = i * 8 + j;
-addr1 = page_number * TARGET_PAGE_SIZE;
-addr = offset + addr1;
-ram_addr = cpu_get_physical_page_desc(addr);
-cpu_physical_memory_set_dirty(ram_addr);
-n++;
+if (bitmap[i] != 0) {
+c = leul_to_cpu(bitmap[i]);
+do {
+j = ffsl(c) - 1;
+c = ~(1ul j);
+page_number = i * HOST_LONG_BITS + j;
+addr1 = page_number * TARGET_PAGE_SIZE;
+addr = offset + addr1;
+ram_addr = cpu_get_physical_page_desc(addr);
+cpu_physical_memory_set_dirty(ram_addr);
+} while (c != 0);
}
}
return 0;
--
1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH v2] qemu-kvm: Speed up of the dirty-bitmap-traveling
On 02/11/2010 Anthony Liguori anth...@codemonkey.ws wrote:
Oh, I see what's happening here. Yes, I think a leul_to_cpu() makes more
sense.
Maybe I'm missing something here.
I couldn't find leul_to_cpu(), so have defined it in bswap.h.
Correct?
--- a/bswap.h
+++ b/bswap.h
@@ -205,8 +205,10 @@ static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
#ifdef HOST_WORDS_BIGENDIAN
#define cpu_to_32wu cpu_to_be32wu
+#define leul_to_cpu(v) le ## HOST_LONG_BITS ## _to_cpu(v)
#else
#define cpu_to_32wu cpu_to_le32wu
+#define leul_to_cpu(v) (v)
#endif
On 02/10/2010 Ulrich Drepper drep...@redhat.com wrote:
If you're optimizing this code you might want to do it all. The
compiler might not see through the bswap call and create unnecessary
data dependencies. Especially problematic if the bitmap is really
sparse. Also, the outer test is != while the inner test is . Be
consistent. I suggest to replace the inner loop with
do {
...
} while (c != 0);
Depending on how sparse the bitmap is populated this might reduce the
number of data dependencies quite a bit.
Combining all comments, the code would be like this.
if (bitmap_ul[i] != 0) {
c = leul_to_cpu(bitmap_ul[i]);
do {
j = ffsl(c) - 1;
c = ~(1ul j);
page_number = i * HOST_LONG_BITS + j;
addr1 = page_number * TARGET_PAGE_SIZE;
addr = offset + addr1;
ram_addr = cpu_get_physical_page_desc(addr);
cpu_physical_memory_set_dirty(ram_addr);
} while (c != 0);
}
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH 1/3] qemu-kvm: Wrap phys_ram_dirty with additional inline functions.
Why do you need a counter? It may be sufficient to set a single bit. This reduces the memory overhead and perhaps cache thrashing. Thanks for looking into this. I agree with your opinion. Our motivation here is to skip traveling when the dirty bitmap is really sparse or dense, so either setting a bit or counting up would be fine. There is one advantage to the counter approach that we can make this large traveling granularity flexible. In case of the bit approach, the maximum granularity is limited to HOST_LONG_BITS. If you think this flexibility is to be useless, we would take the bit approach. By the way, this is about filling the gap of the dirty bitmap management between kvm and qemu. Do you think we should set a bit when qemu's phys_ram_dirty is 0xff or !0? Radically, if we could have a bit-based phys_ram_dirty_by_word, we may just OR the dirty bitmap of kvm with qemu in kvm_get_dirty_pages_log_range()... -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] qemu-kvm: Speed up of the dirty-bitmap-traveling
Please reuse the changelog when reposing a patch, this makes it easier
for me to apply it.
Thanks. Will follow it from next time.
Should be a host long size, not guest. This will fail when running a
32-bit qemu-system-x86_64 binary.
Sorry. That was our mistake.
Instead of using a nested loop if bitmap_ul[i] != 0, it is possible to
use just a single loop (while (c 0)), and process a long's worth of data.
The only trickery is with big endian hosts, where the conversion from
bit number to page number is a bit complicated.
To convert the bitmap from big endian to little endian, le_bswap macro in
bswap.h seems useful, which is now undefined. What do you think about this
approach?
This is an example bitmap-traveling code using le_bswap:
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
* especially when most of the memory is not dirty.
*/
for (i = 0; i len; i++) {
if (bitmap_ul[i] != 0) {
c = le_bswap(bitmap_ul[i], HOST_LONG_BITS);
while (c 0) {
j = ffsl(c) - 1;
c = ~(1ul j);
page_number = i * HOST_LONG_BITS + j;
addr1 = page_number * TARGET_PAGE_SIZE;
addr = offset + addr1;
ram_addr = cpu_get_physical_page_desc(addr);
cpu_physical_memory_set_dirty(ram_addr);
}
}
}
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH v2] qemu-kvm: Speed up of the dirty-bitmap-traveling
dirty-bitmap-traveling is carried out by byte size in qemu-kvm.c.
But We think that dirty-bitmap-traveling by long size is faster than by byte
size especially when most of memory is not dirty.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
bswap.h|1 -
qemu-kvm.c | 30 --
2 files changed, 16 insertions(+), 15 deletions(-)
diff --git a/bswap.h b/bswap.h
index 4558704..d896f01 100644
--- a/bswap.h
+++ b/bswap.h
@@ -209,7 +209,6 @@ static inline void cpu_to_be32wu(uint32_t *p, uint32_t v)
#define cpu_to_32wu cpu_to_le32wu
#endif
-#undef le_bswap
#undef be_bswap
#undef le_bswaps
#undef be_bswaps
diff --git a/qemu-kvm.c b/qemu-kvm.c
index a305907..ea07912 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -2438,27 +2438,29 @@ static int kvm_get_dirty_pages_log_range(unsigned long
start_addr,
unsigned long offset,
unsigned long mem_size)
{
-unsigned int i, j, n = 0;
-unsigned char c;
-unsigned long page_number, addr, addr1;
+unsigned int i, j;
+unsigned long page_number, addr, addr1, c;
ram_addr_t ram_addr;
-unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) /
+HOST_LONG_BITS;
+unsigned long *bitmap_ul = (unsigned long *)bitmap;
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
* especially when most of the memory is not dirty.
*/
for (i = 0; i len; i++) {
-c = bitmap[i];
-while (c 0) {
-j = ffsl(c) - 1;
-c = ~(1u j);
-page_number = i * 8 + j;
-addr1 = page_number * TARGET_PAGE_SIZE;
-addr = offset + addr1;
-ram_addr = cpu_get_physical_page_desc(addr);
-cpu_physical_memory_set_dirty(ram_addr);
-n++;
+if (bitmap_ul[i] != 0) {
+c = le_bswap(bitmap_ul[i], HOST_LONG_BITS);
+while (c 0) {
+j = ffsl(c) - 1;
+c = ~(1ul j);
+page_number = i * HOST_LONG_BITS + j;
+addr1 = page_number * TARGET_PAGE_SIZE;
+addr = offset + addr1;
+ram_addr = cpu_get_physical_page_desc(addr);
+cpu_physical_memory_set_dirty(ram_addr);
+}
}
}
return 0;
--
1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] qemu-kvm: Speed up of the dirty-bitmap-traveling
Thank you for your comments. We have implemented the code which applied your
comments. This is patch for qemu-kvm.c.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
qemu-kvm.c | 31 +++
1 files changed, 19 insertions(+), 12 deletions(-)
diff --git a/qemu-kvm.c b/qemu-kvm.c
index a305907..d7474ea 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -2438,27 +2438,34 @@ static int kvm_get_dirty_pages_log_range(unsigned long
start_addr,
unsigned long offset,
unsigned long mem_size)
{
-unsigned int i, j, n = 0;
+unsigned int i, j, k, start, end;
unsigned char c;
unsigned long page_number, addr, addr1;
ram_addr_t ram_addr;
-unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + TARGET_LONG_BITS - 1) /
+TARGET_LONG_BITS;
+unsigned long *bitmap_ul = (unsigned long *)bitmap;
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
* especially when most of the memory is not dirty.
*/
for (i = 0; i len; i++) {
-c = bitmap[i];
-while (c 0) {
-j = ffsl(c) - 1;
-c = ~(1u j);
-page_number = i * 8 + j;
-addr1 = page_number * TARGET_PAGE_SIZE;
-addr = offset + addr1;
-ram_addr = cpu_get_physical_page_desc(addr);
-cpu_physical_memory_set_dirty(ram_addr);
-n++;
+if (bitmap_ul[i] != 0) {
+start = i * TARGET_LONG_SIZE;
+end = (i + 1) * TARGET_LONG_SIZE;
+for (j = start; j end; j++) {
+c = bitmap[j];
+while (c 0) {
+k = ffsl(c) - 1;
+c = ~(1u k);
+page_number = j * 8 + k;
+addr1 = page_number * TARGET_PAGE_SIZE;
+addr = offset + addr1;
+ram_addr = cpu_get_physical_page_desc(addr);
+cpu_physical_memory_set_dirty(ram_addr);
+}
+}
}
}
return 0;
--
1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] qemu-kvm: Speed up of the dirty-bitmap-traveling
Thank you for your comments. We have implemented the code which applied your
comments. This is patch for upstream.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
kvm-all.c | 54 +++---
1 files changed, 39 insertions(+), 15 deletions(-)
diff --git a/kvm-all.c b/kvm-all.c
index 6c0fd37..603307c 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -287,9 +287,43 @@ int kvm_set_migration_log(int enable)
return 0;
}
-static int test_le_bit(unsigned long nr, unsigned char *addr)
+/* get kvm's dirty pages bitmap and update qemu's */
+static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
+ unsigned char *bitmap,
+ unsigned long offset,
+ unsigned long mem_size)
{
-return (addr[nr 3] (nr 7)) 1;
+unsigned int i, j, k, start, end;
+unsigned char c;
+unsigned long page_number, addr, addr1;
+ram_addr_t ram_addr;
+unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + TARGET_LONG_BITS - 1) /
+TARGET_LONG_BITS;
+unsigned long *bitmap_ul = (unsigned long *)bitmap;
+
+/*
+ * bitmap-traveling is faster than memory-traveling (for addr...)
+ * especially when most of the memory is not dirty.
+ */
+for (i = 0; i len; i++) {
+if (bitmap_ul[i] != 0) {
+start = i * TARGET_LONG_SIZE;
+end = (i + 1) * TARGET_LONG_SIZE;
+for (j = start; j end; j++) {
+c = bitmap[j];
+while (c 0) {
+k = ffsl(c) - 1;
+c = ~(1u k);
+page_number = j * 8 + k;
+addr1 = page_number * TARGET_PAGE_SIZE;
+addr = offset + addr1;
+ram_addr = cpu_get_physical_page_desc(addr);
+cpu_physical_memory_set_dirty(ram_addr);
+}
+}
+}
+}
+return 0;
}
/**
@@ -305,8 +339,6 @@ int kvm_physical_sync_dirty_bitmap(target_phys_addr_t
start_addr,
{
KVMState *s = kvm_state;
unsigned long size, allocated_size = 0;
-target_phys_addr_t phys_addr;
-ram_addr_t addr;
KVMDirtyLog d;
KVMSlot *mem;
int ret = 0;
@@ -335,17 +367,9 @@ int kvm_physical_sync_dirty_bitmap(target_phys_addr_t
start_addr,
break;
}
-for (phys_addr = mem-start_addr, addr = mem-phys_offset;
- phys_addr mem-start_addr + mem-memory_size;
- phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
-unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
-unsigned nr = (phys_addr - mem-start_addr) TARGET_PAGE_BITS;
-
-if (test_le_bit(nr, bitmap)) {
-cpu_physical_memory_set_dirty(addr);
-}
-}
-start_addr = phys_addr;
+kvm_get_dirty_pages_log_range(mem-start_addr, d.dirty_bitmap,
+ mem-start_addr, mem-memory_size);
+start_addr = mem-start_addr + mem-memory_size;
}
qemu_free(d.dirty_bitmap);
--
1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 3/3] qemu-kvm: Change the methods of get dirty pages.
Get number of the dirty and non-dirty pages using
cpu_physical_memory_get_{dirty|non_dirty}_range().
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
vl.c | 57 ++---
1 files changed, 38 insertions(+), 19 deletions(-)
diff --git a/vl.c b/vl.c
index 4ef6a78..0835da6 100644
--- a/vl.c
+++ b/vl.c
@@ -2798,7 +2798,7 @@ static int ram_save_block(QEMUFile *f)
static ram_addr_t current_addr = 0;
ram_addr_t saved_addr = current_addr;
ram_addr_t addr = 0;
-int found = 0;
+int i, found = 0, skip = 0;
while (addr last_ram_offset) {
if (kvm_enabled() current_addr == 0) {
@@ -2810,28 +2810,37 @@ static int ram_save_block(QEMUFile *f)
return 0;
}
}
-if (cpu_physical_memory_get_dirty(current_addr, MIGRATION_DIRTY_FLAG))
{
+if ((found = cpu_physical_memory_get_dirty_range(current_addr,
+last_ram_offset, MIGRATION_DIRTY_FLAG))) {
uint8_t *p;
-cpu_physical_memory_reset_dirty(current_addr,
-current_addr + TARGET_PAGE_SIZE,
-MIGRATION_DIRTY_FLAG);
+for (i = 0; i found; i++) {
+ram_addr_t page_addr = current_addr + (i * TARGET_PAGE_SIZE);
+cpu_physical_memory_reset_dirty(page_addr,
+page_addr + TARGET_PAGE_SIZE,
+MIGRATION_DIRTY_FLAG);
-p = qemu_get_ram_ptr(current_addr);
+p = qemu_get_ram_ptr(page_addr);
-if (is_dup_page(p, *p)) {
-qemu_put_be64(f, current_addr | RAM_SAVE_FLAG_COMPRESS);
-qemu_put_byte(f, *p);
-} else {
-qemu_put_be64(f, current_addr | RAM_SAVE_FLAG_PAGE);
-qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
+if (is_dup_page(p, *p)) {
+qemu_put_be64(f, (page_addr) |
+ RAM_SAVE_FLAG_COMPRESS);
+qemu_put_byte(f, *p);
+} else {
+qemu_put_be64(f, (page_addr) |
+ RAM_SAVE_FLAG_PAGE);
+qemu_put_buffer(f, p, TARGET_PAGE_SIZE);
+}
}
-found = 1;
break;
}
-addr += TARGET_PAGE_SIZE;
-current_addr = (saved_addr + addr) % last_ram_offset;
+while ((skip = cpu_physical_memory_get_non_dirty_range(current_addr,
+last_ram_offset, MIGRATION_DIRTY_FLAG)) != 0) {
+addr += TARGET_PAGE_SIZE * skip;
+current_addr = (saved_addr + addr) % last_ram_offset;
+if (addr = last_ram_offset) break;
+}
}
return found;
@@ -2841,12 +2850,22 @@ static uint64_t bytes_transferred;
static ram_addr_t ram_save_remaining(void)
{
-ram_addr_t addr;
+ram_addr_t addr = 0;
ram_addr_t count = 0;
+int found = 0, skip = 0;
-for (addr = 0; addr last_ram_offset; addr += TARGET_PAGE_SIZE) {
-if (cpu_physical_memory_get_dirty(addr, MIGRATION_DIRTY_FLAG))
-count++;
+while (addr last_ram_offset) {
+if ((found = cpu_physical_memory_get_dirty_range(addr,
+last_ram_offset, MIGRATION_DIRTY_FLAG))) {
+count += found;
+addr += TARGET_PAGE_SIZE * found;
+} else {
+while ((skip = cpu_physical_memory_get_non_dirty_range(addr,
+last_ram_offset, MIGRATION_DIRTY_FLAG)) != 0) {
+addr += TARGET_PAGE_SIZE * skip;
+if (addr = last_ram_offset) break;
+}
+}
}
return count;
--
1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 2/3] Check continuous dirty and non-dirty pages.
We will check the dirty and non-dirty pages as follows:
1. Checked by 64 pages block.
Check pages using phys_ram_dirty_by_word. Count up/down it when
phys_ram_dirty when is 0xff or not.
2. Checked by TARGET_LONG_SIZE pages block.
3. Checked by one page.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
exec.c | 125 +++-
1 files changed, 100 insertions(+), 25 deletions(-)
diff --git a/exec.c b/exec.c
index ade09cb..5770281 100644
--- a/exec.c
+++ b/exec.c
@@ -119,6 +119,7 @@ uint8_t *code_gen_ptr;
#if !defined(CONFIG_USER_ONLY)
int phys_ram_fd;
uint8_t *phys_ram_dirty;
+int *phys_ram_dirty_by_word;
uint8_t *bios_mem;
static int in_migration;
@@ -1843,7 +1844,7 @@ static void tlb_protect_code(ram_addr_t ram_addr)
static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
target_ulong vaddr)
{
-phys_ram_dirty[ram_addr TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
+cpu_physical_memory_set_dirty_flags(ram_addr, CODE_DIRTY_FLAG);
}
static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
@@ -1858,14 +1859,83 @@ static inline void tlb_reset_dirty_range(CPUTLBEntry
*tlb_entry,
}
}
+int cpu_physical_memory_get_dirty_range(ram_addr_t start, ram_addr_t end,
+int dirty_flags)
+{
+static unsigned long mask = 0;
+static int cached_dirty_flags = 0;
+uint8_t *p = phys_ram_dirty + (start TARGET_PAGE_BITS);
+int *p_by_word = phys_ram_dirty_by_word +
+(start TARGET_PAGE_BITS) / TARGET_LONG_BITS;
+
+/*
+ * Since making the mask in every time this function called is too slow,
+ * we will cache the value.
+ */
+if (dirty_flags != cached_dirty_flags) {
+cached_dirty_flags = dirty_flags;
+mask = unroll_flags_to_ul(dirty_flags);
+}
+
+/*
+ * We can get the dirty-pages very fast,
+ * when a lot of continuous pages are dirty.
+ */
+if start TARGET_PAGE_BITS) (TARGET_LONG_BITS - 1)) == 0)
+((end - start) TARGET_PAGE_BITS) = TARGET_LONG_BITS
+*p_by_word == TARGET_LONG_BITS)
+return TARGET_LONG_BITS;
+
+if start TARGET_PAGE_BITS) (TARGET_LONG_SIZE - 1)) == 0)
+((end - start) TARGET_PAGE_BITS) = TARGET_LONG_SIZE
+(*(unsigned long *)p mask) == mask)
+return TARGET_LONG_SIZE;
+
+return (cpu_physical_memory_get_dirty(start, dirty_flags) == dirty_flags);
+}
+
+int cpu_physical_memory_get_non_dirty_range(ram_addr_t start, ram_addr_t end,
+int dirty_flags)
+{
+static unsigned long mask = 0;
+static int cached_dirty_flags = 0;
+uint8_t *p = phys_ram_dirty + (start TARGET_PAGE_BITS);
+int *p_by_word = phys_ram_dirty_by_word +
+(start TARGET_PAGE_BITS) / TARGET_LONG_BITS;
+
+/*
+ * Since making the mask in every time this function called is too slow,
+ * we will cache the value.
+ */
+if (dirty_flags != cached_dirty_flags) {
+cached_dirty_flags = dirty_flags;
+mask = unroll_flags_to_ul(dirty_flags);
+}
+
+/*
+ * We can skip the non-dirty-pages very fast,
+ * when a lot of continuous pages are not dirty.
+ */
+if start TARGET_PAGE_BITS) (TARGET_LONG_BITS - 1)) == 0)
+((end - start) TARGET_PAGE_BITS) = TARGET_LONG_BITS
+*p_by_word == 0)
+return TARGET_LONG_BITS;
+
+if start TARGET_PAGE_BITS) (TARGET_LONG_SIZE - 1)) == 0)
+((end - start) TARGET_PAGE_BITS) = TARGET_LONG_SIZE
+(*(unsigned long *)p mask) == 0)
+return TARGET_LONG_SIZE;
+
+return (cpu_physical_memory_get_dirty(start, dirty_flags) == 0);
+}
+
/* Note: start and end must be within the same ram block. */
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
int dirty_flags)
{
CPUState *env;
unsigned long length, start1;
-int i, mask, len;
-uint8_t *p;
+int i;
start = TARGET_PAGE_MASK;
end = TARGET_PAGE_ALIGN(end);
@@ -1873,11 +1943,7 @@ void cpu_physical_memory_reset_dirty(ram_addr_t start,
ram_addr_t end,
length = end - start;
if (length == 0)
return;
-len = length TARGET_PAGE_BITS;
-mask = ~dirty_flags;
-p = phys_ram_dirty + (start TARGET_PAGE_BITS);
-for(i = 0; i len; i++)
-p[i] = mask;
+cpu_physical_memory_mask_dirty_range(start, length, dirty_flags);
/* we modify the TLB cache so that the dirty bit will be set again
when accessing the range */
@@ -2535,6 +2601,7 @@ extern const char *mem_path;
ram_addr_t qemu_ram_alloc(ram_addr_t size)
{
RAMBlock *new_block;
+int i, *p;
size = TARGET_PAGE_ALIGN(size);
new_block = qemu_malloc(sizeof(*new_block));
@@ -2564,6 +2631,14 @@ ram_addr_t qemu_ram_alloc(ram_addr_t size)
memset
[PATCH 0/3] qemu-kvm: Check the dirty and non-dirty pages by multiple size
The dirty and non-dirty pages are checked one by one in vl.c. Since we believe that most of the memory is not dirty, checking the dirty and non-dirty pages by multiple page size should be much faster than checking them one by one. We think there is mostly two kind of situation. One is almost all the page is clean because there should be small amount of pages become dirty between each round of dirty bitmap check. The other is all the pages is dirty because all the bitmap is marked as dirty at the beginning of migration. To prove our prospect, we have evaluated effect of this patch. we compared runtime of ram_save_remaining with original ram_save_remaining() and ram_save_remaining() using functions of this patch. Test Environment: CPU: 4x Intel Xeon Quad Core 2.66GHz Mem size: 6GB kvm version: 2.6.31-17-server qemu version: commit ed880109f74f0a4dd5b7ec09e6a2d9ba4903d9a5 Host OS: Ubuntu 9.10 (kernel 2.6.31) Guest OS: Debian/GNU Linux lenny (kernel 2.6.26) Guest Mem size: 512MB Conditions of experiments are as follows: Cond1: Guest OS periodically makes the 256MB continuous dirty pages. Cond2: Guest OS periodically makes the 256MB dirty pages and non-dirty pages in turn. Cond3: Guest OS read 3GB file, which is bigger than memory. Cond4: Guest OS read/write 3GB file, which is bigger than memory. Experimental results: Cond1: 8~16 times speed up Cond2: 3~4 times speed down Cond3: 8~16 times speed up Cond4: 2~16 times speed up # Runtime of ram_save_remaining() is changed by the number of remaining # dirty pages. -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH 1/3] qemu-kvm: Wrap phys_ram_dirty with additional inline functions.
We think access phys_ram_dirty through inline functions is better
than directly for encoupseling reason.
We devided the ram in a 64 pages block. Each block has a counter, which is
stored in phys_ram_dirty_by_word. It shows the number of dirty pages.
We will find the 64 pages block is dirty or non-dirty using
phys_ram_dirty_by_word.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
cpu-all.h | 74
cpu-defs.h |1 +
2 files changed, 75 insertions(+), 0 deletions(-)
diff --git a/cpu-all.h b/cpu-all.h
index 8ed76c7..2251f14 100644
--- a/cpu-all.h
+++ b/cpu-all.h
@@ -168,6 +168,33 @@ typedef union {
} CPU_QuadU;
#endif
+static inline unsigned long unroll_flags_to_ul(int flags)
+{
+unsigned long ret = 0, flags_ul = (unsigned long)flags;
+
+#if TARGET_LONG_SIZE == 4
+ret |= flags_ul 0;
+ret |= flags_ul 8;
+ret |= flags_ul 16;
+ret |= flags_ul 24;
+#elif TARGET_LONG_SIZE == 8
+ret |= flags_ul 0;
+ret |= flags_ul 8;
+ret |= flags_ul 16;
+ret |= flags_ul 24;
+ret |= flags_ul 32;
+ret |= flags_ul 40;
+ret |= flags_ul 48;
+ret |= flags_ul 56;
+#else
+int i;
+for (i = 0; i sizeof(unsigned long); i++)
+ret |= flags_ul (i * 8);
+#endif
+
+return ret;
+}
+
/* CPU memory access without any memory or io remapping */
/*
@@ -847,6 +874,7 @@ int cpu_str_to_log_mask(const char *str);
extern int phys_ram_fd;
extern uint8_t *phys_ram_dirty;
+extern int *phys_ram_dirty_by_word;
extern ram_addr_t ram_size;
extern ram_addr_t last_ram_offset;
extern uint8_t *bios_mem;
@@ -882,6 +910,11 @@ static inline int cpu_physical_memory_is_dirty(ram_addr_t
addr)
return phys_ram_dirty[addr TARGET_PAGE_BITS] == 0xff;
}
+static inline int cpu_physical_memory_get_dirty_flags(ram_addr_t addr)
+{
+return phys_ram_dirty[addr TARGET_PAGE_BITS];
+}
+
static inline int cpu_physical_memory_get_dirty(ram_addr_t addr,
int dirty_flags)
{
@@ -890,9 +923,50 @@ static inline int cpu_physical_memory_get_dirty(ram_addr_t
addr,
static inline void cpu_physical_memory_set_dirty(ram_addr_t addr)
{
+if (phys_ram_dirty[addr TARGET_PAGE_BITS] != 0xff)
+++phys_ram_dirty_by_word[(addr TARGET_PAGE_BITS) /
+ TARGET_LONG_BITS];
+
phys_ram_dirty[addr TARGET_PAGE_BITS] = 0xff;
}
+static inline int cpu_physical_memory_set_dirty_flags(ram_addr_t addr,
+ int dirty_flags)
+{
+if ((phys_ram_dirty[addr TARGET_PAGE_BITS] != 0xff)
+((phys_ram_dirty[addr TARGET_PAGE_BITS] | dirty_flags) == 0xff))
+++phys_ram_dirty_by_word[(addr TARGET_PAGE_BITS) /
+ TARGET_LONG_BITS];
+
+return phys_ram_dirty[addr TARGET_PAGE_BITS] |= dirty_flags;
+}
+
+static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
+int length,
+int dirty_flags)
+{
+int i, mask, len, *pw;
+uint8_t *p;
+
+len = length TARGET_PAGE_BITS;
+mask = ~dirty_flags;
+p = phys_ram_dirty + (start TARGET_PAGE_BITS);
+pw = phys_ram_dirty_by_word + (start TARGET_PAGE_BITS) /
+TARGET_LONG_BITS;
+
+for (i = 0; i len; i++) {
+if (p[i] == 0xff)
+--pw[i / TARGET_LONG_BITS];
+p[i] = mask;
+}
+}
+
+int cpu_physical_memory_get_dirty_range(ram_addr_t start, ram_addr_t end,
+int dirty_flags);
+
+int cpu_physical_memory_get_non_dirty_range(ram_addr_t start, ram_addr_t end,
+int dirty_flags);
+
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
int dirty_flags);
void cpu_tlb_update_dirty(CPUState *env);
diff --git a/cpu-defs.h b/cpu-defs.h
index cf502e9..8e89e96 100644
--- a/cpu-defs.h
+++ b/cpu-defs.h
@@ -37,6 +37,7 @@
#endif
#define TARGET_LONG_SIZE (TARGET_LONG_BITS / 8)
+#define TARGET_LONG_ALIGN(addr) (((addr) + TARGET_LONG_BITS - 1) /
TARGET_LONG_BITS)
/* target_ulong is the type of a virtual address */
#if TARGET_LONG_SIZE == 4
--
1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
Re: [PATCH] qemu-kvm: Speed up of the dirty-bitmap-traveling
Would be great if you could provide a version for upstream as well
because it will likely replace this qemu-kvm code on day.
O.K. We'll prepare it.
We have implemented the version for upstream. Some source code are borrowed
from qemu-kvm.c. It is not fully tested yet, though.
We also did performance test against this patch. Test environment is the same
as the email I sent before.
Experimental results:
Test1: Guest OS read 3GB file, which is bigger than memory.
#called orig.(msec) patch(msec) ratio
14 3.790.1820.8
12 3.200.1521.4
11 2.890.1421.0
Test2: Guest OS read/write 3GB file, which is bigger than memory.
#called orig.(msec) patch(msec) ratio
364 180 8.7020.7
326 161 7.7120.9
474 235 11.720.1
---
kvm-all.c | 80 +---
1 files changed, 65 insertions(+), 15 deletions(-)
diff --git a/kvm-all.c b/kvm-all.c
index 15ec38e..9666843 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -279,9 +279,69 @@ int kvm_set_migration_log(int enable)
return 0;
}
-static int test_le_bit(unsigned long nr, unsigned char *addr)
+static inline void kvm_get_dirty_pages_log_range_by_byte(unsigned int start,
+ unsigned int end,
+ unsigned char *bitmap,
+ unsigned long offset)
{
-return (addr[nr 3] (nr 7)) 1;
+unsigned int i, j, n = 0;
+unsigned long page_number, addr, addr1;
+ram_addr_t ram_addr;
+unsigned char c;
+
+/*
+ * bitmap-traveling is faster than memory-traveling (for addr...)
+ * especially when most of the memory is not dirty.
+ */
+for (i = start; i end; i++) {
+c = bitmap[i];
+while (c 0) {
+j = ffsl(c) - 1;
+c = ~(1u j);
+page_number = i * 8 + j;
+addr1 = page_number * TARGET_PAGE_SIZE;
+addr = offset + addr1;
+ram_addr = cpu_get_physical_page_desc(addr);
+cpu_physical_memory_set_dirty(ram_addr);
+n++;
+}
+}
+}
+
+static int kvm_get_dirty_pages_log_range_by_long(unsigned long start_addr,
+ unsigned char *bitmap,
+ unsigned long mem_size)
+{
+unsigned int i;
+unsigned int len;
+unsigned long *bitmap_ul = (unsigned long *)bitmap;
+
+/* bitmap-traveling by long size is faster than by byte size
+ * especially when most of memory is not dirty.
+ * bitmap should be long-size aligned for traveling by long.
+ */
+if (((unsigned long)bitmap (TARGET_LONG_SIZE - 1)) == 0) {
+len = ((mem_size / TARGET_PAGE_SIZE) + TARGET_LONG_BITS - 1) /
+TARGET_LONG_BITS;
+for (i = 0; i len; i++)
+if (bitmap_ul[i] != 0)
+kvm_get_dirty_pages_log_range_by_byte(i * TARGET_LONG_SIZE,
+(i + 1) * TARGET_LONG_SIZE, bitmap, start_addr);
+/*
+ * We will check the remaining dirty-bitmap,
+ * when the mem_size is not a multiple of TARGET_LONG_SIZE.
+ */
+if ((mem_size (TARGET_LONG_SIZE - 1)) != 0) {
+len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+kvm_get_dirty_pages_log_range_by_byte(i * TARGET_LONG_SIZE,
+len, bitmap, start_addr);
+}
+} else { /* slow path: traveling by byte. */
+len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+kvm_get_dirty_pages_log_range_by_byte(0, len, bitmap, start_addr);
+}
+
+return 0;
}
/**
@@ -297,8 +357,6 @@ int kvm_physical_sync_dirty_bitmap(target_phys_addr_t
start_addr,
{
KVMState *s = kvm_state;
unsigned long size, allocated_size = 0;
-target_phys_addr_t phys_addr;
-ram_addr_t addr;
KVMDirtyLog d;
KVMSlot *mem;
int ret = 0;
@@ -327,17 +385,9 @@ int kvm_physical_sync_dirty_bitmap(target_phys_addr_t
start_addr,
break;
}
-for (phys_addr = mem-start_addr, addr = mem-phys_offset;
- phys_addr mem-start_addr + mem-memory_size;
- phys_addr += TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
-unsigned char *bitmap = (unsigned char *)d.dirty_bitmap;
-unsigned nr = (phys_addr - mem-start_addr) TARGET_PAGE_BITS;
-
-if (test_le_bit(nr, bitmap)) {
-cpu_physical_memory_set_dirty(addr);
-}
-}
-start_addr = phys_addr;
+kvm_get_dirty_pages_log_range_by_long(mem-start_addr,
+d.dirty_bitmap, mem-memory_size);
+start_addr = mem-start_addr +
Re: [PATCH] qemu-kvm: Speed up of the dirty-bitmap-traveling
Sounds logical - do you have numbers on the improvement? Sure. The patch showed approximately 3-7 times speed up when measured with rdtsc. The test environment and detailed results are described below. --- tmp = rdtsc(); /* function of original code*/ t1 += rdtsc() - tmp; tmp = rdtsc(); /* function of this patch */ t2 += rdtsc() - tmp; --- Test Envirionment: CPU: 4x Intel Xeon Quad Core 2.66GHz Mem size: 6GB kvm version: 2.6.31-17-server qemu version: commit ed880109f74f0a4dd5b7ec09e6a2d9ba4903d9a5 Host OS: Ubuntu 9.10 (kernel 2.6.31) Guest OS: Debian/GNU Linux lenny (kernel 2.6.26) Guest Mem size: 512MB We executed live migration three times. This data shows, how many times the function is called (#called), runtime of original (orig.), runtime of this patch (patch), speedup ratio (ratio), when live migration run. Experimental results: Test1: Guest OS read 3GB file, which is bigger than memory. #called orig.(msec) patch(msec) ratio 114 1.000.156.76 132 1.570.256.26 96 1.000.166.27 Test2: Guest OS read/write 3GB file, which is bigger than memory. #called orig.(msec) patch(msec) ratio 219638.110.63.59 225639.610.83.68 211236.310.33.53 Would be great if you could provide a version for upstream as well because it will likely replace this qemu-kvm code on day. O.K. We'll prepare it. We'll also post a patch set to quicken dirty pages checking in ram_save_block and ram_save_live soon. -- To unsubscribe from this list: send the line unsubscribe kvm in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html
[PATCH] qemu-kvm: Speed up of the dirty-bitmap-traveling
dirty-bitmap-traveling is carried out by byte size in qemu-kvm.c.
But We think that dirty-bitmap-traveling by long size is faster than by byte
size especially when most of memory is not dirty.
Signed-off-by: OHMURA Kei ohmura@lab.ntt.co.jp
---
qemu-kvm.c | 49 ++---
1 files changed, 42 insertions(+), 7 deletions(-)
diff --git a/qemu-kvm.c b/qemu-kvm.c
index a305907..5459cdd 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -2433,22 +2433,21 @@ int kvm_physical_memory_set_dirty_tracking(int enable)
}
/* get kvm's dirty pages bitmap and update qemu's */
-static int kvm_get_dirty_pages_log_range(unsigned long start_addr,
- unsigned char *bitmap,
- unsigned long offset,
- unsigned long mem_size)
+static void kvm_get_dirty_pages_log_range_by_byte(unsigned int start,
+ unsigned int end,
+ unsigned char *bitmap,
+ unsigned long offset)
{
unsigned int i, j, n = 0;
unsigned char c;
unsigned long page_number, addr, addr1;
ram_addr_t ram_addr;
-unsigned int len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
/*
* bitmap-traveling is faster than memory-traveling (for addr...)
* especially when most of the memory is not dirty.
*/
-for (i = 0; i len; i++) {
+for (i = start; i end; i++) {
c = bitmap[i];
while (c 0) {
j = ffsl(c) - 1;
@@ -2461,13 +2460,49 @@ static int kvm_get_dirty_pages_log_range(unsigned long
start_addr,
n++;
}
}
+}
+
+static int kvm_get_dirty_pages_log_range_by_long(unsigned long start_addr,
+ unsigned char *bitmap,
+ unsigned long offset,
+ unsigned long mem_size)
+{
+unsigned int i;
+unsigned int len;
+unsigned long *bitmap_ul = (unsigned long *)bitmap;
+
+/* bitmap-traveling by long size is faster than by byte size
+ * especially when most of memory is not dirty.
+ * bitmap should be long-size aligned for traveling by long.
+ */
+if (((unsigned long)bitmap (TARGET_LONG_SIZE - 1)) == 0) {
+len = ((mem_size / TARGET_PAGE_SIZE) + TARGET_LONG_BITS - 1) /
+TARGET_LONG_BITS;
+for (i = 0; i len; i++)
+if (bitmap_ul[i] != 0)
+kvm_get_dirty_pages_log_range_by_byte(i * TARGET_LONG_SIZE,
+(i + 1) * TARGET_LONG_SIZE, bitmap, offset);
+/*
+ * We will check the remaining dirty-bitmap,
+ * when the mem_size is not a multiple of TARGET_LONG_SIZE.
+ */
+if ((mem_size (TARGET_LONG_SIZE - 1)) != 0) {
+len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+kvm_get_dirty_pages_log_range_by_byte(i * TARGET_LONG_SIZE,
+len, bitmap, offset);
+}
+} else { /* slow path: traveling by byte. */
+len = ((mem_size / TARGET_PAGE_SIZE) + 7) / 8;
+kvm_get_dirty_pages_log_range_by_byte(0, len, bitmap, offset);
+}
+
return 0;
}
static int kvm_get_dirty_bitmap_cb(unsigned long start, unsigned long len,
void *bitmap, void *opaque)
{
-return kvm_get_dirty_pages_log_range(start, bitmap, start, len);
+return kvm_get_dirty_pages_log_range_by_long(start, bitmap, start, len);
}
/*
-- 1.6.3.3
--
To unsubscribe from this list: send the line unsubscribe kvm in
the body of a message to majord...@vger.kernel.org
More majordomo info at http://vger.kernel.org/majordomo-info.html
