Re: [Libguestfs] [libnbd PATCH v4 11/11] rust: Add some examples
On 8/2/2023 6:53 PM, Richard W.M. Jones wrote:
On Wed, Aug 02, 2023 at 12:40:56PM +, Tage Johansson wrote:
This patch adds a few examples in rust/examples/. The examples are
compiled and run as part of the test suite.
---
rust/Makefile.am | 3 +
rust/examples/concurrent-read-write.rs | 135 +
rust/examples/connect-command.rs | 39 +++
rust/examples/fetch-first-sector.rs| 38 +++
rust/examples/get-size.rs | 29 ++
rust/run-tests.sh | 7 ++
6 files changed, 251 insertions(+)
create mode 100644 rust/examples/concurrent-read-write.rs
create mode 100644 rust/examples/connect-command.rs
create mode 100644 rust/examples/fetch-first-sector.rs
create mode 100644 rust/examples/get-size.rs
diff --git a/rust/Makefile.am b/rust/Makefile.am
index b954b22..d75163d 100644
--- a/rust/Makefile.am
+++ b/rust/Makefile.am
@@ -32,6 +32,9 @@ source_files = \
src/types.rs \
src/utils.rs \
src/async_handle.rs \
+ examples/connect-command.rs \
+ examples/get-size.rs \
+ examples/fetch-first-sector.rs \
This doesn't list all the source files, it is missing
examples/concurrent-read-write.rs.
If you split out examples/connect-command.rs, examples/get-size.rs and
examples/fetch-first-sector.rs into a separate patch (since those
don't depend on asynch), and moved that patch earlier in the sequence,
then it could go upstream earlier.
Yes, I have done that.
Best regards,
Tage
libnbd-sys/Cargo.toml \
libnbd-sys/build.rs \
$(NULL)
diff --git a/rust/examples/concurrent-read-write.rs
b/rust/examples/concurrent-read-write.rs
new file mode 100644
index 000..a1c3e8a
--- /dev/null
+++ b/rust/examples/concurrent-read-write.rs
@@ -0,0 +1,135 @@
+//! Example usage with nbdkit:
+//!
+//! nbdkit -U - memory 100M \
+//! --run 'cargo run --example concurrent-read-write -- $unixsocket'
+//!
+//! This will read and write randomly over the first megabyte of the
+//! plugin using multi-conn, multiple threads and multiple requests in
+//! flight on each thread.
+
+#![deny(warnings)]
+use rand::prelude::*;
+use std::env;
+use std::path::PathBuf;
+use std::sync::Arc;
+use tokio::task::JoinSet;
+
+/// Number of simultaneous connections to the NBD server.
+///
+/// Note that some servers only support a limited number of
+/// simultaneous connections, and/or have a configurable thread pool
+/// internally, and if you exceed those limits then something will break.
+const NR_MULTI_CONN: usize = 8;
+
+/// Number of commands that can be "in flight" at the same time on each
+/// connection. (Therefore the total number of requests in flight may
+/// be up to NR_MULTI_CONN * MAX_IN_FLIGHT).
+const MAX_IN_FLIGHT: usize = 16;
+
+/// The size of large reads and writes, must be > 512.
+const BUFFER_SIZE: usize = 1024;
+
+/// Number of commands we issue (per [task][tokio::task]).
+const NR_CYCLES: usize = 32;
+
+/// Statistics gathered during the run.
+#[derive(Debug, Default)]
+struct Stats {
+/// The total number of requests made.
+requests: usize,
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+let args = env::args_os().collect::>();
+if args.len() != 2 {
+anyhow::bail!("Usage: {:?} socket", args[0]);
+}
+let socket = [1];
+
+// We begin by making a connection to the server to get the export size
+// and ensure that it supports multiple connections and is writable.
+let nbd = libnbd::Handle::new()?;
+nbd.connect_unix()?;
+let export_size = nbd.get_size()?;
+anyhow::ensure!(
+(BUFFER_SIZE as u64) < export_size,
+"export is {export_size}B, must be larger than {BUFFER_SIZE}B"
+);
+anyhow::ensure!(
+!nbd.is_read_only()?,
+"error: this NBD export is read-only"
+);
+anyhow::ensure!(
+nbd.can_multi_conn()?,
+"error: this NBD export does not support multi-conn"
+);
+drop(nbd); // Close the connection.
+
+// Start the worker tasks, one per connection.
+let mut tasks = JoinSet::new();
+for i in 0..NR_MULTI_CONN {
+tasks.spawn(run_thread(i, socket.clone().into(), export_size));
+}
+
+// Wait for the tasks to complete.
+let mut stats = Stats::default();
+while !tasks.is_empty() {
+let this_stats = tasks.join_next().await.unwrap().unwrap()?;
+stats.requests += this_stats.requests;
+}
+
+// Make sure the number of requests that were required matches what
+// we expect.
+assert_eq!(stats.requests, NR_MULTI_CONN * NR_CYCLES);
+
+Ok(())
+}
+
+async fn run_thread(
+task_idx: usize,
+socket: PathBuf,
+export_size: u64,
+) -> anyhow::Result {
+// Start a new connection to the server.
+// We shall spawn many commands concurrently on different tasks and those
+// futures must be `'static`, hence we wrap the handle in an [Arc].
+let
Re: [Libguestfs] [libnbd PATCH v4 11/11] rust: Add some examples
On Wed, Aug 02, 2023 at 12:40:56PM +, Tage Johansson wrote:
> This patch adds a few examples in rust/examples/. The examples are
> compiled and run as part of the test suite.
> ---
> rust/Makefile.am | 3 +
> rust/examples/concurrent-read-write.rs | 135 +
> rust/examples/connect-command.rs | 39 +++
> rust/examples/fetch-first-sector.rs| 38 +++
> rust/examples/get-size.rs | 29 ++
> rust/run-tests.sh | 7 ++
> 6 files changed, 251 insertions(+)
> create mode 100644 rust/examples/concurrent-read-write.rs
> create mode 100644 rust/examples/connect-command.rs
> create mode 100644 rust/examples/fetch-first-sector.rs
> create mode 100644 rust/examples/get-size.rs
>
> diff --git a/rust/Makefile.am b/rust/Makefile.am
> index b954b22..d75163d 100644
> --- a/rust/Makefile.am
> +++ b/rust/Makefile.am
> @@ -32,6 +32,9 @@ source_files = \
> src/types.rs \
> src/utils.rs \
> src/async_handle.rs \
> + examples/connect-command.rs \
> + examples/get-size.rs \
> + examples/fetch-first-sector.rs \
This doesn't list all the source files, it is missing
examples/concurrent-read-write.rs.
If you split out examples/connect-command.rs, examples/get-size.rs and
examples/fetch-first-sector.rs into a separate patch (since those
don't depend on asynch), and moved that patch earlier in the sequence,
then it could go upstream earlier.
> libnbd-sys/Cargo.toml \
> libnbd-sys/build.rs \
> $(NULL)
> diff --git a/rust/examples/concurrent-read-write.rs
> b/rust/examples/concurrent-read-write.rs
> new file mode 100644
> index 000..a1c3e8a
> --- /dev/null
> +++ b/rust/examples/concurrent-read-write.rs
> @@ -0,0 +1,135 @@
> +//! Example usage with nbdkit:
> +//!
> +//! nbdkit -U - memory 100M \
> +//! --run 'cargo run --example concurrent-read-write -- $unixsocket'
> +//!
> +//! This will read and write randomly over the first megabyte of the
> +//! plugin using multi-conn, multiple threads and multiple requests in
> +//! flight on each thread.
> +
> +#![deny(warnings)]
> +use rand::prelude::*;
> +use std::env;
> +use std::path::PathBuf;
> +use std::sync::Arc;
> +use tokio::task::JoinSet;
> +
> +/// Number of simultaneous connections to the NBD server.
> +///
> +/// Note that some servers only support a limited number of
> +/// simultaneous connections, and/or have a configurable thread pool
> +/// internally, and if you exceed those limits then something will break.
> +const NR_MULTI_CONN: usize = 8;
> +
> +/// Number of commands that can be "in flight" at the same time on each
> +/// connection. (Therefore the total number of requests in flight may
> +/// be up to NR_MULTI_CONN * MAX_IN_FLIGHT).
> +const MAX_IN_FLIGHT: usize = 16;
> +
> +/// The size of large reads and writes, must be > 512.
> +const BUFFER_SIZE: usize = 1024;
> +
> +/// Number of commands we issue (per [task][tokio::task]).
> +const NR_CYCLES: usize = 32;
> +
> +/// Statistics gathered during the run.
> +#[derive(Debug, Default)]
> +struct Stats {
> +/// The total number of requests made.
> +requests: usize,
> +}
> +
> +#[tokio::main]
> +async fn main() -> anyhow::Result<()> {
> +let args = env::args_os().collect::>();
> +if args.len() != 2 {
> +anyhow::bail!("Usage: {:?} socket", args[0]);
> +}
> +let socket = [1];
> +
> +// We begin by making a connection to the server to get the export size
> +// and ensure that it supports multiple connections and is writable.
> +let nbd = libnbd::Handle::new()?;
> +nbd.connect_unix()?;
> +let export_size = nbd.get_size()?;
> +anyhow::ensure!(
> +(BUFFER_SIZE as u64) < export_size,
> +"export is {export_size}B, must be larger than {BUFFER_SIZE}B"
> +);
> +anyhow::ensure!(
> +!nbd.is_read_only()?,
> +"error: this NBD export is read-only"
> +);
> +anyhow::ensure!(
> +nbd.can_multi_conn()?,
> +"error: this NBD export does not support multi-conn"
> +);
> +drop(nbd); // Close the connection.
> +
> +// Start the worker tasks, one per connection.
> +let mut tasks = JoinSet::new();
> +for i in 0..NR_MULTI_CONN {
> +tasks.spawn(run_thread(i, socket.clone().into(), export_size));
> +}
> +
> +// Wait for the tasks to complete.
> +let mut stats = Stats::default();
> +while !tasks.is_empty() {
> +let this_stats = tasks.join_next().await.unwrap().unwrap()?;
> +stats.requests += this_stats.requests;
> +}
> +
> +// Make sure the number of requests that were required matches what
> +// we expect.
> +assert_eq!(stats.requests, NR_MULTI_CONN * NR_CYCLES);
> +
> +Ok(())
> +}
> +
> +async fn run_thread(
> +task_idx: usize,
> +socket: PathBuf,
> +export_size: u64,
> +) -> anyhow::Result {
> +// Start a new connection to the server.
> +// We shall spawn many
[Libguestfs] [libnbd PATCH v4 11/11] rust: Add some examples
This patch adds a few examples in rust/examples/. The examples are
compiled and run as part of the test suite.
---
rust/Makefile.am | 3 +
rust/examples/concurrent-read-write.rs | 135 +
rust/examples/connect-command.rs | 39 +++
rust/examples/fetch-first-sector.rs| 38 +++
rust/examples/get-size.rs | 29 ++
rust/run-tests.sh | 7 ++
6 files changed, 251 insertions(+)
create mode 100644 rust/examples/concurrent-read-write.rs
create mode 100644 rust/examples/connect-command.rs
create mode 100644 rust/examples/fetch-first-sector.rs
create mode 100644 rust/examples/get-size.rs
diff --git a/rust/Makefile.am b/rust/Makefile.am
index b954b22..d75163d 100644
--- a/rust/Makefile.am
+++ b/rust/Makefile.am
@@ -32,6 +32,9 @@ source_files = \
src/types.rs \
src/utils.rs \
src/async_handle.rs \
+ examples/connect-command.rs \
+ examples/get-size.rs \
+ examples/fetch-first-sector.rs \
libnbd-sys/Cargo.toml \
libnbd-sys/build.rs \
$(NULL)
diff --git a/rust/examples/concurrent-read-write.rs
b/rust/examples/concurrent-read-write.rs
new file mode 100644
index 000..a1c3e8a
--- /dev/null
+++ b/rust/examples/concurrent-read-write.rs
@@ -0,0 +1,135 @@
+//! Example usage with nbdkit:
+//!
+//! nbdkit -U - memory 100M \
+//! --run 'cargo run --example concurrent-read-write -- $unixsocket'
+//!
+//! This will read and write randomly over the first megabyte of the
+//! plugin using multi-conn, multiple threads and multiple requests in
+//! flight on each thread.
+
+#![deny(warnings)]
+use rand::prelude::*;
+use std::env;
+use std::path::PathBuf;
+use std::sync::Arc;
+use tokio::task::JoinSet;
+
+/// Number of simultaneous connections to the NBD server.
+///
+/// Note that some servers only support a limited number of
+/// simultaneous connections, and/or have a configurable thread pool
+/// internally, and if you exceed those limits then something will break.
+const NR_MULTI_CONN: usize = 8;
+
+/// Number of commands that can be "in flight" at the same time on each
+/// connection. (Therefore the total number of requests in flight may
+/// be up to NR_MULTI_CONN * MAX_IN_FLIGHT).
+const MAX_IN_FLIGHT: usize = 16;
+
+/// The size of large reads and writes, must be > 512.
+const BUFFER_SIZE: usize = 1024;
+
+/// Number of commands we issue (per [task][tokio::task]).
+const NR_CYCLES: usize = 32;
+
+/// Statistics gathered during the run.
+#[derive(Debug, Default)]
+struct Stats {
+/// The total number of requests made.
+requests: usize,
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+let args = env::args_os().collect::>();
+if args.len() != 2 {
+anyhow::bail!("Usage: {:?} socket", args[0]);
+}
+let socket = [1];
+
+// We begin by making a connection to the server to get the export size
+// and ensure that it supports multiple connections and is writable.
+let nbd = libnbd::Handle::new()?;
+nbd.connect_unix()?;
+let export_size = nbd.get_size()?;
+anyhow::ensure!(
+(BUFFER_SIZE as u64) < export_size,
+"export is {export_size}B, must be larger than {BUFFER_SIZE}B"
+);
+anyhow::ensure!(
+!nbd.is_read_only()?,
+"error: this NBD export is read-only"
+);
+anyhow::ensure!(
+nbd.can_multi_conn()?,
+"error: this NBD export does not support multi-conn"
+);
+drop(nbd); // Close the connection.
+
+// Start the worker tasks, one per connection.
+let mut tasks = JoinSet::new();
+for i in 0..NR_MULTI_CONN {
+tasks.spawn(run_thread(i, socket.clone().into(), export_size));
+}
+
+// Wait for the tasks to complete.
+let mut stats = Stats::default();
+while !tasks.is_empty() {
+let this_stats = tasks.join_next().await.unwrap().unwrap()?;
+stats.requests += this_stats.requests;
+}
+
+// Make sure the number of requests that were required matches what
+// we expect.
+assert_eq!(stats.requests, NR_MULTI_CONN * NR_CYCLES);
+
+Ok(())
+}
+
+async fn run_thread(
+task_idx: usize,
+socket: PathBuf,
+export_size: u64,
+) -> anyhow::Result {
+// Start a new connection to the server.
+// We shall spawn many commands concurrently on different tasks and those
+// futures must be `'static`, hence we wrap the handle in an [Arc].
+let nbd = Arc::new(libnbd::AsyncHandle::new()?);
+nbd.connect_unix(socket).await?;
+
+let mut rng = SmallRng::seed_from_u64(44 as u64);
+
+// Issue commands.
+let mut stats = Stats::default();
+let mut join_set = JoinSet::new();
+//tokio::time::sleep(std::time::Duration::from_secs(1)).await;
+while stats.requests < NR_CYCLES || !join_set.is_empty() {
+while stats.requests < NR_CYCLES && join_set.len() < MAX_IN_FLIGHT {
+// If we want to issue another request, do
