This is an automated email from the ASF dual-hosted git repository.
psiace pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-opendal.git
The following commit(s) were added to refs/heads/main by this push:
new d2b4315ac refactor: Polish fuzz build time (#2721)
d2b4315ac is described below
commit d2b4315ac76674877a5bf41c171658e9e525f045
Author: Xuanwo <[email protected]>
AuthorDate: Thu Jul 27 16:12:23 2023 +0800
refactor: Polish fuzz build time (#2721)
* refactor: Polish fuzz build time
Signed-off-by: Xuanwo <[email protected]>
* Fix build
Signed-off-by: Xuanwo <[email protected]>
* try fix build
Signed-off-by: Xuanwo <[email protected]>
* Don't upload all files
Signed-off-by: Xuanwo <[email protected]>
* Use bash64 is enough
Signed-off-by: Xuanwo <[email protected]>
* Fix fuzz
Signed-off-by: Xuanwo <[email protected]>
* fix
Signed-off-by: Xuanwo <[email protected]>
---------
Signed-off-by: Xuanwo <[email protected]>
---
.github/workflows/fuzz_test.yml | 59 +++++-----
Cargo.lock | 1 +
core/fuzz/Cargo.toml | 13 +--
core/fuzz/fuzz_range_reader.rs | 252 ----------------------------------------
core/fuzz/fuzz_reader.rs | 204 ++++++++++++++++++--------------
core/fuzz/fuzz_writer.rs | 128 +++++++++-----------
core/fuzz/utils.rs | 29 +++--
7 files changed, 222 insertions(+), 464 deletions(-)
diff --git a/.github/workflows/fuzz_test.yml b/.github/workflows/fuzz_test.yml
index 1a11e06aa..7feaf3969 100644
--- a/.github/workflows/fuzz_test.yml
+++ b/.github/workflows/fuzz_test.yml
@@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
-
name: Fuzz Test
on:
@@ -39,12 +38,16 @@ concurrency:
cancel-in-progress: true
jobs:
- fuzz-test-build-target:
+ fuzz-build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Setup Rust toolchain
uses: ./.github/actions/setup
+ - name: Install libfuzz
+ shell: bash
+ run: sudo apt-get install -y libfuzzer-14-dev
+
- name: Install cargo fuzz
shell: bash
run: rustup install nightly && cargo +nightly install cargo-fuzz
@@ -52,14 +55,20 @@ jobs:
shell: bash
working-directory: core/fuzz
run: cargo +nightly fuzz build
+ env:
+ CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
+
- name: Upload build artifacts
uses: actions/upload-artifact@v3
with:
name: fuzz_targets
- path: ./target/x86_64-unknown-linux-gnu/release/fuzz_*
- fuzz-test-run-s3:
+ path: |
+ ./target/x86_64-unknown-linux-gnu/release/fuzz_reader
+ ./target/x86_64-unknown-linux-gnu/release/fuzz_writer
+
+ fuzz-test-s3:
runs-on: ubuntu-latest
- needs: fuzz-test-build-target
+ needs: fuzz-build
services:
minio:
image: wktk/minio-server
@@ -71,8 +80,10 @@ jobs:
strategy:
fail-fast: true
matrix:
- fuzz-targets: [ fuzz_reader, fuzz_range_reader, fuzz_writer ]
+ fuzz-targets: [ fuzz_reader, fuzz_writer ]
steps:
+ - name: Install libfuzzer
+ run: sudo apt-get install -y libfuzzer-14-dev
- name: Setup Test Bucket
env:
AWS_ACCESS_KEY_ID: "minioadmin"
@@ -96,20 +107,17 @@ jobs:
OPENDAL_S3_ENDPOINT: "http://127.0.0.1:9000";
OPENDAL_S3_ACCESS_KEY_ID: minioadmin
OPENDAL_S3_SECRET_ACCESS_KEY: minioadmin
- - name: Upload Crash Files
- if: ${{ failure() }}
- uses: actions/upload-artifact@v3
- with:
- name: crash_s3_${{ matrix.fuzz-targets }}_${{ github.event_name
}}_${{ github.run_attempt }}_${{ github.sha }}
- path: ./crash*
- fuzz-test-run-fs:
+
+ fuzz-test-fs:
runs-on: ubuntu-latest
- needs: fuzz-test-build-target
+ needs: fuzz-build
strategy:
fail-fast: true
matrix:
- fuzz-targets: [ fuzz_reader, fuzz_range_reader, fuzz_writer ]
+ fuzz-targets: [ fuzz_reader, fuzz_writer ]
steps:
+ - name: Install libfuzzer
+ run: sudo apt-get install -y libfuzzer-14-dev
- name: Download Fuzz Targets
uses: actions/download-artifact@v3
with:
@@ -124,20 +132,17 @@ jobs:
env:
OPENDAL_FS_TEST: on
OPENDAL_FS_ROOT: ${{ runner.temp }}/
- - name: Upload Crash Files
- uses: actions/upload-artifact@v3
- if: ${{ failure() }}
- with:
- name: crash_fs_${{ matrix.fuzz-targets }}_${{ github.event_name
}}_${{ github.run_attempt }}_${{ github.sha }}
- path: ./crash*
- fuzz-test-run-memory:
+
+ fuzz-test-memory:
runs-on: ubuntu-latest
- needs: fuzz-test-build-target
+ needs: fuzz-build
strategy:
fail-fast: true
matrix:
- fuzz-targets: [ fuzz_reader, fuzz_range_reader, fuzz_writer ]
+ fuzz-targets: [ fuzz_reader, fuzz_writer ]
steps:
+ - name: Install libfuzzer
+ run: sudo apt-get install -y libfuzzer-14-dev
- name: Download Fuzz Targets
uses: actions/download-artifact@v3
with:
@@ -151,9 +156,3 @@ jobs:
run: ./target/${{ matrix.fuzz-targets }} -max_total_time=120
env:
OPENDAL_MEMORY_TEST: on
- - name: Upload Crash Files
- if: ${{ failure() }}
- uses: actions/upload-artifact@v3
- with:
- name: crash_memory_${{ matrix.fuzz-targets }}_${{ github.event_name
}}_${{ github.run_attempt }}_${{ github.sha }}
- path: ./crash*
diff --git a/Cargo.lock b/Cargo.lock
index cb0b29ed6..29292833f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3194,6 +3194,7 @@ dependencies = [
"dotenvy",
"libfuzzer-sys",
"opendal",
+ "rand 0.8.5",
"sha2",
"tokio",
"uuid",
diff --git a/core/fuzz/Cargo.toml b/core/fuzz/Cargo.toml
index 37184ff5a..8020dfaf4 100644
--- a/core/fuzz/Cargo.toml
+++ b/core/fuzz/Cargo.toml
@@ -31,24 +31,15 @@ bytes = "1.2"
dotenvy = "0.15.6"
libfuzzer-sys = "0.4"
opendal = { path = ".." }
+rand = "0.8"
sha2 = { version = "0.10.6" }
tokio = { version = "1", features = ["full"] }
-uuid = { version = "1.3.0", features = ["v4"] }
+uuid = { version = "1", features = ["v4"] }
[[bin]]
-doc = false
name = "fuzz_reader"
path = "fuzz_reader.rs"
-test = false
[[bin]]
-doc = false
name = "fuzz_writer"
path = "fuzz_writer.rs"
-test = false
-
-[[bin]]
-doc = false
-name = "fuzz_range_reader"
-path = "fuzz_range_reader.rs"
-test = false
diff --git a/core/fuzz/fuzz_range_reader.rs b/core/fuzz/fuzz_range_reader.rs
deleted file mode 100644
index ba7ade043..000000000
--- a/core/fuzz/fuzz_range_reader.rs
+++ /dev/null
@@ -1,252 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#![no_main]
-
-use std::io::SeekFrom;
-
-use bytes::Bytes;
-use libfuzzer_sys::arbitrary::Arbitrary;
-use libfuzzer_sys::arbitrary::Result;
-use libfuzzer_sys::arbitrary::Unstructured;
-use libfuzzer_sys::fuzz_target;
-use sha2::Digest;
-use sha2::Sha256;
-
-use opendal::raw::oio::ReadExt;
-use opendal::Operator;
-
-mod utils;
-
-const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
-
-#[derive(Debug, Clone)]
-enum ReaderAction {
- Read { size: usize },
- Seek(SeekFrom),
- Next,
-}
-
-#[derive(Debug, Clone)]
-struct FuzzInput {
- actions: Vec<ReaderAction>,
- data: Vec<u8>,
-
- range: (u64, u64),
-}
-
-impl Arbitrary<'_> for FuzzInput {
- fn arbitrary(u: &mut Unstructured<'_>) -> Result<Self> {
- let data_len = u.int_in_range(1..=MAX_DATA_SIZE)?;
- let data: Vec<u8> = u.bytes(data_len)?.to_vec();
-
- let range_start = u.int_in_range(0..=data_len as u64 - 1)?;
- let range_end = u.int_in_range(range_start + 1..=data_len as u64)?;
-
- let range = (range_start, range_end);
-
- let mut actions = vec![];
- let mut action_count = u.int_in_range(128..=1024)?;
-
- while action_count != 0 {
- action_count -= 1;
- match u.int_in_range(0..=2)? {
- 0 => {
- let size = u.int_in_range(0..=data_len * 2)?;
- actions.push(ReaderAction::Read { size });
- }
- 1 => {
- let offset: i64 = u.int_in_range(-(data_len as
i64)..=(data_len as i64))?;
- let seek_from = match u.int_in_range(0..=2)? {
- 0 => SeekFrom::Start(offset.unsigned_abs()),
- 1 => SeekFrom::End(offset),
- _ => SeekFrom::Current(offset),
- };
- actions.push(ReaderAction::Seek(seek_from));
- }
- _ => actions.push(ReaderAction::Next),
- }
- }
- Ok(FuzzInput {
- actions,
- data,
- range,
- })
- }
-}
-
-struct ReaderFuzzerChecker {
- data: Vec<u8>,
- size: usize,
- cur: usize,
- start: usize,
-}
-
-impl ReaderFuzzerChecker {
- fn new(data: Vec<u8>, start: usize, end: usize) -> Self {
- Self {
- size: end - start,
- data,
- cur: 0,
- start,
- }
- }
-
- fn check_read(&mut self, n: usize, output: &[u8]) {
- if n == 0 {
- return;
- }
-
- let current = self.cur + self.start;
- let expected = &self.data[current..current + n];
-
- // Check the read result
- assert_eq!(
- format!("{:x}", Sha256::digest(output)),
- format!("{:x}", Sha256::digest(expected)),
- "check read failed: output bs is different with expected bs",
- );
-
- // Update the current position
- self.cur += n;
- }
-
- fn check_seek(&mut self, seek_from: SeekFrom, output:
opendal::Result<u64>) {
- let expected = match seek_from {
- SeekFrom::Start(offset) => offset as i64,
- SeekFrom::End(offset) => self.size as i64 + offset,
- SeekFrom::Current(offset) => self.cur as i64 + offset,
- };
-
- if expected < 0 {
- assert!(output.is_err(), "check seek failed: seek should fail");
- assert_eq!(
- output.unwrap_err().kind(),
- opendal::ErrorKind::InvalidInput,
- "check seek failed: seek result is different with expected
result",
- );
- } else {
- assert_eq!(
- output.unwrap(),
- expected as u64,
- "check seek failed: seek result is different with expected
result",
- );
-
- self.cur = expected as usize;
- }
- }
-
- fn check_next(&mut self, output: Option<Bytes>) {
- if let Some(output) = output {
- assert!(
- self.cur + output.len() <= self.size,
- "check next failed: output bs is larger than remaining bs",
- );
-
- let current = self.cur + self.start;
- let expected = &self.data[current..current + output.len()];
-
- assert_eq!(
- format!("{:x}", Sha256::digest(&output)),
- format!("{:x}", Sha256::digest(expected)),
- "check next failed: output bs is different with expected bs",
- );
-
- // update the current position
- self.cur += output.len();
- } else {
- assert!(
- self.cur >= self.size,
- "check next failed: output bs is None, we still have bytes to
read",
- )
- }
- }
-}
-
-async fn fuzz_range_reader_process(input: FuzzInput, op: &Operator, name:
&str) -> Result<()> {
- let path = uuid::Uuid::new_v4().to_string();
-
- let mut checker = ReaderFuzzerChecker::new(
- input.data.clone(),
- input.range.0 as usize,
- input.range.1 as usize,
- );
-
- op.write(&path, input.data)
- .await
- .unwrap_or_else(|_| panic!("{} write must succeed", name));
-
- let mut o = op
- .range_reader(&path, input.range.0..input.range.1)
- .await
- .unwrap_or_else(|_| panic!("{} init reader must succeed", name));
-
- for action in input.actions {
- match action {
- ReaderAction::Read { size } => {
- let mut buf = vec![0; size];
- let n = o
- .read(&mut buf)
- .await
- .unwrap_or_else(|_| panic!("{} read must succeed", name));
- checker.check_read(n, &buf[..n]);
- }
-
- ReaderAction::Seek(seek_from) => {
- let res = o.seek(seek_from).await;
- checker.check_seek(seek_from, res);
- }
-
- ReaderAction::Next => {
- let res = o
- .next()
- .await
- .map(|v| v.unwrap_or_else(|_| panic!("{} next should not
return error", name)));
- checker.check_next(res);
- }
- }
- }
-
- op.delete(&path)
- .await
- .unwrap_or_else(|_| panic!("{} delete must succeed", name));
- Ok(())
-}
-
-fn fuzz_reader(name: &str, op: &Operator, input: FuzzInput) {
- let runtime = tokio::runtime::Runtime::new().unwrap();
-
- runtime.block_on(async {
- fuzz_range_reader_process(input, op, name)
- .await
- .unwrap_or_else(|_| panic!("{} fuzz range reader must succeed",
name));
- });
-}
-
-fuzz_target!(|input: FuzzInput| {
- let _ = dotenvy::dotenv();
-
- for service in utils::init_services() {
- if service.1.is_none() {
- continue;
- }
-
- let op = service.1.unwrap();
-
- fuzz_reader(service.0, &op, input.clone());
- }
-});
diff --git a/core/fuzz/fuzz_reader.rs b/core/fuzz/fuzz_reader.rs
index 8ec87f591..4cd8a68ff 100644
--- a/core/fuzz/fuzz_reader.rs
+++ b/core/fuzz/fuzz_reader.rs
@@ -21,21 +21,23 @@ use std::io::SeekFrom;
use bytes::Bytes;
use libfuzzer_sys::arbitrary::Arbitrary;
-use libfuzzer_sys::arbitrary::Result;
use libfuzzer_sys::arbitrary::Unstructured;
use libfuzzer_sys::fuzz_target;
+use rand::prelude::*;
use sha2::Digest;
use sha2::Sha256;
use opendal::raw::oio::ReadExt;
+use opendal::raw::BytesRange;
use opendal::Operator;
+use opendal::Result;
mod utils;
const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
#[derive(Debug, Clone)]
-enum ReaderAction {
+enum ReadAction {
Read { size: usize },
Seek(SeekFrom),
Next,
@@ -43,62 +45,115 @@ enum ReaderAction {
#[derive(Debug, Clone)]
struct FuzzInput {
- actions: Vec<ReaderAction>,
- data: Vec<u8>,
+ size: usize,
+ range: BytesRange,
+ actions: Vec<ReadAction>,
}
impl Arbitrary<'_> for FuzzInput {
- fn arbitrary(u: &mut Unstructured<'_>) -> Result<Self> {
- let data_len = u.int_in_range(1..=MAX_DATA_SIZE)?;
- let data: Vec<u8> = u.bytes(data_len)?.to_vec();
+ fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
+ let total_size = u.int_in_range(1..=MAX_DATA_SIZE)?;
+
+ // TODO: it's valid that size is larger than total_size.
+ let (offset, size) = match u.int_in_range(0..=3)? {
+ // Full range
+ 0 => (None, None),
+ 1 => {
+ let offset = u.int_in_range(0..=total_size as u64 - 1)?;
+ (Some(offset), None)
+ }
+ 2 => {
+ let size = u.int_in_range(1..=total_size as u64)?;
+ (None, Some(size))
+ }
+ 3 => {
+ let offset = u.int_in_range(0..=total_size as u64 - 1)?;
+ let size = u.int_in_range(1..=total_size as u64 - offset)?;
+ (Some(offset), Some(size))
+ }
+ _ => unreachable!("invalid int generated by arbitrary"),
+ };
+ let range = BytesRange::new(offset, size);
+ let count = u.int_in_range(1..=1024)?;
let mut actions = vec![];
- let mut action_count = u.int_in_range(128..=1024)?;
- while action_count != 0 {
- action_count -= 1;
- match u.int_in_range(0..=2)? {
+ for _ in 0..count {
+ let action = match u.int_in_range(0..=4)? {
+ // Read
0 => {
- let size = u.int_in_range(0..=data_len * 2)?;
- actions.push(ReaderAction::Read { size });
+ let size = u.int_in_range(0..=total_size * 2)?;
+ ReadAction::Read { size }
}
- 1 => {
- let offset: i64 = u.int_in_range(-(data_len as
i64)..=(data_len as i64))?;
- let seek_from = match u.int_in_range(0..=2)? {
- 0 => SeekFrom::Start(offset.unsigned_abs()),
- 1 => SeekFrom::End(offset),
- _ => SeekFrom::Current(offset),
- };
- actions.push(ReaderAction::Seek(seek_from));
+ // Next
+ 1 => ReadAction::Next,
+ // Seek Start
+ 2 => {
+ // NOTE: seek out of the end of file is valid.
+ let offset = u.int_in_range(0..=total_size * 2)?;
+ ReadAction::Seek(SeekFrom::Start(offset as u64))
}
- _ => actions.push(ReaderAction::Next),
- }
+ // Seek Current
+ 3 => {
+ let offset = u.int_in_range(-(total_size as
i64)..=(total_size as i64))?;
+ ReadAction::Seek(SeekFrom::Current(offset))
+ }
+ // Seek End
+ 4 => {
+ let offset = u.int_in_range(-(total_size as
i64)..=(total_size as i64))?;
+ ReadAction::Seek(SeekFrom::End(offset))
+ }
+ _ => unreachable!("invalid int generated by arbitrary"),
+ };
+
+ actions.push(action);
}
- Ok(FuzzInput { actions, data })
+
+ Ok(FuzzInput {
+ size: total_size,
+ range,
+ actions,
+ })
}
}
-struct ReaderFuzzerChecker {
- data: Vec<u8>,
- size: usize,
+struct ReadChecker {
+ /// Raw Data is the data we write to the storage.
+ raw_data: Bytes,
+ /// Ranged Data is the data that we read from the storage.
+ ranged_data: Bytes,
+
cur: usize,
}
-impl ReaderFuzzerChecker {
- fn new(data: Vec<u8>) -> Self {
+impl ReadChecker {
+ fn new(size: usize, range: BytesRange) -> Self {
+ let mut rng = thread_rng();
+ let mut data = vec![0; size];
+ rng.fill_bytes(&mut data);
+
+ let raw_data = Bytes::from(data);
+ let ranged_data = range.apply_on_bytes(raw_data.clone());
+
Self {
- size: data.len(),
- data,
+ raw_data,
+ ranged_data,
+
cur: 0,
}
}
fn check_read(&mut self, n: usize, output: &[u8]) {
if n == 0 {
+ assert_eq!(
+ output.len(),
+ 0,
+ "check read failed: output bs is not empty when read size is 0"
+ );
return;
}
- let expected = &self.data[self.cur..self.cur + n];
+ let expected = &self.ranged_data[self.cur..self.cur + n];
// Check the read result
assert_eq!(
@@ -111,10 +166,10 @@ impl ReaderFuzzerChecker {
self.cur += n;
}
- fn check_seek(&mut self, seek_from: SeekFrom, output:
opendal::Result<u64>) {
+ fn check_seek(&mut self, seek_from: SeekFrom, output: Result<u64>) {
let expected = match seek_from {
SeekFrom::Start(offset) => offset as i64,
- SeekFrom::End(offset) => self.size as i64 + offset,
+ SeekFrom::End(offset) => self.ranged_data.len() as i64 + offset,
SeekFrom::Current(offset) => self.cur as i64 + offset,
};
@@ -125,22 +180,24 @@ impl ReaderFuzzerChecker {
opendal::ErrorKind::InvalidInput,
"check seek failed: seek result is different with expected
result"
);
- } else {
- assert_eq!(
- output.unwrap(),
- expected as u64,
- "check seek failed: seek result is different with expected
result",
- );
- // only update the current position when seek succeed
- self.cur = expected as usize;
+ return;
}
+
+ assert_eq!(
+ output.unwrap(),
+ expected as u64,
+ "check seek failed: seek result is different with expected result",
+ );
+
+ // only update the current position when seek succeed
+ self.cur = expected as usize;
}
fn check_next(&mut self, output: Option<Bytes>) {
if let Some(output) = output {
assert!(
- self.cur + output.len() <= self.size,
+ self.cur + output.len() <= self.ranged_data.len(),
"check next failed: output bs is larger than remaining bs",
);
@@ -148,7 +205,7 @@ impl ReaderFuzzerChecker {
format!("{:x}", Sha256::digest(&output)),
format!(
"{:x}",
- Sha256::digest(&self.data[self.cur..self.cur +
output.len()])
+ Sha256::digest(&self.ranged_data[self.cur..self.cur +
output.len()])
),
"check next failed: output bs is different with expected bs",
);
@@ -157,78 +214,55 @@ impl ReaderFuzzerChecker {
self.cur += output.len();
} else {
assert!(
- self.cur >= self.size,
+ self.cur >= self.ranged_data.len(),
"check next failed: output bs is None, we still have bytes to
read",
)
}
}
}
-async fn fuzz_reader_process(input: FuzzInput, op: &Operator, name: &str) ->
Result<()> {
+async fn fuzz_reader(op: Operator, input: FuzzInput) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let mut checker = ReaderFuzzerChecker::new(input.data.clone());
- op.write(&path, input.data)
- .await
- .unwrap_or_else(|_| panic!("{} write must succeed", name));
+ let mut checker = ReadChecker::new(input.size, input.range);
+ op.write(&path, checker.raw_data.clone()).await?;
- let mut o = op
- .reader(&path)
- .await
- .unwrap_or_else(|_| panic!("{} init reader must succeed", name));
+ let mut o = op.range_reader(&path, input.range.to_range()).await?;
for action in input.actions {
match action {
- ReaderAction::Read { size } => {
+ ReadAction::Read { size } => {
let mut buf = vec![0; size];
- let n = o
- .read(&mut buf)
- .await
- .unwrap_or_else(|_| panic!("{} read must succeed", name));
+ let n = o.read(&mut buf).await?;
checker.check_read(n, &buf[..n]);
}
- ReaderAction::Seek(seek_from) => {
+ ReadAction::Seek(seek_from) => {
let res = o.seek(seek_from).await;
checker.check_seek(seek_from, res);
}
- ReaderAction::Next => {
- let res = o
- .next()
- .await
- .map(|v| v.unwrap_or_else(|_| panic!("{} next should not
return error", name)));
+ ReadAction::Next => {
+ let res = o.next().await.transpose()?;
checker.check_next(res);
}
}
}
- op.delete(&path)
- .await
- .unwrap_or_else(|_| panic!("{} delete must succeed", name));
+ op.delete(&path).await?;
Ok(())
}
-fn fuzz_reader(name: &str, op: &Operator, input: FuzzInput) {
- let runtime = tokio::runtime::Runtime::new().unwrap();
-
- runtime.block_on(async {
- fuzz_reader_process(input, op, name)
- .await
- .unwrap_or_else(|_| panic!("{} fuzz reader must succeed", name));
- });
-}
-
fuzz_target!(|input: FuzzInput| {
let _ = dotenvy::dotenv();
- for service in utils::init_services() {
- if service.1.is_none() {
- continue;
- }
-
- let op = service.1.unwrap();
+ let runtime = tokio::runtime::Runtime::new().expect("init runtime must
succeed");
- fuzz_reader(service.0, &op, input.clone());
+ for op in utils::init_services() {
+ runtime.block_on(async {
+ fuzz_reader(op, input.clone())
+ .await
+ .unwrap_or_else(|_| panic!("fuzz reader must succeed"));
+ })
}
});
diff --git a/core/fuzz/fuzz_writer.rs b/core/fuzz/fuzz_writer.rs
index 261ffedf5..63c3ebb3a 100644
--- a/core/fuzz/fuzz_writer.rs
+++ b/core/fuzz/fuzz_writer.rs
@@ -17,15 +17,16 @@
#![no_main]
-use bytes::Bytes;
+use bytes::{Bytes, BytesMut};
use libfuzzer_sys::arbitrary::Arbitrary;
-use libfuzzer_sys::arbitrary::Result;
use libfuzzer_sys::arbitrary::Unstructured;
use libfuzzer_sys::fuzz_target;
+use rand::prelude::*;
use sha2::Digest;
use sha2::Sha256;
use opendal::Operator;
+use opendal::Result;
mod utils;
@@ -33,7 +34,7 @@ const MAX_DATA_SIZE: usize = 16 * 1024 * 1024;
#[derive(Debug, Clone)]
enum WriterAction {
- Write { data: Bytes },
+ Write { size: usize },
}
#[derive(Debug, Clone)]
@@ -42,40 +43,46 @@ struct FuzzInput {
}
impl Arbitrary<'_> for FuzzInput {
- fn arbitrary(u: &mut Unstructured<'_>) -> Result<Self> {
+ fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
let mut actions = vec![];
- let mut action_count = u.int_in_range(128..=1024)?;
-
- while action_count != 0 {
- action_count -= 1;
- let data_len = u.int_in_range(1..=MAX_DATA_SIZE)?;
- let data: Vec<u8> = u.bytes(data_len)?.to_vec();
- actions.push(WriterAction::Write {
- data: Bytes::from(data),
- });
+
+ let count = u.int_in_range(128..=1024)?;
+
+ for _ in 0..count {
+ let size = u.int_in_range(1..=MAX_DATA_SIZE)?;
+ actions.push(WriterAction::Write { size });
}
Ok(FuzzInput { actions })
}
}
-struct WriterFuzzChecker {
- data: Vec<u8>,
+struct WriteChecker {
+ chunks: Vec<Bytes>,
+ data: Bytes,
}
-impl WriterFuzzChecker {
- fn new(input: FuzzInput) -> Self {
- let mut data = vec![];
+impl WriteChecker {
+ fn new(size: Vec<usize>) -> Self {
+ let mut rng = thread_rng();
- for action in input.actions {
- match action {
- WriterAction::Write { data: d } => {
- data.extend_from_slice(&d);
- }
- }
+ let mut chunks = Vec::with_capacity(size.len());
+
+ for i in size {
+ let mut bs = vec![0u8; i];
+ rng.fill_bytes(&mut bs);
+ chunks.push(Bytes::from(bs));
}
- WriterFuzzChecker { data }
+ let data = chunks.iter().fold(BytesMut::new(), |mut acc, x| {
+ acc.extend_from_slice(x);
+ acc
+ });
+
+ WriteChecker {
+ chunks,
+ data: data.freeze(),
+ }
}
fn check(&self, actual: &[u8]) {
@@ -87,64 +94,45 @@ impl WriterFuzzChecker {
}
}
-async fn fuzz_writer_process(input: FuzzInput, op: &Operator, name: &str) ->
Result<()> {
+async fn fuzz_writer(op: Operator, input: FuzzInput) -> Result<()> {
let path = uuid::Uuid::new_v4().to_string();
- let checker = WriterFuzzChecker::new(input.clone());
-
- let mut writer = op
- .writer(&path)
- .await
- .unwrap_or_else(|_| panic!("{} create must succeed", name));
-
- for action in input.actions {
- match action {
- WriterAction::Write { data } => {
- writer
- .write(data)
- .await
- .unwrap_or_else(|_| panic!("{} write must succeed", name));
- }
- }
+ let total_size = input
+ .actions
+ .iter()
+ .map(|a| match a {
+ WriterAction::Write { size } => *size,
+ })
+ .collect();
+
+ let checker = WriteChecker::new(total_size);
+
+ let mut writer = op.writer(&path).await?;
+
+ for chunk in &checker.chunks {
+ writer.write(chunk.clone()).await?;
}
- writer
- .close()
- .await
- .unwrap_or_else(|_| panic!("{} close must succeed", name));
- let result = op
- .read(&path)
- .await
- .unwrap_or_else(|_| panic!("{} read must succeed", name));
+ writer.close().await?;
+
+ let result = op.read(&path).await?;
checker.check(&result);
- op.delete(&path)
- .await
- .unwrap_or_else(|_| panic!("{} delete must succeed", name));
+ op.delete(&path).await?;
Ok(())
}
-fn fuzz_writer(name: &str, op: &Operator, input: FuzzInput) {
- let runtime = tokio::runtime::Runtime::new().unwrap();
-
- runtime.block_on(async {
- fuzz_writer_process(input, op, name)
- .await
- .unwrap_or_else(|_| panic!("{} fuzz writer must succeed", name));
- });
-}
-
fuzz_target!(|input: FuzzInput| {
let _ = dotenvy::dotenv();
- for service in utils::init_services() {
- if service.1.is_none() {
- continue;
- }
-
- let op = service.1.unwrap();
+ let runtime = tokio::runtime::Runtime::new().expect("init runtime must
succeed");
- fuzz_writer(service.0, &op, input.clone());
+ for op in utils::init_services() {
+ runtime.block_on(async {
+ fuzz_writer(op, input.clone())
+ .await
+ .unwrap_or_else(|_| panic!("fuzz reader must succeed"));
+ })
}
});
diff --git a/core/fuzz/utils.rs b/core/fuzz/utils.rs
index 83b708a9c..adc301557 100644
--- a/core/fuzz/utils.rs
+++ b/core/fuzz/utils.rs
@@ -17,17 +17,16 @@
use std::env;
-use opendal::services;
-use opendal::Builder;
use opendal::Operator;
+use opendal::Scheme;
-fn service<B: Builder>() -> Option<Operator> {
- let test_key = format!("opendal_{}_test", B::SCHEME).to_uppercase();
+fn service(scheme: Scheme) -> Option<Operator> {
+ let test_key = format!("opendal_{}_test", scheme).to_uppercase();
if env::var(test_key).unwrap_or_default() != "on" {
return None;
}
- let prefix = format!("opendal_{}_", B::SCHEME);
+ let prefix = format!("opendal_{}_", scheme);
let envs = env::vars()
.filter_map(move |(k, v)| {
k.to_lowercase()
@@ -36,17 +35,15 @@ fn service<B: Builder>() -> Option<Operator> {
})
.collect();
- Some(
- Operator::from_map::<B>(envs)
- .unwrap_or_else(|_| panic!("init {} must succeed", B::SCHEME))
- .finish(),
- )
+ Some(Operator::via_map(scheme, envs).unwrap_or_else(|_| panic!("init {}
must succeed", scheme)))
}
-pub fn init_services() -> Vec<(&'static str, Option<Operator>)> {
- vec![
- ("fs", service::<services::Fs>()),
- ("memory", service::<services::Memory>()),
- ("s3", service::<services::S3>()),
- ]
+pub fn init_services() -> Vec<Operator> {
+ let ops = vec![
+ service(Scheme::Memory),
+ service(Scheme::Fs),
+ service(Scheme::S3),
+ ];
+
+ ops.into_iter().flatten().collect()
}
