Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions .github/workflows/auto-approve.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: Auto approve

on: pull_request_target

jobs:
auto-approve:
runs-on: ubuntu-latest
permissions:
pull-requests: write
if: github.actor == 'dependabot[bot]' || github.actor == 'jmesmon'
steps:
- uses: hmarr/auto-approve-action@v3
12 changes: 12 additions & 0 deletions .github/workflows/auto-merge.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
name: Auto-Merge
on:
pull_request_target:

jobs:
enable-auto-merge:
runs-on: ubuntu-latest

steps:
- uses: alexwilson/enable-github-automerge-action@main
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
7 changes: 4 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ jobs:
- stable
- beta
- nightly
- 1.47.0
- 1.64.0

steps:
- uses: actions/checkout@v2
Expand All @@ -26,7 +26,8 @@ jobs:
override: true

- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v1
uses: Swatinem/[email protected]


- name: Run all tests
uses: actions-rs/cargo@v1
Expand All @@ -47,7 +48,7 @@ jobs:
components: rustfmt, clippy

- name: Cache Rust dependencies
uses: Swatinem/rust-cache@v1
uses: Swatinem/rust-cache@v2.4.0

- uses: actions-rs/cargo@v1
with:
Expand Down
20 changes: 11 additions & 9 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ keywords = [ "hash", "rolling", "incremental", "split" , "cdc"]
license = "AGPL-3.0-or-later"
repository = "https://github.com/jmesmon/hash-roll.git"
documentation = "https://docs.rs/hash-roll"
include = ["Cargo.toml", "**/*.rs", "README.md", "COMPARE.md"]
include = ["**/*.rs", "README.md", "COMPARE.md"]
edition = "2018"

[features]
Expand All @@ -21,7 +21,8 @@ default = [
"pigz",
"ram",
"zpaq",
"zstd"
"zstd",

]

bup = []
Expand All @@ -35,18 +36,19 @@ ram = []
zpaq = []
zstd = []

slow_tests = []

[dependencies]
fmt-extra = "0.2"
#circbuf = "0.1.4"

[dev-dependencies]
rand = "0.7.3"
histogram = "0.6"
quickcheck = "0.9"
rand = "0.8"
histogram = "0.7"
quickcheck = "1"
rollsum = "0.3"
criterion = "0.3"
rand_pcg = "0.2.1"
proptest = "0.10.0"
criterion = "0.5"
rand_pcg = "0.3"
proptest = "1"

[[bench]]
name = "compare"
Expand Down
8 changes: 0 additions & 8 deletions bors.toml

This file was deleted.

3 changes: 0 additions & 3 deletions rustfmt.toml

This file was deleted.

2 changes: 1 addition & 1 deletion src/buzhash.rs
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ impl fmt::Debug for BuzHashTableBufHash {
}
}

impl<'a> From<Box<[u32; 256]>> for BuzHashTableBufHash {
impl From<Box<[u32; 256]>> for BuzHashTableBufHash {
fn from(table: Box<[u32; 256]>) -> Self {
Self { table }
}
Expand Down
10 changes: 2 additions & 8 deletions src/gear.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,14 +72,8 @@ impl<'a> ToChunkIncr for Gear32<'a> {
type Incr = GearIncr32<'a>;

fn to_chunk_incr(&self) -> Self::Incr {
self.into()
}
}

impl<'a> From<&Gear32<'a>> for GearIncr32<'a> {
fn from(params: &Gear32<'a>) -> Self {
Self {
params: params.clone(),
Self::Incr {
params: self.clone(),
state: Default::default(),
}
}
Expand Down
15 changes: 9 additions & 6 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -110,10 +110,10 @@ pub mod gzip;
pub mod mii;
pub mod pigz;
pub mod ram;
pub mod range;
pub mod zpaq;
pub mod zstd;

mod range;
pub(crate) use range::RangeExt;

/// Accept incrimental input and provide indexes of split points
Expand Down Expand Up @@ -287,9 +287,12 @@ pub trait Chunk {

/// Find the next "chunk" in `data` to emit
///
/// The return value is a pair of a range representing the start and end of the chunk being
/// emitted, and the offset from which subsequent `data` subsets should be passed to the next
/// call to `find_chunk_edge`.
/// The return value is a tuple of:
///
/// 1. an optional offset in `data` which is just after a chunk to be emitted (this is called
/// the "cut point")
/// 2. the number of bytes to remove from the start of `data` before passing `data` to
/// `find_chunk_edge()` again (this is called the "discard count")
///
/// `state` is mutated so that it does not rexamine previously examined data, even when a chunk
/// is not emitted.
Expand All @@ -312,9 +315,9 @@ pub trait Chunk {
/// let mut prev_cut = 0;
///
/// loop {
/// let (chunk, discard_ct) = chunk.find_chunk_edge(&mut ss, data);
/// let (cut_point_in_data, discard_ct) = chunk.find_chunk_edge(&mut ss, data);
///
/// match chunk {
/// match cut_point_in_data {
/// Some(cut_point) => {
/// // map `cut_point` from the current slice back into the original slice so we can
/// // have consistent indexes
Expand Down
22 changes: 7 additions & 15 deletions src/mii.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,12 @@ impl crate::Chunk for Mii {
) -> (Option<usize>, usize) {
match state.push(data) {
Some(v) => {
state.reset();
// NOTE: these check that we don't carry state across chunk boundaries. The
// original paper is unclear if this is _exactly_ the case, and there are no test
// vectors for MII. That said: given the goals, it's unlikely that `prev` should
// persist across chunks (as doing so would cause inconsistent chunking).
debug_assert_eq!(state.incr.prev, 0xff);
debug_assert_eq!(state.incr.increment, 0);
(Some(v), v)
}
None => (None, data.len()),
Expand All @@ -78,12 +83,6 @@ pub struct MiiSearchState {
incr: MiiIncr,
}

impl MiiSearchState {
fn reset(&mut self) {
self.incr.reset();
}
}

impl From<MiiIncr> for MiiSearchState {
fn from(incr: MiiIncr) -> Self {
Self { incr }
Expand Down Expand Up @@ -127,7 +126,7 @@ impl ChunkIncr for MiiIncr {
if self.increment == self.w {
// this is a split
self.increment = 0;
self.prev = 0;
self.prev = 0xff;
return Some(i + 1);
}
} else {
Expand All @@ -139,10 +138,3 @@ impl ChunkIncr for MiiIncr {
None
}
}

impl MiiIncr {
fn reset(&mut self) {
self.prev = 0xff;
self.increment = 0;
}
}
8 changes: 4 additions & 4 deletions src/range.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,12 @@ impl<T, R: std::ops::RangeBounds<T>> RangeExt<T> for R {
T: PartialOrd<T>,
{
match self.end_bound() {
Included(ref i) => {
Included(i) => {
if item > i {
return true;
}
}
Excluded(ref i) => {
Excluded(i) => {
if item >= i {
return true;
}
Expand All @@ -45,12 +45,12 @@ impl<T, R: std::ops::RangeBounds<T>> RangeExt<T> for R {
T: PartialOrd<T>,
{
match self.start_bound() {
Included(ref i) => {
Included(i) => {
if item < i {
return true;
}
}
Excluded(ref i) => {
Excluded(i) => {
if item <= i {
return true;
}
Expand Down
2 changes: 1 addition & 1 deletion src/zpaq.rs
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ impl Chunk for Zpaq {
) -> (Option<usize>, usize) {
for (i, v) in data.iter().enumerate() {
let h = state.feed(*v);
if self.split_here(h, (state.idx + 1) as u64) {
if self.split_here(h, state.idx + 1) {
*state = self.to_search_state();
return (Some(i + 1), i + 1);
}
Expand Down
4 changes: 4 additions & 0 deletions tests/cuts_qc.proptest-regressions

Large diffs are not rendered by default.

Loading