Skip to content

Commit 39bbde1

Browse files
committed
main
1 parent cef0645 commit 39bbde1

File tree

12 files changed

+274
-90
lines changed

12 files changed

+274
-90
lines changed

pegy/Cargo.lock

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

pegy/Cargo.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ rust-version = "1.75"
1515
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
1616

1717
[dependencies]
18+
rustversion = "1.0"
1819
pegy-derive = "0.1.2"
1920
futures = {version="^0.3", optional=true}
2021
unicode-ident = {version="^1", optional=true}
@@ -35,4 +36,5 @@ tokio = "1"
3536
[features]
3637
default = ["futures", "unicode"]
3738
futures = ["dep:futures"]
38-
unicode = ["dep:unicode-ident"]
39+
unicode = ["dep:unicode-ident"]
40+
simd = []

pegy/examples/float.rs

Lines changed: 0 additions & 12 deletions
This file was deleted.

pegy/examples/json.rs

Lines changed: 2 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
use std::pin::Pin;
2-
use std::task::Poll;
3-
1+
use pegy::io::AsyncStreamRead;
42
use pegy::util::{Recursive, Repeat, RepeatQuiet, ANY, DIGIT, WHITESPACE};
53
use pegy::Parse;
64
use pegy::Span;
@@ -46,74 +44,6 @@ pub struct JsonString(Span);
4644
#[grammar(!"\"" ("\\" ("r" | "n" | "t" | "v" | "\"" |("u" DIGIT<16> DIGIT<16> DIGIT<16> DIGIT<16>)) | ANY))]
4745
pub struct StringChar;
4846

49-
struct StreamAsyncRead<S: futures::Stream<Item = Result<B, reqwest::Error>> + Unpin, B: AsRef<[u8]>>
50-
{
51-
buffer: Vec<u8>,
52-
stream: S,
53-
}
54-
55-
impl<S: futures::Stream<Item = Result<B, reqwest::Error>> + Unpin, B: AsRef<[u8]>>
56-
futures::AsyncRead for StreamAsyncRead<S, B>
57-
{
58-
fn poll_read(
59-
mut self: std::pin::Pin<&mut Self>,
60-
cx: &mut std::task::Context<'_>,
61-
buf: &mut [u8],
62-
) -> std::task::Poll<std::io::Result<usize>> {
63-
use futures::Stream;
64-
65-
if !self.buffer.is_empty() {
66-
let buffer_len = self.buffer.len();
67-
if buffer_len > buf.len() {
68-
// copy from buffer
69-
buf.copy_from_slice(&self.buffer[0..buf.len()]);
70-
// copy the remain bytes to start
71-
self.buffer.copy_within(buf.len().., 0);
72-
// resize buffer
73-
self.buffer.resize(buffer_len - buf.len(), 0);
74-
75-
// return copied length
76-
return Poll::Ready(Ok(buf.len()));
77-
};
78-
79-
// copy from buffer
80-
(&mut buf[0..self.buffer.len()]).copy_from_slice(&self.buffer);
81-
// clear buffer
82-
self.buffer.clear();
83-
84-
return Poll::Ready(Ok(buffer_len));
85-
}
86-
87-
let next = Stream::poll_next(Pin::new(&mut self.stream), cx);
88-
89-
match next {
90-
Poll::Ready(r) => match r {
91-
Some(Ok(bytes)) => {
92-
let mut bytes = bytes.as_ref();
93-
94-
if bytes.len() > buf.len() {
95-
self.buffer.extend_from_slice(&bytes[buf.len()..]);
96-
bytes = &bytes[..buf.len()];
97-
};
98-
99-
for (i, b) in bytes.iter().enumerate() {
100-
buf[i] = *b;
101-
}
102-
103-
return Poll::Ready(Ok(bytes.len()));
104-
}
105-
Some(Err(e)) => return Poll::Ready(Err(std::io::Error::other(e))),
106-
None => {
107-
return Poll::Ready(Err(std::io::Error::from(
108-
std::io::ErrorKind::UnexpectedEof,
109-
)))
110-
}
111-
},
112-
Poll::Pending => return Poll::Pending,
113-
}
114-
}
115-
}
116-
11747
pub fn main() {
11848
let rt = tokio::runtime::Builder::new_current_thread()
11949
.enable_time()
@@ -127,10 +57,7 @@ pub fn main() {
12757
let request = reqwest::get("https://raw.githubusercontent.com/serde-rs/json-benchmark/8b4046037a166b84575635b3662bbd5f9c9d7508/data/canada.json").await
12858
.unwrap();
12959

130-
let src = pegy::AsyncStrSource::new(StreamAsyncRead{
131-
buffer: Vec::new(),
132-
stream: request.bytes_stream()
133-
});
60+
let src = pegy::AsyncStrSource::new(AsyncStreamRead::new(request.bytes_stream()));
13461

13562
let _value = pegy::parse::<JsonValue, _>(src).await;
13663
});

pegy/examples/recusive.rs

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
use pegy::util::{Boxed, Recursive, ALPHANUMERIC};
2+
3+
type RecursiveName = Recursive<Boxed<Name>>;
4+
5+
#[derive(Debug, Default, PartialEq, Eq, pegy::Parse)]
6+
#[grammar($item0:ALPHANUMERIC $item1:RecursiveName?)]
7+
struct Name(char, Option<Box<Name>>);
8+
9+
pub fn main() {
10+
let name = pegy::parse_blocking::<Name, _>("ty46ncis");
11+
12+
assert_eq!(
13+
name,
14+
Ok(Name(
15+
't',
16+
Some(Box::new(Name(
17+
'y',
18+
Some(Box::new(Name(
19+
'4',
20+
Some(Box::new(Name(
21+
'6',
22+
Some(Box::new(Name(
23+
'n',
24+
Some(Box::new(Name(
25+
'c',
26+
Some(Box::new(Name('i', Some(Box::new(Name('s', None))))))
27+
)))
28+
)))
29+
)))
30+
)))
31+
)))
32+
))
33+
)
34+
}

pegy/examples/test.rs

Lines changed: 0 additions & 1 deletion
This file was deleted.

pegy/examples/unicode_ident.rs

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
use pegy::util::{
2+
UNICODE_ID_CONTINUE,
3+
UNICODE_ID_START
4+
};
5+
6+
#[derive(Debug, Default, pegy::Parse)]
7+
#[grammar($item0:UNICODE_ID_START $item1:UNICODE_ID_CONTINUE*)]
8+
pub struct UnicodeIdent(char, Vec<char>);
9+
10+
impl ToString for UnicodeIdent{
11+
fn to_string(&self) -> String {
12+
let mut buf = String::with_capacity(self.1.len() + 1);
13+
buf.push(self.0);
14+
for c in &self.1{
15+
buf.push(*c);
16+
}
17+
return buf
18+
}
19+
}
20+
21+
pub fn main(){
22+
let name = pegy::parse_blocking::<UnicodeIdent, _>("MyIdent");
23+
24+
assert!(name.is_ok());
25+
26+
assert_eq!(name.unwrap().to_string(), "MyIdent")
27+
}

pegy/src/lib.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#![cfg_attr(feature = "simd", feature(portable_simd))]
2+
13
extern crate alloc;
24

35
mod error;
@@ -17,6 +19,10 @@ pub use error::{Error, Span};
1719
pub use parse::Parse;
1820
pub use source::{AsyncStrSource, Character, IntoSource, Source, StrSource};
1921

22+
pub mod io {
23+
pub use crate::source::AsyncStreamRead;
24+
}
25+
2026
pub type Result<T> = core::result::Result<T, Error>;
2127

2228
pub async fn parse<T: Parse, S: IntoSource>(src: S) -> Result<T::Output> {

pegy/src/pratt/mod.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,9 +174,11 @@ impl<N: Clone, T: crate::Parse> Pratt<N, T> {
174174
// repeat inffix -> preffix -> primary -> suffix
175175
loop {
176176
// only match one inffix
177+
let mut has_infix = false;
177178
for (id, i) in self.inffixes.iter().rev() {
178179
let start = src.current_position();
179180
if src.match_str(i).await {
181+
has_infix = true;
180182
tokens.push(ParsedToken::Inffix(
181183
*id,
182184
Span::new(start, src.current_position()),
@@ -185,6 +187,12 @@ impl<N: Clone, T: crate::Parse> Pratt<N, T> {
185187
}
186188
}
187189

190+
// must have infix before next primary expression
191+
if !has_infix {
192+
break;
193+
}
194+
195+
// parse prefixes
188196
'prefix: loop {
189197
for (id, p) in self.prefixes.iter().rev() {
190198
let start = src.current_position();
@@ -197,6 +205,7 @@ impl<N: Clone, T: crate::Parse> Pratt<N, T> {
197205
break;
198206
}
199207

208+
// parse primary expression
200209
match T::parse(src).await {
201210
Ok(prim) => {
202211
tokens.push(ParsedToken::Primary(prim));

0 commit comments

Comments
 (0)