From ba3ee4ca1af4f7327c1f88e185924b131d64fad6 Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Tue, 29 Aug 2023 23:20:23 +0200 Subject: [PATCH 01/10] update merge --- Cargo.toml | 20 ++++++++++++++++++++ requirements-dev.txt | 5 ++++- src/lib.rs | 19 +++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 Cargo.toml create mode 100644 src/lib.rs diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..8961bd0 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "iotext" +version = "0.1.0" +rust-version = "1.57.0" + +[lib] +name = "_iotext" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = { version = "0.19.0", features = ["auto-initialize", "extension-module"] } + +# dependencies +fancy-regex = "0.11.0" +regex = "1.8.3" +rustc-hash = "1.1.0" +bstr = "1.5.0" + +[profile.release] +incremental = true diff --git a/requirements-dev.txt b/requirements-dev.txt index 168cb73..a206bbd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,4 +3,7 @@ black==22.3.0 parameterized==0.9.0 pytest-benchmark==4.0.0 aiohttp==3.8.5 -pytest-aiohttp==0.3.0 \ No newline at end of file +pytest-aiohttp==0.3.0 + +# required for RUST development +maturin==1.2.1 diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..e285bb4 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,19 @@ +// This check is new and seems buggy (possibly with PyO3 interaction) +#![allow(clippy::borrow_deref_ref)] + +use pyo3::prelude::*; + +/// Formats the sum of two numbers as string. +#[pyfunction] +fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) +} + +/// A Python module implemented in Rust. The name of this function must match +/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to +/// import the module. +#[pymodule] +fn string_sum(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + Ok(()) +} \ No newline at end of file From 4bfb6496ae71dd5cf9e77ef6b4b5a9a33a99befe Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Tue, 15 Aug 2023 15:01:39 +0200 Subject: [PATCH 02/10] RUST code coverage with CODECOV in github actions --- .github/workflows/rust_coverage.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/rust_coverage.yaml diff --git a/.github/workflows/rust_coverage.yaml b/.github/workflows/rust_coverage.yaml new file mode 100644 index 0000000..c5a5f9e --- /dev/null +++ b/.github/workflows/rust_coverage.yaml @@ -0,0 +1,23 @@ +name: RUST code coverage + +on: [pull_request, push] + +jobs: + coverage: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v3 + - name: Install Rust + run: rustup update stable + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Generate code coverage + run: cargo llvm-cov --all-features --workspace --codecov --output-path codecov.json + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos + files: codecov.json + fail_ci_if_error: true From 6d29d3aa35ad3460722996e185711e505cee66ad Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 16 Aug 2023 00:50:35 +0200 Subject: [PATCH 03/10] add parser prototype for IoText message in RUST --- src/test_parse.rs | 119 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 src/test_parse.rs diff --git a/src/test_parse.rs b/src/test_parse.rs new file mode 100644 index 0000000..ee2bf45 --- /dev/null +++ b/src/test_parse.rs @@ -0,0 +1,119 @@ +use std::*; + + +// COPY -> PASTE to https://play.rust-lang.org/ for check parser prototype + + + +//use std::intrinsics::type_name::*; +//use std::num::ParseIntError; +//use rust_decimal::Decimal; + + +struct MetricDataTypes {} + +impl MetricDataTypes { + const INTEGER: &str = "i"; + //const BOOL: &str = "b"; + //const DECIMAL: &str = "d"; + //const TEXT: &str = "t"; +} + + +struct ItemType {} +impl ItemType { + const TIME_UNIX_MILIS: &str = "t"; + const DEVICE_ID: &str = "d"; + //const METRIC: &str = "m"; +} + +#[derive(Debug)] +enum MetricValueType { + IntegerItemType(i64), + //BoolItemType(bool), + //DecimalItemType(i64), + //TextItemType(String), +} + +#[derive(Debug)] +enum ItemTypeEnum { + TimeUnixMilis(u64), + DeviceId(String), +} + +//#![feature(core_intrinsics)] +//fn print_type_of(_: &T) { +// println!("{}", unsafe { std::intrinsics::type_name::() }); +//} + +fn main() { + let item_parts: Vec<&str> = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42" + .split(',') + .collect(); + + for part in item_parts { + println!("part: {}", part); + let item_part: Vec<&str> = part.split('|').collect(); + println!("item_part: {:?}", item_part); + let item_type_tmp: &str = item_part[0]; + let item_type_metric: &str = "m"; + if item_type_tmp.eq(item_type_metric) { + println!("\tmetric: {}", item_part[1]); + let metric_parts: Vec<&str> = item_part[1].split('=').collect(); + println!("\tmetric_parts: {:?}", metric_parts); + //let metric_name: String = item_parts[0].to_string(); + //println!("metric_name: {}", metric_name); + let metric_parts_values: Vec<&str> = metric_parts[1].split(':').collect(); + println!("\t\tmetric_parts_values: {:?}", metric_parts_values); + match metric_parts_values[0] { + MetricDataTypes::INTEGER => { + //value: Result = metric_parts_values[0].parse(); + //let value: Option = metric_parts_values[0].parse(); + //let value: Result = metric_parts_values[0].parse::(); + let value = match metric_parts_values[1].parse::() { + Ok(number) => number, + Err(_) => todo!(), + }; + println!( + "\t\t\tIntegerItemType: {:?}", + MetricValueType::IntegerItemType(value) + ) + //MetricValueType::IntegerItemType(value); + } + _ => println!("\t\t\tother"), + } + } else { + match item_part[0] { + ItemType::TIME_UNIX_MILIS => { + let value = match item_part[1].parse::() { + Ok(number) => number, + Err(_) => todo!(), + }; + println!( + "\t\t\tTIME_UNIX_MILIS: {:?}", + ItemTypeEnum::TimeUnixMilis(value) + ) + } + ItemType::DEVICE_ID => { + + println!("\t\t\tDEVICE_ID: {:?}", ItemTypeEnum::DeviceId(String::from(item_part[1]))) + } + val => { + println!("\t\t\t OTHER: {:?}", val); + //print_type_of(val) + } + } + println!("\t\tcontext: {}", item_part[1]) + } + /*match item_part.split('=').collect() { + [item, details] => { + println!("{:?}", item); + println!("{:?}", details); + } + _ => println!("rest"), + }*/ + } + //assert_eq!(v, ["Mary", "had", "a", "little", "lamb"]); + //println!("{:?}", v); + //MetricValueType::IntegerItemType(0) +} From 26f0ddaac8862911380e9a538431c7b2d80d8fd0 Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 16 Aug 2023 00:51:15 +0200 Subject: [PATCH 04/10] add parser prototype for IoText message in RUST --- src/test_parse.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/test_parse.rs b/src/test_parse.rs index ee2bf45..40c7d86 100644 --- a/src/test_parse.rs +++ b/src/test_parse.rs @@ -117,3 +117,24 @@ fn main() { //println!("{:?}", v); //MetricValueType::IntegerItemType(0) } + +/* +STDOUT: + + +part: t|3900237526042 +item_part: ["t", "3900237526042"] + TIME_UNIX_MILIS: TimeUnixMilis(3900237526042) + context: 3900237526042 +part: d|device_name_001 +item_part: ["d", "device_name_001"] + DEVICE_ID: DeviceId("device_name_001") + context: device_name_001 +part: m|val_water_level1=i:42 +item_part: ["m", "val_water_level1=i:42"] + metric: val_water_level1=i:42 + metric_parts: ["val_water_level1", "i:42"] + metric_parts_values: ["i", "42"] + IntegerItemType: IntegerItemType(42) + +*/ From ed665f2d52eb4eccbcd7db5fed1256f4143bac4d Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 16 Aug 2023 01:00:56 +0200 Subject: [PATCH 05/10] add parser prototype for IoText message in RUST --- src/test_parse.rs | 61 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/src/test_parse.rs b/src/test_parse.rs index 40c7d86..4a7b6a2 100644 --- a/src/test_parse.rs +++ b/src/test_parse.rs @@ -14,9 +14,9 @@ struct MetricDataTypes {} impl MetricDataTypes { const INTEGER: &str = "i"; - //const BOOL: &str = "b"; + const BOOL: &str = "b"; //const DECIMAL: &str = "d"; - //const TEXT: &str = "t"; + const TEXT: &str = "t"; } @@ -24,15 +24,15 @@ struct ItemType {} impl ItemType { const TIME_UNIX_MILIS: &str = "t"; const DEVICE_ID: &str = "d"; - //const METRIC: &str = "m"; + const METRIC: &str = "m"; } #[derive(Debug)] enum MetricValueType { IntegerItemType(i64), - //BoolItemType(bool), + BoolItemType(bool), //DecimalItemType(i64), - //TextItemType(String), + TextItemType(String), } #[derive(Debug)] @@ -41,13 +41,10 @@ enum ItemTypeEnum { DeviceId(String), } -//#![feature(core_intrinsics)] -//fn print_type_of(_: &T) { -// println!("{}", unsafe { std::intrinsics::type_name::() }); -//} +const MSG_EXAMPLE: &str = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42,m|light_on=b:1,m|bulb_on=b:0,m|msg_machine_01=t:hello"; fn main() { - let item_parts: Vec<&str> = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42" + let item_parts: Vec<&str> = MSG_EXAMPLE .split(',') .collect(); @@ -67,9 +64,6 @@ fn main() { println!("\t\tmetric_parts_values: {:?}", metric_parts_values); match metric_parts_values[0] { MetricDataTypes::INTEGER => { - //value: Result = metric_parts_values[0].parse(); - //let value: Option = metric_parts_values[0].parse(); - //let value: Result = metric_parts_values[0].parse::(); let value = match metric_parts_values[1].parse::() { Ok(number) => number, Err(_) => todo!(), @@ -78,7 +72,23 @@ fn main() { "\t\t\tIntegerItemType: {:?}", MetricValueType::IntegerItemType(value) ) - //MetricValueType::IntegerItemType(value); + } + MetricDataTypes::BOOL => { + let value = match metric_parts_values[1] { + "1" => true, + "0" => false, + _ => todo!(), + }; + println!( + "\t\t\tBoolItemType: {:?}", + MetricValueType::BoolItemType(value) + ) + } + MetricDataTypes::TEXT => { + println!( + "\t\t\tBoolItemType: {:?}", + MetricValueType::TextItemType(metric_parts_values[1].to_string()) + ) } _ => println!("\t\t\tother"), } @@ -95,9 +105,11 @@ fn main() { ) } ItemType::DEVICE_ID => { - println!("\t\t\tDEVICE_ID: {:?}", ItemTypeEnum::DeviceId(String::from(item_part[1]))) } + ItemType::METRIC => { + println!("\t\t\tMETRIC: {}", String::from(item_part[1])) + } val => { println!("\t\t\t OTHER: {:?}", val); //print_type_of(val) @@ -136,5 +148,22 @@ item_part: ["m", "val_water_level1=i:42"] metric_parts: ["val_water_level1", "i:42"] metric_parts_values: ["i", "42"] IntegerItemType: IntegerItemType(42) - +part: m|light_on=b:1 +item_part: ["m", "light_on=b:1"] + metric: light_on=b:1 + metric_parts: ["light_on", "b:1"] + metric_parts_values: ["b", "1"] + BoolItemType: BoolItemType(true) +part: m|bulb_on=b:0 +item_part: ["m", "bulb_on=b:0"] + metric: bulb_on=b:0 + metric_parts: ["bulb_on", "b:0"] + metric_parts_values: ["b", "0"] + BoolItemType: BoolItemType(false) +part: m|msg_machine_01=t:hello +item_part: ["m", "msg_machine_01=t:hello"] + metric: msg_machine_01=t:hello + metric_parts: ["msg_machine_01", "t:hello"] + metric_parts_values: ["t", "hello"] + BoolItemType: TextItemType("hello") */ From 4b8aee35fccbf8d1cba8e4066190199df6f4a937 Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 16 Aug 2023 01:11:11 +0200 Subject: [PATCH 06/10] add parser prototype for IoText message in RUST --- src/test_parse.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/test_parse.rs b/src/test_parse.rs index 4a7b6a2..1dd37f3 100644 --- a/src/test_parse.rs +++ b/src/test_parse.rs @@ -31,7 +31,7 @@ impl ItemType { enum MetricValueType { IntegerItemType(i64), BoolItemType(bool), - //DecimalItemType(i64), + //DecimalItemType(Decimal), TextItemType(String), } @@ -41,7 +41,7 @@ enum ItemTypeEnum { DeviceId(String), } -const MSG_EXAMPLE: &str = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42,m|light_on=b:1,m|bulb_on=b:0,m|msg_machine_01=t:hello"; +const MSG_EXAMPLE: &str = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42,m|light_on=b:1,m|bulb_on=b:0,m|msg_machine_01=t:hello,m|wind_speed=d:1234.5678"; fn main() { let item_parts: Vec<&str> = MSG_EXAMPLE @@ -90,6 +90,14 @@ fn main() { MetricValueType::TextItemType(metric_parts_values[1].to_string()) ) } + /* + MetricDataTypes::DECIMAL => { + println!( + "\t\t\tDecimalItemType: {:?}", + MetricValueType::DecimalItemType(metric_parts_values[1].to_string()) + ) + } + */ _ => println!("\t\t\tother"), } } else { @@ -166,4 +174,10 @@ item_part: ["m", "msg_machine_01=t:hello"] metric_parts: ["msg_machine_01", "t:hello"] metric_parts_values: ["t", "hello"] BoolItemType: TextItemType("hello") +part: m|wind_speed=d:1234.5678 +item_part: ["m", "wind_speed=d:1234.5678"] + metric: wind_speed=d:1234.5678 + metric_parts: ["wind_speed", "d:1234.5678"] + metric_parts_values: ["d", "1234.5678"] + other */ From acec1d20b10a3caeebb8ebd936bd041f2490cfed Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 16 Aug 2023 01:11:32 +0200 Subject: [PATCH 07/10] add parser prototype for IoText message in RUST --- src/test_parse.rs | 66 ++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/src/test_parse.rs b/src/test_parse.rs index 1dd37f3..a00a325 100644 --- a/src/test_parse.rs +++ b/src/test_parse.rs @@ -1,15 +1,11 @@ use std::*; - // COPY -> PASTE to https://play.rust-lang.org/ for check parser prototype - - //use std::intrinsics::type_name::*; //use std::num::ParseIntError; //use rust_decimal::Decimal; - struct MetricDataTypes {} impl MetricDataTypes { @@ -19,7 +15,6 @@ impl MetricDataTypes { const TEXT: &str = "t"; } - struct ItemType {} impl ItemType { const TIME_UNIX_MILIS: &str = "t"; @@ -44,9 +39,7 @@ enum ItemTypeEnum { const MSG_EXAMPLE: &str = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42,m|light_on=b:1,m|bulb_on=b:0,m|msg_machine_01=t:hello,m|wind_speed=d:1234.5678"; fn main() { - let item_parts: Vec<&str> = MSG_EXAMPLE - .split(',') - .collect(); + let item_parts: Vec<&str> = MSG_EXAMPLE.split(',').collect(); for part in item_parts { println!("part: {}", part); @@ -76,7 +69,7 @@ fn main() { MetricDataTypes::BOOL => { let value = match metric_parts_values[1] { "1" => true, - "0" => false, + "0" => false, _ => todo!(), }; println!( @@ -90,7 +83,7 @@ fn main() { MetricValueType::TextItemType(metric_parts_values[1].to_string()) ) } - /* + /* MetricDataTypes::DECIMAL => { println!( "\t\t\tDecimalItemType: {:?}", @@ -113,7 +106,10 @@ fn main() { ) } ItemType::DEVICE_ID => { - println!("\t\t\tDEVICE_ID: {:?}", ItemTypeEnum::DeviceId(String::from(item_part[1]))) + println!( + "\t\t\tDEVICE_ID: {:?}", + ItemTypeEnum::DeviceId(String::from(item_part[1])) + ) } ItemType::METRIC => { println!("\t\t\tMETRIC: {}", String::from(item_part[1])) @@ -144,40 +140,40 @@ STDOUT: part: t|3900237526042 item_part: ["t", "3900237526042"] - TIME_UNIX_MILIS: TimeUnixMilis(3900237526042) - context: 3900237526042 + TIME_UNIX_MILIS: TimeUnixMilis(3900237526042) + context: 3900237526042 part: d|device_name_001 item_part: ["d", "device_name_001"] - DEVICE_ID: DeviceId("device_name_001") - context: device_name_001 + DEVICE_ID: DeviceId("device_name_001") + context: device_name_001 part: m|val_water_level1=i:42 item_part: ["m", "val_water_level1=i:42"] - metric: val_water_level1=i:42 - metric_parts: ["val_water_level1", "i:42"] - metric_parts_values: ["i", "42"] - IntegerItemType: IntegerItemType(42) + metric: val_water_level1=i:42 + metric_parts: ["val_water_level1", "i:42"] + metric_parts_values: ["i", "42"] + IntegerItemType: IntegerItemType(42) part: m|light_on=b:1 item_part: ["m", "light_on=b:1"] - metric: light_on=b:1 - metric_parts: ["light_on", "b:1"] - metric_parts_values: ["b", "1"] - BoolItemType: BoolItemType(true) + metric: light_on=b:1 + metric_parts: ["light_on", "b:1"] + metric_parts_values: ["b", "1"] + BoolItemType: BoolItemType(true) part: m|bulb_on=b:0 item_part: ["m", "bulb_on=b:0"] - metric: bulb_on=b:0 - metric_parts: ["bulb_on", "b:0"] - metric_parts_values: ["b", "0"] - BoolItemType: BoolItemType(false) + metric: bulb_on=b:0 + metric_parts: ["bulb_on", "b:0"] + metric_parts_values: ["b", "0"] + BoolItemType: BoolItemType(false) part: m|msg_machine_01=t:hello item_part: ["m", "msg_machine_01=t:hello"] - metric: msg_machine_01=t:hello - metric_parts: ["msg_machine_01", "t:hello"] - metric_parts_values: ["t", "hello"] - BoolItemType: TextItemType("hello") + metric: msg_machine_01=t:hello + metric_parts: ["msg_machine_01", "t:hello"] + metric_parts_values: ["t", "hello"] + BoolItemType: TextItemType("hello") part: m|wind_speed=d:1234.5678 item_part: ["m", "wind_speed=d:1234.5678"] - metric: wind_speed=d:1234.5678 - metric_parts: ["wind_speed", "d:1234.5678"] - metric_parts_values: ["d", "1234.5678"] - other + metric: wind_speed=d:1234.5678 + metric_parts: ["wind_speed", "d:1234.5678"] + metric_parts_values: ["d", "1234.5678"] + other */ From 6a378c1809f5b55507f54d2269d3d9daaaa75dab Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Tue, 29 Aug 2023 23:28:45 +0200 Subject: [PATCH 08/10] fix bug in Cargo.toml in case of PyO3 building - we need to add one line edition = "2021" --- Cargo.toml | 9 +-- src/test_parse.rs | 179 ---------------------------------------------- 2 files changed, 5 insertions(+), 183 deletions(-) delete mode 100644 src/test_parse.rs diff --git a/Cargo.toml b/Cargo.toml index 8961bd0..6f29488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,19 +2,20 @@ name = "iotext" version = "0.1.0" rust-version = "1.57.0" +edition = "2021" [lib] name = "_iotext" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.0", features = ["auto-initialize", "extension-module"] } - +pyo3 = { version = "0.19.2", features = ["auto-initialize", "extension-module"] } +iotext_rs = { git = "https://github.com/bieli/IoText-rs.git" } # dependencies fancy-regex = "0.11.0" -regex = "1.8.3" +regex = "1.9.4" rustc-hash = "1.1.0" -bstr = "1.5.0" +bstr = "1.6.1" [profile.release] incremental = true diff --git a/src/test_parse.rs b/src/test_parse.rs deleted file mode 100644 index a00a325..0000000 --- a/src/test_parse.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::*; - -// COPY -> PASTE to https://play.rust-lang.org/ for check parser prototype - -//use std::intrinsics::type_name::*; -//use std::num::ParseIntError; -//use rust_decimal::Decimal; - -struct MetricDataTypes {} - -impl MetricDataTypes { - const INTEGER: &str = "i"; - const BOOL: &str = "b"; - //const DECIMAL: &str = "d"; - const TEXT: &str = "t"; -} - -struct ItemType {} -impl ItemType { - const TIME_UNIX_MILIS: &str = "t"; - const DEVICE_ID: &str = "d"; - const METRIC: &str = "m"; -} - -#[derive(Debug)] -enum MetricValueType { - IntegerItemType(i64), - BoolItemType(bool), - //DecimalItemType(Decimal), - TextItemType(String), -} - -#[derive(Debug)] -enum ItemTypeEnum { - TimeUnixMilis(u64), - DeviceId(String), -} - -const MSG_EXAMPLE: &str = "t|3900237526042,d|device_name_001,m|val_water_level1=i:42,m|light_on=b:1,m|bulb_on=b:0,m|msg_machine_01=t:hello,m|wind_speed=d:1234.5678"; - -fn main() { - let item_parts: Vec<&str> = MSG_EXAMPLE.split(',').collect(); - - for part in item_parts { - println!("part: {}", part); - let item_part: Vec<&str> = part.split('|').collect(); - println!("item_part: {:?}", item_part); - let item_type_tmp: &str = item_part[0]; - let item_type_metric: &str = "m"; - if item_type_tmp.eq(item_type_metric) { - println!("\tmetric: {}", item_part[1]); - let metric_parts: Vec<&str> = item_part[1].split('=').collect(); - println!("\tmetric_parts: {:?}", metric_parts); - //let metric_name: String = item_parts[0].to_string(); - //println!("metric_name: {}", metric_name); - let metric_parts_values: Vec<&str> = metric_parts[1].split(':').collect(); - println!("\t\tmetric_parts_values: {:?}", metric_parts_values); - match metric_parts_values[0] { - MetricDataTypes::INTEGER => { - let value = match metric_parts_values[1].parse::() { - Ok(number) => number, - Err(_) => todo!(), - }; - println!( - "\t\t\tIntegerItemType: {:?}", - MetricValueType::IntegerItemType(value) - ) - } - MetricDataTypes::BOOL => { - let value = match metric_parts_values[1] { - "1" => true, - "0" => false, - _ => todo!(), - }; - println!( - "\t\t\tBoolItemType: {:?}", - MetricValueType::BoolItemType(value) - ) - } - MetricDataTypes::TEXT => { - println!( - "\t\t\tBoolItemType: {:?}", - MetricValueType::TextItemType(metric_parts_values[1].to_string()) - ) - } - /* - MetricDataTypes::DECIMAL => { - println!( - "\t\t\tDecimalItemType: {:?}", - MetricValueType::DecimalItemType(metric_parts_values[1].to_string()) - ) - } - */ - _ => println!("\t\t\tother"), - } - } else { - match item_part[0] { - ItemType::TIME_UNIX_MILIS => { - let value = match item_part[1].parse::() { - Ok(number) => number, - Err(_) => todo!(), - }; - println!( - "\t\t\tTIME_UNIX_MILIS: {:?}", - ItemTypeEnum::TimeUnixMilis(value) - ) - } - ItemType::DEVICE_ID => { - println!( - "\t\t\tDEVICE_ID: {:?}", - ItemTypeEnum::DeviceId(String::from(item_part[1])) - ) - } - ItemType::METRIC => { - println!("\t\t\tMETRIC: {}", String::from(item_part[1])) - } - val => { - println!("\t\t\t OTHER: {:?}", val); - //print_type_of(val) - } - } - println!("\t\tcontext: {}", item_part[1]) - } - /*match item_part.split('=').collect() { - [item, details] => { - println!("{:?}", item); - println!("{:?}", details); - } - _ => println!("rest"), - }*/ - } - //assert_eq!(v, ["Mary", "had", "a", "little", "lamb"]); - //println!("{:?}", v); - //MetricValueType::IntegerItemType(0) -} - -/* -STDOUT: - - -part: t|3900237526042 -item_part: ["t", "3900237526042"] - TIME_UNIX_MILIS: TimeUnixMilis(3900237526042) - context: 3900237526042 -part: d|device_name_001 -item_part: ["d", "device_name_001"] - DEVICE_ID: DeviceId("device_name_001") - context: device_name_001 -part: m|val_water_level1=i:42 -item_part: ["m", "val_water_level1=i:42"] - metric: val_water_level1=i:42 - metric_parts: ["val_water_level1", "i:42"] - metric_parts_values: ["i", "42"] - IntegerItemType: IntegerItemType(42) -part: m|light_on=b:1 -item_part: ["m", "light_on=b:1"] - metric: light_on=b:1 - metric_parts: ["light_on", "b:1"] - metric_parts_values: ["b", "1"] - BoolItemType: BoolItemType(true) -part: m|bulb_on=b:0 -item_part: ["m", "bulb_on=b:0"] - metric: bulb_on=b:0 - metric_parts: ["bulb_on", "b:0"] - metric_parts_values: ["b", "0"] - BoolItemType: BoolItemType(false) -part: m|msg_machine_01=t:hello -item_part: ["m", "msg_machine_01=t:hello"] - metric: msg_machine_01=t:hello - metric_parts: ["msg_machine_01", "t:hello"] - metric_parts_values: ["t", "hello"] - BoolItemType: TextItemType("hello") -part: m|wind_speed=d:1234.5678 -item_part: ["m", "wind_speed=d:1234.5678"] - metric: wind_speed=d:1234.5678 - metric_parts: ["wind_speed", "d:1234.5678"] - metric_parts_values: ["d", "1234.5678"] - other -*/ From 7728bd0769b1e8d30cdddb29af21ff93fc61ee4a Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 30 Aug 2023 00:03:22 +0200 Subject: [PATCH 09/10] add files required to compile rust code and use as Python module - example rust lib, not required in this iteration --- .gitignore | 1 + Cargo.toml | 25 +++++++++++++++++-------- iotext/__init__.py | 5 +++++ noxfile.py | 18 ++++++++++++++++++ requirements-dev.txt | 1 + src/lib.rs | 29 +++++++++++++++++++++-------- 6 files changed, 63 insertions(+), 16 deletions(-) create mode 100644 iotext/__init__.py create mode 100644 noxfile.py diff --git a/.gitignore b/.gitignore index 2dc53ca..deb74ca 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,4 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ +.vscode/ \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 6f29488..1a16e09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,30 @@ [package] name = "iotext" +authors = ["Marcin Bielak "] version = "0.1.0" rust-version = "1.57.0" edition = "2021" [lib] -name = "_iotext" +name = "iotext" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.19.2", features = ["auto-initialize", "extension-module"] } +pyo3 = { version = "0.19.2", features = ["extension-module"] } iotext_rs = { git = "https://github.com/bieli/IoText-rs.git" } +rayon = "1.0.2" + +#[lib] +#name = "_iotext" +#crate-type = ["cdylib"] + +#pyo3 = { version = "0.19.2", features = ["auto-initialize", "extension-module"] } +#iotext_rs = { git = "https://github.com/bieli/IoText-rs.git" } # dependencies -fancy-regex = "0.11.0" -regex = "1.9.4" -rustc-hash = "1.1.0" -bstr = "1.6.1" +#fancy-regex = "0.11.0" +#regex = "1.9.4" +#rustc-hash = "1.1.0" +#bstr = "1.6.1" -[profile.release] -incremental = true +#[profile.release] +#incremental = true diff --git a/iotext/__init__.py b/iotext/__init__.py new file mode 100644 index 0000000..88a4067 --- /dev/null +++ b/iotext/__init__.py @@ -0,0 +1,5 @@ +from .iotext import search + +__all__ = [ + "search" +] diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 0000000..884a21f --- /dev/null +++ b/noxfile.py @@ -0,0 +1,18 @@ +import nox + +nox.options.sessions = ["test"] + + +@nox.session +def test(session): + session.install("-rrequirements-dev.txt") + session.install("maturin") + session.run_always("maturin", "develop") + session.run("pytest") + + +@nox.session +def bench(session): + session.install("-rrequirements-dev.txt") + session.install(".") + session.run("pytest", "--benchmark-enable") \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index a206bbd..21d2f40 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +nox mypy black==22.3.0 parameterized==0.9.0 diff --git a/src/lib.rs b/src/lib.rs index e285bb4..4f49b5f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,18 +2,31 @@ #![allow(clippy::borrow_deref_ref)] use pyo3::prelude::*; +use rayon::prelude::*; -/// Formats the sum of two numbers as string. +/// Searches for the word, parallelized by rayon #[pyfunction] -fn sum_as_string(a: usize, b: usize) -> PyResult { - Ok((a + b).to_string()) +fn search(contents: &str, needle: &str) -> usize { + contents + .par_lines() + .map(|line| count_line(line, needle)) + .sum() +} + +/// Count the occurrences of needle in line, case insensitive +fn count_line(line: &str, needle: &str) -> usize { + let mut total = 0; + for word in line.split(' ') { + if word == needle { + total += 1; + } + } + total } -/// A Python module implemented in Rust. The name of this function must match -/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to -/// import the module. #[pymodule] -fn string_sum(_py: Python<'_>, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; +fn iotext(_py: Python<'_>, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(search, m)?)?; + Ok(()) } \ No newline at end of file From feda63ff77eb31956392a6bc98cdbc87847e0648 Mon Sep 17 00:00:00 2001 From: Marcin Bielak Date: Wed, 30 Aug 2023 02:05:24 +0200 Subject: [PATCH 10/10] PyO3 library research: how to optimal build bridge between IoText-rs RUST module and Python --- iotext/__init__.py | 14 +++- noxfile.py | 2 +- src/lib.rs | 173 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/iotext/__init__.py b/iotext/__init__.py index 88a4067..29d5e75 100644 --- a/iotext/__init__.py +++ b/iotext/__init__.py @@ -1,5 +1,15 @@ -from .iotext import search +from .iotext import search, decode, return_myclass, \ + MyClass, return_myiotextclass, MyIoTextClass, length, \ + Person, give_me_a_person __all__ = [ - "search" + "search", + "decode", + "return_myclass", + "MyClass", + "return_myiotextclass", + "MyIoTextClass", + "length", + "Person", + "give_me_a_person" ] diff --git a/noxfile.py b/noxfile.py index 884a21f..39230d2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -15,4 +15,4 @@ def test(session): def bench(session): session.install("-rrequirements-dev.txt") session.install(".") - session.run("pytest", "--benchmark-enable") \ No newline at end of file + session.run("pytest", "--benchmark-enable") diff --git a/src/lib.rs b/src/lib.rs index 4f49b5f..d60fe98 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,9 +1,34 @@ // This check is new and seems buggy (possibly with PyO3 interaction) #![allow(clippy::borrow_deref_ref)] +extern crate pyo3; + +use std::collections::{HashSet, hash_map, HashMap}; + +use iotext_rs::IoTextDataRow; +// use iotext_rs::IoTextData; +// use iotext_rs::IoTextDataRow; +use pyo3::PyResult; +use pyo3::exceptions::PyTypeError; +// use pyo3::PyErr; use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyTuple, IntoPyDict}; +use pyo3::wrap_pyfunction; use rayon::prelude::*; + +#[pyfunction] +fn length(py: Python, obj: PyObject) -> PyResult { + if let Ok(s) = obj.extract::(py) { + return Ok(s.len().to_object(py)); + } + if let Ok(s) = obj.extract::>(py) { + return Ok(s.len().to_object(py)); + } + Err(PyTypeError::new_err("Not Supported")) +} + + /// Searches for the word, parallelized by rayon #[pyfunction] fn search(contents: &str, needle: &str) -> usize { @@ -13,6 +38,117 @@ fn search(contents: &str, needle: &str) -> usize { .sum() } +#[pyfunction] +fn decode(py: Python, obj: PyObject) -> PyResult { + if let Ok(s) = obj.extract::(py) { + return Ok(s.len().to_object(py)); + } + Err(PyTypeError::new_err("Not Supported")) +// let data_obj: IoTextDataRow = IoTextDataRow::default(); +// return PyObject(data_obj.parse_iotext_str(iot_ext_data_row)).into_py() +} + +//#[pyfunction] +//fn decode() -> PyResult { +// Ok(IoTextDataRow::default()) +//} + +//#[pyfunction] +//fn decode2() -> PyResult> { +// // let gil = Python::acquire_gil(); +// // let py = gil.python(); +// +// Py::new(py, IoTextDataRow::default()).into() +//} + +#[pyclass] +struct Nonzero { + value: i32, +} + +//#[pymethods] +//impl Nonzero { +// #[new] +// fn py_new(value: i32) -> PyResult { +// if value == 0 { +// Err(PyErr::new("cannot be zero")) +// } else { +// Ok(Nonzero { value: value }) +// } +// } +//} + +#[pyclass] +struct MyClass { + num: i32, +} + +#[pyfunction] +fn return_myclass() -> Py { + Python::with_gil(|py| Py::new(py, MyClass { num: 1 }).unwrap()) +} + + +#[pyclass] +#[derive(Default)] +struct MyIoTextClass { + value: IoTextDataRow, +} + +#[pymethods] +impl MyIoTextClass { + #[new] + fn new() -> Self { + MyIoTextClass { value: IoTextDataRow::default() } + } + + pub fn example_list(&mut self, py: Python) -> PyResult { + //let l: &PyList = PyList::empty(py); + let elements: Vec<&str> = vec!["a", "b", "c"]; + let l: &PyList = PyList::new(py, elements); + Ok(l.into()) + } + + //pub fn example_dict_1(&mut self, py: Python) -> PyResult { + //let l: &PyList = PyList::empty(py); + //let elements: HashMap<&str, &str> = (0..10).map(|i| (i.to_string(), i.to_string())).collect(); + //let l: &PyDict = PyDict::new(py); + // let key_vals: Vec<(&str, PyObject)> = vec![ + // ("num", 8.to_object(py)), ("str", "asd".to_object(py)) + // ]; + //let dict = key_vals.into_py_dict(py); + + // Ok(key_vals.into_py_dict(py)) + //} + + /// Formats the sum of two numbers as string. + pub fn get_result(&mut self, py: Python) -> PyResult> { + let mut result = HashMap::new(); + result.insert("name".to_string(), "kushal".to_string()); + result.insert("age".to_string(), "36".to_string()); + Ok(result) + } + + + //fn method1() -> PyResult<&PyDict> { + //} + + //#[getter] + //fn value(&self) -> PyResult { + // Ok(self.value) + //} +} + +#[pyfunction] +fn return_myiotextclass() -> Py { + Python::with_gil(|py| Py::new(py, MyIoTextClass { value: IoTextDataRow::default() }).unwrap()) +} + + + + + + /// Count the occurrences of needle in line, case insensitive fn count_line(line: &str, needle: &str) -> usize { let mut total = 0; @@ -24,9 +160,44 @@ fn count_line(line: &str, needle: &str) -> usize { total } +#[pyfunction] +// Returns a Person class, takes a dict with {"name": "age", "age": 100} format. +fn give_me_a_person(data: &PyDict) -> PyResult { + let name: String = data.get_item("name").unwrap().extract().unwrap(); + let age: i64 = data.get_item("age").unwrap().extract().unwrap(); + + let p: Person = Person::new(name, age); + Ok(p) +} + +#[pyclass] +#[derive(Debug)] +struct Person { + #[pyo3(get, set)] + name: String, + #[pyo3(get, set)] + age: i64, +} + +#[pymethods] +impl Person { + #[new] + fn new(name: String, age: i64) -> Self { + Person { name, age } + } +} + #[pymodule] fn iotext(_py: Python<'_>, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(search, m)?)?; - + m.add_function(wrap_pyfunction!(return_myclass, m)?)?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(return_myiotextclass, m)?)?; + m.add_class::()?; + m.add_wrapped(wrap_pyfunction!(length))?; + m.add_wrapped(wrap_pyfunction!(decode))?; + m.add_wrapped(wrap_pyfunction!(give_me_a_person))?; + m.add_class::()?; + Ok(()) } \ No newline at end of file