diff --git a/.gitignore b/.gitignore index 088ba6b..3a67993 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ # will have compiled files and executables /target/ +/ffi/target + # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..267825a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "nimbus_experiments" +version = "0.1.0" +authors = ["Tarik Eshaq "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +url = "2.1" +serde = { version = "1", features = ["rc"] } +serde_derive = "1" +serde_json = "1" +anyhow = "1.0" +rand = "0.7" +log = "0.4" +viaduct = { git = "https://github.com/mozilla/application-services" } +ffi-support = "0.4" +thiserror = "1" +rkv = "0.10" +lazy_static = "1.4" +uuid = { version = "0.8", features = ["serde", "v4"]} +prost = "0.6" + +[build-dependencies] +prost-build = { version = "0.6" } + +[lib] +name = "nimbus_experiments" +crate-type = ["lib"] + +[dev-dependencies] +viaduct-reqwest = { git = "https://github.com/mozilla/application-services" } + diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..bd19228 --- /dev/null +++ b/build.rs @@ -0,0 +1,7 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +fn main() { + prost_build::compile_protos(&["src/experiments_msg_types.proto"], &["src/"]).unwrap(); +} diff --git a/examples/experiment.rs b/examples/experiment.rs new file mode 100644 index 0000000..4c1db5a --- /dev/null +++ b/examples/experiment.rs @@ -0,0 +1,24 @@ +use anyhow::Result; +use nimbus_experiments::{AppContext, Experiments}; +fn main() -> Result<()> { + viaduct_reqwest::use_reqwest_backend(); + let exp = Experiments::new(AppContext::default(), "./mydb"); + let enrolled_exp = exp.get_enrolled_experiments(); + exp.get_experiments().iter().for_each(|e| { + print!( + "Experiment: \"{}\", Buckets: {} to {}, Branches: ", + e.id, e.buckets.count, e.buckets.start + ); + e.branches.iter().for_each(|b| print!(" \"{}\", ", b.name)); + println!() + }); + println!("You are in bucket: {}", exp.get_bucket()); + enrolled_exp.iter().for_each(|ee| { + println!( + "Enrolled in experiment \"{}\" in branch \"{}\"", + ee.get_id(), + ee.get_branch() + ) + }); + Ok(()) +} diff --git a/src/buckets.rs b/src/buckets.rs new file mode 100644 index 0000000..e8766d3 --- /dev/null +++ b/src/buckets.rs @@ -0,0 +1,9 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! This might be where the bucketing logic can go +//! It would be different from current experimentation tools +//! There is a namespacing concept to allow users to be in multiple +//! unrelated experiments at the same time. +//! Not implemented yet diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..d64a406 --- /dev/null +++ b/src/error.rs @@ -0,0 +1,29 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Not implemented yet!!! +//! This is purely boilerplate to communicate over the ffi +//! We should define real variants for our error and use proper +//! error propegation (we can use the `thiserror` crate for that) +use ffi_support::{ErrorCode, ExternError}; +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("Invalid")] + Invalid, +} + +pub type Result = std::result::Result; + +impl From for ExternError { + fn from(_: Error) -> ExternError { + let code = ErrorCode::new(1); + ExternError::new_error(code, "UNEXPECTED") + } +} + +impl Into for anyhow::Error { + fn into(self) -> Error { + Error::Invalid + } +} diff --git a/src/experiments.idl b/src/experiments.idl new file mode 100644 index 0000000..823f66f --- /dev/null +++ b/src/experiments.idl @@ -0,0 +1,9 @@ +# This is a test file for defining WebIDL for uniffi +# For the time being, it is not used for anything! +# However, if we use uniffi in the future, we could define +# The api here. (Unless uniffi changes to a non WebIDL way (looking at you proc-macros)) +namespace experiments {}; +interface Experiments { + constructor(); + void get_experiment_branch(); +}; \ No newline at end of file diff --git a/src/experiments_msg_types.proto b/src/experiments_msg_types.proto new file mode 100644 index 0000000..6af5140 --- /dev/null +++ b/src/experiments_msg_types.proto @@ -0,0 +1,24 @@ +syntax = "proto2"; + +// This kinda beats the purpose of using protobufs since we have one file here/ +// And a duplicate in the glean PR, but bear with me :) +// Eventually once we figure out the details of where each part lives, we'll merge the proto files +// into one + +package mozilla.telemetry.glean.protobuf; + +option java_package = "mozilla.telemtery.glean"; +option java_outer_classname = "MsgTypes"; +option swift_prefix = "MsgTypes_"; +option optimize_for = LITE_RUNTIME; + +message AppContext { + optional string app_id = 1; + optional string app_version = 2; + optional string locale_language = 3; + optional string locale_country = 4; + optional string device_manufacturer = 5; + optional string device_model = 6; + optional string region = 7; + optional string debug_tag = 8; +} \ No newline at end of file diff --git a/src/ffi.rs b/src/ffi.rs new file mode 100644 index 0000000..e12c7ff --- /dev/null +++ b/src/ffi.rs @@ -0,0 +1,69 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use std::os::raw::c_char; + +use super::{error::Result, msg_types, AppContext, Experiments}; +use ffi_support::{define_handle_map_deleter, ConcurrentHandleMap, ExternError, FfiStr}; + +lazy_static::lazy_static! { + static ref EXPERIMENTS: ConcurrentHandleMap = ConcurrentHandleMap::new(); +} + +#[no_mangle] +pub extern "C" fn experiments_new( + app_ctx: *const u8, + app_ctx_len: i32, + db_path: FfiStr<'_>, + error: &mut ExternError, +) -> u64 { + EXPERIMENTS.insert_with_result(error, || -> Result { + let app_ctx = unsafe { + from_protobuf_ptr::(app_ctx, app_ctx_len).unwrap() + }; // Todo: make the whole function unsafe and implement proper error handling in error.rs + log::info!("=================== Initializing experiments ========================"); + Ok(Experiments::new(app_ctx, db_path.as_str())) + }) +} + +#[no_mangle] +pub extern "C" fn experiments_get_branch( + handle: u64, + branch: FfiStr<'_>, + error: &mut ExternError, +) -> *mut c_char { + EXPERIMENTS.call_with_result(error, handle, |experiment| -> Result { + log::info!("==================== Getting branch ========================"); + let branch_name = experiment.get_experiment_branch(branch.as_str())?; + Ok(branch_name) + }) +} + +define_handle_map_deleter!(EXPERIMENTS, experiements_destroy); + +/// # Safety +/// data is a raw pointer to the protobuf data +/// get_buffer will return an error if the length is invalid, +/// or if the pointer is a null pointer +pub unsafe fn from_protobuf_ptr>( + data: *const u8, + len: i32, +) -> anyhow::Result { + let buffer = get_buffer(data, len)?; + let item: Result = prost::Message::decode(buffer); + item.map(|inner| inner.into()).map_err(|e| e.into()) +} + +unsafe fn get_buffer<'a>(data: *const u8, len: i32) -> anyhow::Result<&'a [u8]> { + match len { + len if len < 0 => anyhow::bail!("Invalid length"), + 0 => Ok(&[]), + _ => { + if data.is_null() { + anyhow::bail!("Null pointer") + } + Ok(std::slice::from_raw_parts(data, len as usize)) + } + } +} diff --git a/src/http_client.rs b/src/http_client.rs new file mode 100644 index 0000000..ee29efe --- /dev/null +++ b/src/http_client.rs @@ -0,0 +1,81 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! This is a simple Http client that uses viaduct to retrieve experiment data from the server +//! Currently configured to use Kinto and the old schema, although that would change once we start +//! Working on the real Nimbus schema. + +use super::Experiment; +use anyhow::Result; +use serde_derive::*; +use url::Url; +use viaduct::{status_codes, Request, Response}; + +// Making this a trait so that we can mock those later. +pub(crate) trait SettingsClient { + fn get_experiements_metadata(&self) -> Result; + fn get_experiments(&self) -> Result>; +} + +#[derive(Deserialize)] +struct RecordsResponse { + data: Vec, +} + +pub struct Client { + base_url: Url, + collection_name: String, + bucket_name: String, +} + +impl Client { + pub fn new(base_url: Url, collection_name: String, bucket_name: String) -> Self { + Self { + base_url, + collection_name, + bucket_name, + } + } + + fn make_request(&self, request: Request) -> Result { + let resp = request.send()?; + if resp.is_success() || resp.status == status_codes::NOT_MODIFIED { + Ok(resp) + } else { + anyhow::bail!("Error in request: {}", resp.text()) + } + } +} + +impl SettingsClient for Client { + fn get_experiements_metadata(&self) -> Result { + let path = format!( + "buckets/{}/collections/{}", + &self.bucket_name, &self.collection_name + ); + let url = self.base_url.join(&path)?; + let req = Request::get(url).header( + "User-Agent", + "Experiments Rust Component ", + )?; + let resp = self.make_request(req)?; + let res = serde_json::to_string(&resp.body)?; + Ok(res) + } + + fn get_experiments(&self) -> Result> { + let path = format!( + "buckets/{}/collections/{}/records", + &self.bucket_name, &self.collection_name + ); + let url = self.base_url.join(&path)?; + let req = Request::get(url).header( + "User-Agent", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0", + )?; + // TODO: Add authentication based on server requirements + let resp = self.make_request(req)?.json::()?; + Ok(resp.data) + } +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..7cc7a46 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,234 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! Experiments library that hopes to be cross-plateform. +//! Still a work in progress, but good enough for people to poke around + +mod buckets; +pub mod error; +pub mod ffi; +mod http_client; +mod persistence; + +use error::Result; +pub use ffi::{experiements_destroy, experiments_get_branch, experiments_new}; +use http_client::{Client, SettingsClient}; +use persistence::Database; +use persistence::PersistedData; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use serde_derive::*; +use std::convert::TryInto; +use std::path::Path; +use url::Url; +use uuid::Uuid; +pub use viaduct; + +const BASE_URL: &str = "https://kinto.dev.mozaws.net/v1/"; +const COLLECTION_NAME: &str = "messaging-collection"; +const BUCKET_NAME: &str = "main"; +const MAX_BUCKET_NO: u32 = 10000; + +// We'll probably end up doing what is done in A-S with regards to +// protobufs if we take this route... +// But for now, using the build.rs seems convenient +// ref: https://github.com/mozilla/application-services/tree/main/tools/protobuf-gen +pub mod msg_types { + include!(concat!( + env!("OUT_DIR"), + "/mozilla.telemetry.glean.protobuf.rs" + )); +} + +/// Experiments is the main struct representing the experiements state +/// It should hold all the information needed to communcate a specific user's +/// Experiementation status (note: This should have some type of uuid) +pub struct Experiments { + // Uuid not used yet, but we'll be using it later + #[allow(unused)] + uuid: Uuid, + #[allow(unused)] + app_ctx: AppContext, + experiments: Vec, + enrolled_experiments: Vec, + bucket_no: u32, +} + +impl Experiments { + /// A new experiments struct is created this is where some preprocessing happens + /// It should look for persisted state first and setup some type + /// Of interval retrieval from the server for any experiment updates (not implemented) + pub fn new>(app_ctx: AppContext, path: P) -> Self { + let database = Database::new(path).unwrap(); + let persisted_data = database.get("persisted").unwrap(); + if let Some(data) = persisted_data { + log::info!("Retrieving data from persisted state..."); + let persisted_data = serde_json::from_str::(&data).unwrap(); + return Self { + app_ctx, + uuid: persisted_data.uuid, + experiments: persisted_data.experiments, + enrolled_experiments: persisted_data.enrolled_experiments, + bucket_no: persisted_data.bucket_no, + }; + } + let http_client = Client::new( + Url::parse(BASE_URL).unwrap(), + COLLECTION_NAME.to_string(), + BUCKET_NAME.to_string(), + ); + let resp = http_client.get_experiments().unwrap(); + + let uuid = uuid::Uuid::new_v4(); + let bucket_no: u32 = + u32::from_be_bytes(uuid.as_bytes()[..4].try_into().unwrap()) % MAX_BUCKET_NO; + let mut num = StdRng::seed_from_u64(bucket_no as u64); + let enrolled_experiments = resp + .iter() + .filter_map(|e| { + let branch = num.gen::() % e.branches.len(); + if bucket_no > e.buckets.count && bucket_no < e.buckets.start { + Some(EnrolledExperiment { + id: e.id.clone(), + branch: e.branches[branch].name.clone(), + }) + } else { + None + } + }) + .collect::>(); + database + .put( + "persisted", + PersistedData { + app_ctx: app_ctx.clone(), + uuid, + bucket_no, + enrolled_experiments: enrolled_experiments.clone(), + experiments: resp.clone(), + }, + ) + .unwrap(); + Self { + app_ctx, + uuid, + experiments: resp, + bucket_no, + enrolled_experiments, + } + } + + /// Retrieves the branch the user is in, in the experiment. Errors if the user is not enrolled (This should be an option, but for ffi + test it errors) + pub fn get_experiment_branch(&self, exp_name: &str) -> Result { + self.enrolled_experiments + .iter() + .find(|e| e.id == exp_name) + .map(|e| e.branch.clone()) + .ok_or_else(|| anyhow::format_err!("No branch").into()) // Should be returning an option! But for now... + } + + pub fn get_enrolled_experiments(&self) -> &Vec { + &self.enrolled_experiments + } + + pub fn get_experiments(&self) -> &Vec { + &self.experiments + } + + pub fn get_bucket(&self) -> u32 { + self.bucket_no + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct EnrolledExperiment { + id: String, + branch: String, +} + +impl EnrolledExperiment { + pub fn get_id(&self) -> &String { + &self.id + } + + pub fn get_branch(&self) -> &String { + &self.branch + } +} + +// ============ Below are a bunch of types that gets serialized/deserialized and stored in our `Experiments` struct ============ +// ============ They currently follow the old schema, and need to be updated to match the new Nimbus schema ============ + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Experiment { + pub id: String, + pub description: String, + pub last_modified: u64, + pub schema_modified: Option, + pub buckets: Bucket, + pub branches: Vec, + #[serde(rename = "match")] + pub matcher: Matcher, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Bucket { + pub count: u32, + pub start: u32, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct Branch { + pub name: String, + ratio: u32, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct Matcher { + pub app_id: Option, + pub app_display_version: Option, + pub app_min_version: Option, // Do what AC does and have a VersionOptionString instead? + pub app_max_version: Option, //Dito + pub locale_language: Option, + pub locale_country: Option, + pub device_manufacturer: Option, + pub device_model: Option, + pub regions: Vec, + pub debug_tags: Vec, +} + +#[derive(Deserialize, Serialize, Debug, Clone, Default)] +pub struct AppContext { + pub app_id: Option, + pub app_version: Option, + pub locale_language: Option, + pub locale_country: Option, + pub device_manufacturer: Option, + pub device_model: Option, + pub region: Option, + pub debug_tag: Option, +} + +impl From for AppContext { + fn from(proto_ctx: msg_types::AppContext) -> Self { + Self { + app_id: proto_ctx.app_id, + app_version: proto_ctx.app_version, + locale_language: proto_ctx.locale_language, + locale_country: proto_ctx.locale_country, + device_manufacturer: proto_ctx.device_manufacturer, + device_model: proto_ctx.device_model, + region: proto_ctx.region, + debug_tag: proto_ctx.debug_tag, + } + } +} + +// No tests implemented just yet +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} diff --git a/src/persistence.rs b/src/persistence.rs new file mode 100644 index 0000000..a4703c9 --- /dev/null +++ b/src/persistence.rs @@ -0,0 +1,76 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +//! This is where the persistence logic might go. +//! An idea for what to use here might be [RKV](https://github.com/mozilla/rkv) +//! And that's what's used on this prototype, +//! Either ways, the solution implemented should work regardless of the platform +//! on the other side of the FFI. This means that this module might require the FFI to allow consumers +//! To pass in a path to a database, or somewhere in the file system that the state will be persisted +use anyhow::Result; +use rkv::{Rkv, SingleStore, StoreOptions}; +use serde_derive::*; +use std::fs; +use std::path::Path; + +pub struct Database { + rkv: Rkv, + + experiment_store: SingleStore, +} + +impl Database { + pub fn new>(path: P) -> Result { + let rkv = Self::open_rkv(path)?; + let experiment_store = rkv + .open_single("experiments", StoreOptions::create()) + .unwrap(); + Ok(Self { + rkv, + experiment_store, + }) + } + + fn open_rkv>(path: P) -> Result { + let path = std::path::Path::new(path.as_ref()).join("db"); + log::debug!("Database path: {:?}", path.display()); + fs::create_dir_all(&path)?; + + let rkv = Rkv::new(&path).unwrap(); // Rkv errors should impl std::error::Error :( TODO: Impl proper error handling in an error.rs that can propagate + log::info!("Database initialized"); + Ok(rkv) + } + + pub fn get(&self, key: &str) -> Result> { + let reader = self.rkv.read().unwrap(); + let val = self.experiment_store.get(&reader, key).unwrap(); + Ok(val.map(|v| { + if let rkv::Value::Json(val) = v { + val.to_string() + } else { + "".to_string() // BAD IDEA! Remove this! + } + })) + } + + pub fn put(&self, key: &str, persisted_data: PersistedData) -> Result<()> { + let mut writer = self.rkv.write().unwrap(); + let persisted_json = serde_json::to_string(&persisted_data).unwrap(); + self.experiment_store + .put(&mut writer, key, &rkv::Value::Json(&persisted_json)) + .unwrap(); + writer.commit().unwrap(); + Ok(()) + } +} + +use super::{AppContext, EnrolledExperiment, Experiment}; +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct PersistedData { + pub app_ctx: AppContext, + pub experiments: Vec, + pub enrolled_experiments: Vec, + pub bucket_no: u32, + pub uuid: uuid::Uuid, +}