Stub generating fix

MANIFEST.in

@@ -1,4 +1,4 @@
-include snet/cli/resources/*
+recursive-include snet/cli/resources *
 
 recursive-exclude * .git
 recursive-exclude * node_modules

snet/cli/commands/mpe_channel.py

@@ -17,7 +17,8 @@
 from snet.cli.utils.token2cogs import cogs2strtoken
 from snet.cli.utils.ipfs_utils import get_from_ipfs_and_checkhash
 from snet.cli.utils.utils import abi_decode_struct_to_dict, abi_get_element_by_name, \
-    compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto
+    compile_proto, type_converter, bytesuri_to_hash, get_file_from_filecoin, download_and_safe_extract_proto, \
+    check_training_in_proto
 
 
 # we inherit MPEServiceCommand because we need _get_service_metadata_from_registry
@@ -602,9 +603,10 @@ def _init_or_update_service_if_needed(self, metadata, service_registration):
             os.makedirs(spec_dir, mode=0o700)
             service_api_source = metadata.get("service_api_source") or metadata.get("model_ipfs_hash")
             download_and_safe_extract_proto(service_api_source, spec_dir, self._get_ipfs_client())
+            training_added = check_training_in_proto(spec_dir)
 
             # compile .proto files
-            if not compile_proto(Path(spec_dir), service_dir):
+            if not compile_proto(Path(spec_dir), service_dir, add_training = training_added):
                 raise Exception("Fail to compile %s/*.proto" % spec_dir)
 
             # save service_metadata.json in channel_dir

snet/cli/commands/sdk_command.py

@@ -1,7 +1,7 @@
 import os
 from pathlib import Path, PurePath
 
-from snet.cli.utils.utils import compile_proto, download_and_safe_extract_proto
+from snet.cli.utils.utils import compile_proto, download_and_safe_extract_proto, check_training_in_proto
 from snet.cli.commands.mpe_service import MPEServiceCommand
 
 
@@ -28,8 +28,10 @@ def generate_client_library(self):
         # Receive proto files
         download_and_safe_extract_proto(service_api_source, library_dir_path, self._get_ipfs_client())
 
+        training_added = check_training_in_proto(library_dir_path)
+
         # Compile proto files
-        compile_proto(Path(library_dir_path), library_dir_path)
+        compile_proto(Path(library_dir_path), library_dir_path, add_training = training_added)
 
         self._printout(
             'client libraries for service with id "{}" in org with id "{}" generated at {}'.format(library_service_id,

snet/cli/resources/proto/training.proto

@@ -1,112 +1,128 @@
 syntax = "proto3";
-import "google/protobuf/descriptor.proto";
+import "google/protobuf/descriptor.proto";  // Required for indicators to work
 package training;
-option go_package = "../training";
-//Please note that the AI developers need to provide a server implementation of the gprc server of this proto.
-message ModelDetails {
-  //This Id will be generated when you invoke the create_model method and hence doesnt need to be filled when you
-  //invoke the create model
-  string model_id = 1;
-  //define the training method name
-  string grpc_method_name = 2;
-  //define the grpc service name , under which the method is defined
-  string grpc_service_name = 3;
-  string description = 4;
+option go_package = "github.com/singnet/snet-daemon/v5/training";
 
-  string status = 6;
-  string updated_date = 7;
-  //List of all the addresses that will have access to this model
-  repeated string address_list = 8;
-  // this is optional
-  string training_data_link = 9;
-  string model_name = 10;
+// Methods that the service provider must implement
+service Model {
 
+  // Free
+  // Can pass the address of the model creator
+  rpc create_model(NewModel) returns (ModelID) {}
 
-  string organization_id = 11;
-  string service_id = 12 ;
-  string group_id = 13;
+  // Free
+  rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {}
 
-  //set this to true if you want your model to be used by other AI consumers
-  bool is_publicly_accessible = 14;
+  // Paid
+  rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {}
 
-}
+  // Paid
+  rpc validate_model(ValidateRequest) returns (StatusResponse) {}
 
-message AuthorizationDetails {
-  uint64 current_block = 1;
-  //Signer can fill in any message here
-  string message = 2;
-  //signature of the following message:
-  //("user specified message", user_address, current_block_number)
-  bytes signature = 3;
-  string signer_address = 4;
+  // Free, one signature for both train_model_price & train_model methods
+  rpc train_model_price(ModelID) returns (PriceInBaseUnit) {}
 
-}
+  // Paid
+  rpc train_model(ModelID) returns (StatusResponse) {}
 
-enum Status {
-  CREATED = 0;
-  IN_PROGRESS = 1;
-  ERRORED = 2;
-  COMPLETED = 3;
-  DELETED = 4;
-}
+  // Free
+  rpc delete_model(ModelID) returns (StatusResponse) {
+    // After model deletion, the status becomes DELETED in etcd
+  }
 
-message CreateModelRequest {
-  AuthorizationDetails authorization = 1;
-  ModelDetails model_details = 2;
+  // Free
+  rpc get_model_status(ModelID) returns (StatusResponse) {}
 }
 
-//the signer address will get to know all the models associated with this address.
-message AccessibleModelsRequest {
-  string grpc_method_name = 1;
-  string grpc_service_name = 2;
-  AuthorizationDetails authorization = 3;
-}
+message ModelResponse {
+  string model_id = 1;
+  Status status = 2;
+  string created_date = 3;
+  string updated_date = 4;
+  string name = 5;
+  string description = 6;
+  string grpc_method_name = 7;
+  string grpc_service_name = 8;
 
-message AccessibleModelsResponse {
-  repeated ModelDetails list_of_models = 1;
-}
+  // List of all addresses that will have access to this model
+  repeated string address_list = 9;
+
+  // Access to the model is granted only for use and viewing
+  bool is_public = 10;
+
+  string training_data_link = 11;
 
-message ModelDetailsRequest {
-  ModelDetails model_details = 1 ;
-  AuthorizationDetails authorization = 2;
+  string created_by_address = 12;
+  string updated_by_address = 13;
 }
 
-//helps determine which service end point to call for model training
-//format is of type "packageName/serviceName/MethodName", Example :"/example_service.Calculator/estimate_add"
-//Daemon will invoke the model training end point , when the below method option is specified
-message TrainingMethodOption {
-  string trainingMethodIndicator = 1;
+// Used as input for new_model requests
+// The service provider decides whether to use these fields; returning model_id is mandatory
+message NewModel {
+  string name = 1;
+  string description = 2;
+  string grpc_method_name = 3;
+  string grpc_service_name = 4;
+
+  // List of all addresses that will have access to this model
+  repeated string address_list = 5;
+
+  // Set this to true if you want your model to be accessible by other AI consumers
+  bool is_public = 6;
+
+  // These parameters will be passed by the daemon
+  string organization_id = 7;
+  string service_id = 8;
+  string group_id = 9;
 }
 
-extend google.protobuf.MethodOptions {
-  TrainingMethodOption my_method_option = 9999197;
+// This structure must be used by the service provider
+message ModelID {
+  string model_id = 1;
 }
 
-message UpdateModelRequest {
-  ModelDetails update_model_details = 1 ;
-  AuthorizationDetails authorization = 2;
+// This structure must be used by the service provider
+// Used in the train_model_price method to get the training/validation price
+message PriceInBaseUnit {
+  uint64 price = 1; // cogs, weis, afet, aasi, etc.
 }
 
+enum Status {
+  CREATED = 0;
+  VALIDATING = 1;
+  VALIDATED = 2;
+  TRAINING = 3;
+  READY_TO_USE = 4; // After training is completed
+  ERRORED = 5;
+  DELETED = 6;
+}
 
-message ModelDetailsResponse {
+message StatusResponse {
   Status status = 1;
-  ModelDetails model_details = 2;
-
 }
 
-service Model {
-
-  // The AI developer needs to Implement this service and Daemon will call these
-  // There will be no cost borne by the consumer in calling these methods,
-  // Pricing will apply when you actually call the training methods defined.
-  // AI consumer will call all these methods
-  rpc create_model(CreateModelRequest) returns (ModelDetailsResponse) {}
-  rpc delete_model(UpdateModelRequest) returns (ModelDetailsResponse) {}
-  rpc get_model_status(ModelDetailsRequest) returns (ModelDetailsResponse) {}
-
-  // Daemon will implement , however the AI developer should skip implementing these and just provide dummy code.
-  rpc update_model_access(UpdateModelRequest) returns (ModelDetailsResponse) {}
-  rpc get_all_models(AccessibleModelsRequest) returns (AccessibleModelsResponse) {}
+message UploadInput {
+  string model_id = 1;
+  bytes data = 2;
+  string file_name = 3;
+  uint64 file_size = 4; // in bytes
+  uint64 batch_size = 5;
+  uint64 batch_number = 6;
+  uint64 batch_count = 7;
+}
 
+message ValidateRequest {
+  string model_id = 2;
+  string training_data_link = 3;
+}
 
-}
+extend google.protobuf.MethodOptions {
+  string default_model_id = 50001;
+  uint64 max_models_per_user = 50002; // max models per method & user
+  uint64 dataset_max_size_mb = 50003; // max size of dataset
+  uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset
+  uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset
+  string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3
+  string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar
+  string dataset_description = 50008; // additional free-form requirements
+}

snet/cli/resources/proto/training/training.proto

@@ -0,0 +1,128 @@
+syntax = "proto3";
+import "google/protobuf/descriptor.proto";  // Required for indicators to work
+package training;
+option go_package = "github.com/singnet/snet-daemon/v5/training";
+
+// Methods that the service provider must implement
+service Model {
+
+  // Free
+  // Can pass the address of the model creator
+  rpc create_model(NewModel) returns (ModelID) {}
+
+  // Free
+  rpc validate_model_price(ValidateRequest) returns (PriceInBaseUnit) {}
+
+  // Paid
+  rpc upload_and_validate(stream UploadInput) returns (StatusResponse) {}
+
+  // Paid
+  rpc validate_model(ValidateRequest) returns (StatusResponse) {}
+
+  // Free, one signature for both train_model_price & train_model methods
+  rpc train_model_price(ModelID) returns (PriceInBaseUnit) {}
+
+  // Paid
+  rpc train_model(ModelID) returns (StatusResponse) {}
+
+  // Free
+  rpc delete_model(ModelID) returns (StatusResponse) {
+    // After model deletion, the status becomes DELETED in etcd
+  }
+
+  // Free
+  rpc get_model_status(ModelID) returns (StatusResponse) {}
+}
+
+message ModelResponse {
+  string model_id = 1;
+  Status status = 2;
+  string created_date = 3;
+  string updated_date = 4;
+  string name = 5;
+  string description = 6;
+  string grpc_method_name = 7;
+  string grpc_service_name = 8;
+
+  // List of all addresses that will have access to this model
+  repeated string address_list = 9;
+
+  // Access to the model is granted only for use and viewing
+  bool is_public = 10;
+
+  string training_data_link = 11;
+
+  string created_by_address = 12;
+  string updated_by_address = 13;
+}
+
+// Used as input for new_model requests
+// The service provider decides whether to use these fields; returning model_id is mandatory
+message NewModel {
+  string name = 1;
+  string description = 2;
+  string grpc_method_name = 3;
+  string grpc_service_name = 4;
+
+  // List of all addresses that will have access to this model
+  repeated string address_list = 5;
+
+  // Set this to true if you want your model to be accessible by other AI consumers
+  bool is_public = 6;
+
+  // These parameters will be passed by the daemon
+  string organization_id = 7;
+  string service_id = 8;
+  string group_id = 9;
+}
+
+// This structure must be used by the service provider
+message ModelID {
+  string model_id = 1;
+}
+
+// This structure must be used by the service provider
+// Used in the train_model_price method to get the training/validation price
+message PriceInBaseUnit {
+  uint64 price = 1; // cogs, weis, afet, aasi, etc.
+}
+
+enum Status {
+  CREATED = 0;
+  VALIDATING = 1;
+  VALIDATED = 2;
+  TRAINING = 3;
+  READY_TO_USE = 4; // After training is completed
+  ERRORED = 5;
+  DELETED = 6;
+}
+
+message StatusResponse {
+  Status status = 1;
+}
+
+message UploadInput {
+  string model_id = 1;
+  bytes data = 2;
+  string file_name = 3;
+  uint64 file_size = 4; // in bytes
+  uint64 batch_size = 5;
+  uint64 batch_number = 6;
+  uint64 batch_count = 7;
+}
+
+message ValidateRequest {
+  string model_id = 2;
+  string training_data_link = 3;
+}
+
+extend google.protobuf.MethodOptions {
+  string default_model_id = 50001;
+  uint64 max_models_per_user = 50002; // max models per method & user
+  uint64 dataset_max_size_mb = 50003; // max size of dataset
+  uint64 dataset_max_count_files = 50004; // maximum number of files in the dataset
+  uint64 dataset_max_size_single_file_mb = 50005; // maximum size of a single file in the dataset
+  string dataset_files_type = 50006; // allowed files types in dataset. string with array or single value, example: jpg, png, mp3
+  string dataset_type = 50007; // string with array or single value, example: zip, tar.gz, tar
+  string dataset_description = 50008; // additional free-form requirements
+}