text_tokenizer_preprocessor_fallback.md

text_tokenizer_preprocessor_fallback

Source: emel/text/tokenizer/preprocessor/fallback/sm.hpp

Mermaid

stateDiagram-v2
  direction TB
  [*] --> idle
  idle --> request_buffer_decision : preprocess_runtime [always] / none
  done --> request_buffer_decision : preprocess_runtime [always] / none
  errored --> request_buffer_decision : preprocess_runtime [always] / none
  unexpected --> request_buffer_decision : preprocess_runtime [always] / none
  request_buffer_decision --> request_capacity_nonzero_decision : completion_preprocess_runtime_ [fragments_buffer_present_] / none
  request_buffer_decision --> errored : completion_preprocess_runtime_ [fragments_buffer_missing_] / reject_invalid_
  request_buffer_decision --> errored : completion_preprocess_runtime_ [always] / reject_invalid_
  request_capacity_nonzero_decision --> request_capacity_limit_decision : completion_preprocess_runtime_ [fragments_capacity_nonzero_] / none
  request_capacity_nonzero_decision --> errored : completion_preprocess_runtime_ [fragments_capacity_zero_] / reject_invalid_
  request_capacity_nonzero_decision --> errored : completion_preprocess_runtime_ [always] / reject_invalid_
  request_capacity_limit_decision --> preparing : completion_preprocess_runtime_ [fragments_capacity_within_limit_] / begin_preprocess_
  request_capacity_limit_decision --> errored : completion_preprocess_runtime_ [fragments_capacity_exceeds_limit_] / reject_invalid_
  request_capacity_limit_decision --> errored : completion_preprocess_runtime_ [always] / reject_invalid_
  preparing --> build_specials_decision : completion_preprocess_runtime_ [always] / build_specials_
  build_specials_decision --> partition_specials_decision : completion_preprocess_runtime_ [build_specials_ok_] / none
  build_specials_decision --> errored : completion_preprocess_runtime_ [build_specials_invalid_request_error_] / ensure_last_error_
  build_specials_decision --> errored : completion_preprocess_runtime_ [build_specials_backend_error_] / ensure_last_error_
  build_specials_decision --> errored : completion_preprocess_runtime_ [build_specials_unknown_error_] / ensure_last_error_
  partition_specials_decision --> partitioning_no_specials_input_decision : completion_preprocess_runtime_ [no_specials_] / none
  partition_specials_decision --> partition_parse_special_decision : completion_preprocess_runtime_ [has_specials_] / none
  partition_specials_decision --> errored : completion_preprocess_runtime_ [always] / ensure_last_error_
  partition_parse_special_decision --> partitioning_non_bpe_parse_input_decision : completion_preprocess_runtime_ [parse_special_enabled_] / none
  partition_parse_special_decision --> partitioning_non_bpe_skip_input_decision : completion_preprocess_runtime_ [parse_special_disabled_] / none
  partition_parse_special_decision --> errored : completion_preprocess_runtime_ [always] / ensure_last_error_
  partitioning_no_specials_input_decision --> partition_decision : completion_preprocess_runtime_ [request_text_empty_] / set_empty_partition_result_
  partitioning_no_specials_input_decision --> partitioning_no_specials : completion_preprocess_runtime_ [request_text_nonempty_] / none
  partitioning_no_specials_input_decision --> errored : completion_preprocess_runtime_ [always] / ensure_last_error_
  partitioning_non_bpe_parse_input_decision --> partition_decision : completion_preprocess_runtime_ [request_text_empty_] / set_empty_partition_result_
  partitioning_non_bpe_parse_input_decision --> partitioning_non_bpe_parse_special : completion_preprocess_runtime_ [request_text_nonempty_] / none
  partitioning_non_bpe_parse_input_decision --> errored : completion_preprocess_runtime_ [always] / ensure_last_error_
  partitioning_non_bpe_skip_input_decision --> partition_decision : completion_preprocess_runtime_ [request_text_empty_] / set_empty_partition_result_
  partitioning_non_bpe_skip_input_decision --> partitioning_non_bpe_skip_special : completion_preprocess_runtime_ [request_text_nonempty_] / none
  partitioning_non_bpe_skip_input_decision --> errored : completion_preprocess_runtime_ [always] / ensure_last_error_
  partitioning_no_specials --> partition_decision : completion_preprocess_runtime_ [always] / partition_no_specials_
  partitioning_non_bpe_parse_special --> partition_decision : completion_preprocess_runtime_ [always] / partition_non_bpe_parse_special_
  partitioning_non_bpe_skip_special --> partition_decision : completion_preprocess_runtime_ [always] / partition_non_bpe_skip_special_
  partition_decision --> done : completion_preprocess_runtime_ [partition_ok_] / mark_done_
  partition_decision --> errored : completion_preprocess_runtime_ [partition_invalid_request_error_] / ensure_last_error_
  partition_decision --> errored : completion_preprocess_runtime_ [partition_backend_error_] / ensure_last_error_
  partition_decision --> errored : completion_preprocess_runtime_ [partition_unknown_error_] / ensure_last_error_
  idle --> unexpected : _ [always] / on_unexpected_
  request_buffer_decision --> unexpected : _ [always] / on_unexpected_
  request_capacity_nonzero_decision --> unexpected : _ [always] / on_unexpected_
  request_capacity_limit_decision --> unexpected : _ [always] / on_unexpected_
  preparing --> unexpected : _ [always] / on_unexpected_
  build_specials_decision --> unexpected : _ [always] / on_unexpected_
  partition_specials_decision --> unexpected : _ [always] / on_unexpected_
  partition_parse_special_decision --> unexpected : _ [always] / on_unexpected_
  partitioning_no_specials_input_decision --> unexpected : _ [always] / on_unexpected_
  partitioning_non_bpe_parse_input_decision --> unexpected : _ [always] / on_unexpected_
  partitioning_non_bpe_skip_input_decision --> unexpected : _ [always] / on_unexpected_
  partitioning_no_specials --> unexpected : _ [always] / on_unexpected_
  partitioning_non_bpe_parse_special --> unexpected : _ [always] / on_unexpected_
  partitioning_non_bpe_skip_special --> unexpected : _ [always] / on_unexpected_
  partition_decision --> unexpected : _ [always] / on_unexpected_
  done --> unexpected : _ [always] / on_unexpected_
  errored --> unexpected : _ [always] / on_unexpected_
  unexpected --> unexpected : _ [always] / on_unexpected_

Loading

Transitions

Source	Event	Guard	Action	Target
idle	preprocess_runtime	always	none	request_buffer_decision
done	preprocess_runtime	always	none	request_buffer_decision
errored	preprocess_runtime	always	none	request_buffer_decision
unexpected	preprocess_runtime	always	none	request_buffer_decision
request_buffer_decision	completion<preprocess_runtime>	fragments_buffer_present>	none	request_capacity_nonzero_decision
request_buffer_decision	completion<preprocess_runtime>	fragments_buffer_missing>	reject_invalid>	errored
request_buffer_decision	completion<preprocess_runtime>	always	reject_invalid>	errored
request_capacity_nonzero_decision	completion<preprocess_runtime>	fragments_capacity_nonzero>	none	request_capacity_limit_decision
request_capacity_nonzero_decision	completion<preprocess_runtime>	fragments_capacity_zero>	reject_invalid>	errored
request_capacity_nonzero_decision	completion<preprocess_runtime>	always	reject_invalid>	errored
request_capacity_limit_decision	completion<preprocess_runtime>	fragments_capacity_within_limit>	begin_preprocess>	preparing
request_capacity_limit_decision	completion<preprocess_runtime>	fragments_capacity_exceeds_limit>	reject_invalid>	errored
request_capacity_limit_decision	completion<preprocess_runtime>	always	reject_invalid>	errored
preparing	completion<preprocess_runtime>	always	build_specials>	build_specials_decision
build_specials_decision	completion<preprocess_runtime>	build_specials_ok>	none	partition_specials_decision
build_specials_decision	completion<preprocess_runtime>	build_specials_invalid_request_error>	ensure_last_error>	errored
build_specials_decision	completion<preprocess_runtime>	build_specials_backend_error>	ensure_last_error>	errored
build_specials_decision	completion<preprocess_runtime>	build_specials_unknown_error>	ensure_last_error>	errored
partition_specials_decision	completion<preprocess_runtime>	no_specials>	none	partitioning_no_specials_input_decision
partition_specials_decision	completion<preprocess_runtime>	has_specials>	none	partition_parse_special_decision
partition_specials_decision	completion<preprocess_runtime>	always	ensure_last_error>	errored
partition_parse_special_decision	completion<preprocess_runtime>	parse_special_enabled>	none	partitioning_non_bpe_parse_input_decision
partition_parse_special_decision	completion<preprocess_runtime>	parse_special_disabled>	none	partitioning_non_bpe_skip_input_decision
partition_parse_special_decision	completion<preprocess_runtime>	always	ensure_last_error>	errored
partitioning_no_specials_input_decision	completion<preprocess_runtime>	request_text_empty>	set_empty_partition_result>	partition_decision
partitioning_no_specials_input_decision	completion<preprocess_runtime>	request_text_nonempty>	none	partitioning_no_specials
partitioning_no_specials_input_decision	completion<preprocess_runtime>	always	ensure_last_error>	errored
partitioning_non_bpe_parse_input_decision	completion<preprocess_runtime>	request_text_empty>	set_empty_partition_result>	partition_decision
partitioning_non_bpe_parse_input_decision	completion<preprocess_runtime>	request_text_nonempty>	none	partitioning_non_bpe_parse_special
partitioning_non_bpe_parse_input_decision	completion<preprocess_runtime>	always	ensure_last_error>	errored
partitioning_non_bpe_skip_input_decision	completion<preprocess_runtime>	request_text_empty>	set_empty_partition_result>	partition_decision
partitioning_non_bpe_skip_input_decision	completion<preprocess_runtime>	request_text_nonempty>	none	partitioning_non_bpe_skip_special
partitioning_non_bpe_skip_input_decision	completion<preprocess_runtime>	always	ensure_last_error>	errored
partitioning_no_specials	completion<preprocess_runtime>	always	partition_no_specials>	partition_decision
partitioning_non_bpe_parse_special	completion<preprocess_runtime>	always	partition_non_bpe_parse_special>	partition_decision
partitioning_non_bpe_skip_special	completion<preprocess_runtime>	always	partition_non_bpe_skip_special>	partition_decision
partition_decision	completion<preprocess_runtime>	partition_ok>	mark_done>	done
partition_decision	completion<preprocess_runtime>	partition_invalid_request_error>	ensure_last_error>	errored
partition_decision	completion<preprocess_runtime>	partition_backend_error>	ensure_last_error>	errored
partition_decision	completion<preprocess_runtime>	partition_unknown_error>	ensure_last_error>	errored
idle	_	always	on_unexpected>	unexpected
request_buffer_decision	_	always	on_unexpected>	unexpected
request_capacity_nonzero_decision	_	always	on_unexpected>	unexpected
request_capacity_limit_decision	_	always	on_unexpected>	unexpected
preparing	_	always	on_unexpected>	unexpected
build_specials_decision	_	always	on_unexpected>	unexpected
partition_specials_decision	_	always	on_unexpected>	unexpected
partition_parse_special_decision	_	always	on_unexpected>	unexpected
partitioning_no_specials_input_decision	_	always	on_unexpected>	unexpected
partitioning_non_bpe_parse_input_decision	_	always	on_unexpected>	unexpected
partitioning_non_bpe_skip_input_decision	_	always	on_unexpected>	unexpected
partitioning_no_specials	_	always	on_unexpected>	unexpected
partitioning_non_bpe_parse_special	_	always	on_unexpected>	unexpected
partitioning_non_bpe_skip_special	_	always	on_unexpected>	unexpected
partition_decision	_	always	on_unexpected>	unexpected
done	_	always	on_unexpected>	unexpected
errored	_	always	on_unexpected>	unexpected
unexpected	_	always	on_unexpected>	unexpected