Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .github/styles/config/vocabularies/TraceMachina/accept.txt
Original file line number Diff line number Diff line change
Expand Up @@ -289,3 +289,10 @@ gcc
[Tt]eardown
repo
buildbox
subgraph
genrule
hardcoded
shellexpand
pluggable
[Cc]allout
hostnames
4 changes: 4 additions & 0 deletions .vale.ini
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ TokenIgnores = ['"]\.\/[\w-]+\.mdx['"], (?<=\])\([^)]+\)
# Ignore filenames and directory names in tree visualizations.
BlockIgnores = (?s)(<FileTree>.*?</FileTree>)

# Mermaid diagram source (node ids, `subgraph`/`end` keywords) isn't prose —
# scanning it produces spurious Vale.Repetition hits on short node labels.
BlockIgnores = (?s)(<Mermaid>.*?</Mermaid>)

# Too harsh. The `write-good.Passive` check already covers many cases.
write-good.E-Prime = NO

Expand Down
172 changes: 172 additions & 0 deletions nativelink-config/examples/local_rbe_self_test.json5
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
// A complete NativeLink cluster — CAS, AC, scheduler, and one worker — in a
// single process on localhost. Everything a Bazel client needs (cache AND
// execution) is on one port; nothing here talks to any external service.
//
// Use this to prove your Bazel setup can do remote caching and remote
// execution before you ever point it at a real cluster. See
// docs/rbe/local-remote-execution for the full walkthrough.
//
// Run it:
// nativelink ./nativelink-config/examples/local_rbe_self_test.json5
//
// Then, from a Bazel workspace:
// bazel test --remote_cache=grpc://127.0.0.1:50051 \
// --remote_executor=grpc://127.0.0.1:50051 \
// --remote_default_exec_properties=cpu_count=1 \
// //some:target
{
stores: [
{
name: "CAS_MAIN_STORE",
filesystem: {
content_path: "/tmp/nativelink-local-rbe-test/data/content_path-cas",
temp_path: "/tmp/nativelink-local-rbe-test/data/tmp_path-cas",
eviction_policy: {
max_bytes: 2000000000,
},
},
},
{
name: "AC_MAIN_STORE",
filesystem: {
content_path: "/tmp/nativelink-local-rbe-test/data/content_path-ac",
temp_path: "/tmp/nativelink-local-rbe-test/data/tmp_path-ac",
eviction_policy: {
max_bytes: 200000000,
},
},
},
{
name: "WORKER_FAST_SLOW_STORE",
fast_slow: {
// "fast" must be a filesystem store — the worker hardlinks out of it
// to build each action's sandbox.
fast: {
filesystem: {
content_path: "/tmp/nativelink-local-rbe-test/data/content_path-worker",
temp_path: "/tmp/nativelink-local-rbe-test/data/tmp_path-worker",
eviction_policy: {
max_bytes: 2000000000,
},
},
},

// The worker and the client-facing CAS share one store instance, so
// whatever the worker produces is immediately visible to Bazel.
slow: {
ref_store: {
name: "CAS_MAIN_STORE",
},
},
},
},
],
schedulers: [
{
name: "MAIN_SCHEDULER",
simple: {
supported_platform_properties: {
cpu_count: "minimum",
OSFamily: "priority",
"container-image": "priority",
},
},
},
],
workers: [
{
local: {
worker_api_endpoint: {
uri: "grpc://127.0.0.1:50061",
},
cas_fast_slow_store: "WORKER_FAST_SLOW_STORE",
upload_action_result: {
ac_store: "AC_MAIN_STORE",
},
work_directory: "/tmp/nativelink-local-rbe-test/work",
platform_properties: {
cpu_count: {
values: [
"1",
],
},
OSFamily: {
values: [
"",
],
},
"container-image": {
values: [
"",
],
},
},

// use_namespaces / use_mount_namespace default to false, which is
// required here — they're Linux-only and this config is meant to
// also run on macOS.
},
},
],
servers: [
{
// Everything a Bazel client needs — cache AND execution — on one
// local endpoint. No separate/remote cache service involved.
name: "local",
listener: {
http: {
socket_address: "0.0.0.0:50051",
},
},
services: {
cas: [
{
cas_store: "CAS_MAIN_STORE",
},
],
ac: [
{
ac_store: "AC_MAIN_STORE",
},
],
bytestream: [
{
cas_store: "CAS_MAIN_STORE",
},
],
execution: [
{
cas_store: "CAS_MAIN_STORE",
scheduler: "MAIN_SCHEDULER",
},
],
capabilities: [
{
remote_execution: {
scheduler: "MAIN_SCHEDULER",
},
},
],
},
},
{
// Workers talk to this port, clients never do — keep it off the
// client-facing listener (see nativelink-config/examples/README.md).
name: "worker_api",
listener: {
http: {
socket_address: "0.0.0.0:50061",
},
},
services: {
worker_api: {
scheduler: "MAIN_SCHEDULER",
},
health: {},
},
},
],
global: {
max_open_files: 24576,
},
}
60 changes: 46 additions & 14 deletions nativelink-config/examples/stores-config.json5
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,38 @@
},
{
name: "6",
"experimental_cloud_object_store": {
"provider": "r2",
"account_id": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4",
"bucket": "nativelink-cas",
"key_prefix": "test-prefix/",
"retry": {
"max_retries": 6,
"delay": 0.3,
"jitter": 0.5
},
"multipart_max_concurrent_uploads": 10
}
},
{
name: "7",
"experimental_cloud_object_store": {
"provider": "oci",
"namespace": "your-object-storage-namespace",
"region": "us-phoenix-1",
"bucket": "nativelink-cas",
"access_key_id": "oci_access_key_id",
"secret_access_key": "oci_secret_access_key",
"key_prefix": "test-prefix/",
"retry": {
"max_retries": 6,
"delay": 0.3,
"jitter": 0.5
}
}
},
{
name: "8",
"ontap_s3_existence_cache": {
"index_path": "/path/to/cache/index.json",
"sync_interval_seconds": 300,
Expand All @@ -97,7 +129,7 @@
}
},
{
name: "7",
name: "9",
"verify": {
"backend": {
"memory": {
Expand All @@ -111,7 +143,7 @@
}
},
{
name: "8",
name: "10",
"completeness_checking": {
"backend": {
"filesystem": {
Expand All @@ -130,7 +162,7 @@
}
},
{
name: "9",
name: "11",
"compression": {
"compression_algorithm": {
"lz4": {}
Expand All @@ -147,7 +179,7 @@
}
},
{
name: "10",
name: "12",
"dedup": {
"index_store": {
"memory": {
Expand Down Expand Up @@ -186,7 +218,7 @@
}
},
{
name: "11",
name: "13",
"existence_cache": {
"backend": {
"memory": {
Expand All @@ -202,7 +234,7 @@
}
},
{
name: "12",
name: "14",
"fast_slow": {
"fast": {
"filesystem": {
Expand All @@ -225,7 +257,7 @@
}
},
{
name: "13",
name: "15",
"shard": {
"stores": [
{
Expand All @@ -241,7 +273,7 @@
}
},
{
name: "14",
name: "16",
"filesystem": {
"content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
Expand All @@ -251,13 +283,13 @@
}
},
{
name: "15",
name: "17",
"ref_store": {
"name": "FS_CONTENT_STORE"
}
},
{
name: "16",
name: "18",
"size_partitioning": {
"size": "128mib",
"lower_store": {
Expand All @@ -274,7 +306,7 @@
}
},
{
name: "17",
name: "19",
"grpc": {
"instance_name": "main",
"endpoints": [
Expand All @@ -295,7 +327,7 @@
}
},
{
name: "18",
name: "20",
"redis_store": {
"addresses": [
"redis://127.0.0.1:6379/",
Expand All @@ -304,11 +336,11 @@
}
},
{
name: "19",
name: "21",
"noop": {}
},
{
name: "20",
name: "22",
"experimental_mongo": {
"connection_string": "mongodb://localhost:27017",
"database": "nativelink",
Expand Down
Loading
Loading