diff --git a/README.adoc b/README.adoc index 5b94bd2da8f..710837e1919 100644 --- a/README.adoc +++ b/README.adoc @@ -23,7 +23,8 @@ However, it should be used with caution due to its low-level operations which, i Here is a summary of the library's key features: -*Operating System Calls:* Chronicle-Core provides access to various system calls such as retrieving the process ID, checking the operating system, and obtaining the hostname, among others. +=== Operating System Calls +Chronicle-Core provides access to various system calls such as retrieving the process ID, checking the operating system, and obtaining the hostname, among others. [source,java] ---- @@ -34,11 +35,13 @@ String hostname = OS.getHostName(); See the section on <> -*JVM Access Methods:* To access platform specific features of the JVM. +=== JVM Access Methods +To access platform specific features of the JVM. See the section on <> -*Memory Mapped Files:* The library offers an interface for managing memory mapped files, which is useful for high-performance I/O operations. +=== Memory Mapped Files +The library offers an interface for managing memory mapped files, which is useful for high-performance I/O operations. [source,java] ---- @@ -48,9 +51,11 @@ OS.memory().writeLong(1024L, 0x1234567890ABCDEFL); OS.unmap(address, 64 << 10); ---- -*Deterministic Resource Management:* Chronicle-Core features components that can be closed or reference-counted, and released deterministically without waiting for garbage collection. +=== Deterministic Resource Management +Chronicle-Core features components that can be closed or reference-counted, and released deterministically without waiting for garbage collection. -*Closeable Resources:* Chronicle-Core provides an interface for managing closeable resources, which are open when created and can't be used once closed. +=== Closeable Resources +Chronicle-Core provides an interface for managing closeable resources, which are open when created and can't be used once closed. This helps in preventing resource leaks. [source,java] @@ -69,7 +74,8 @@ public class AbstractCloseableTest { } ---- -*Resource Reference Counting:* The library enables the use of reference counting for deterministic resource release. +=== Resource Reference Counting +The library enables the use of reference counting for deterministic resource release. A reference-counted resource can add reservations until it's closed. [source,java] @@ -90,7 +96,8 @@ public class AbstractReferenceCountedTest { See the section on <> -*Thread safety checks:* The library enables the use of thread safety checks for single-threaded components. +=== Thread safety checks +The library enables the use of thread safety checks for single-threaded components. See the section on <> This library also wraps up low level access to diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000000..ccebc1ccabf --- /dev/null +++ b/TODO.md @@ -0,0 +1,8 @@ +# Chronicle Core Follow-ups + +- [ ] CORE-SB-201: Replace Netty-style GC retry in `OS.map0` once segmented mapping lands (track under CORE-NF-P-201). +- [ ] CORE-SB-202: Rework `CpuCoolers.SERIALIZATION` to avoid `XMLDecoder` when the load harness is refactored (CORE-TEST-202). +- [ ] CORE-SB-203: Introduce injectable clock for `SystemTimeProvider` so static field can become final (CORE-TEST-203). +- [ ] CORE-PMD-301: Restore stricter Checkstyle/PMD coverage after legacy clean-up; revisit thresholds and rule set (CORE-DOC-301). +- [ ] CORE-COV-001: Lift JaCoCo gates from the interim 0.73/0.62 line/branch targets to the standard 0.80/0.70 once new tests land. +- [ ] CORE-PMD-302: Re-enable NullAssignment/AvoidInstantiatingObjectsInLoops once legacy reference-counting and interner refactors complete. diff --git a/pom.xml b/pom.xml index 52d44d212b7..7ee13851789 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ net.openhft java-parent-pom 1.27ea1 - + chronicle-core 2.27ea9-SNAPSHOT @@ -40,6 +40,15 @@ disabled openhft https://sonarcloud.io + 3.6.0 + 8.45.1 + 4.8.6.6 + 1.14.0 + 3.28.0 + 0.8.14 + 0.80 + 0.70 + 1.23ea6 4.9.0 @@ -328,6 +337,154 @@ + + code-review + + false + + + 0.73 + 0.62 + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + ${checkstyle.version} + + src/main/config/checkstyle.xml + true + true + warning + false + + + + com.puppycrawl.tools + checkstyle + ${puppycrawl.version} + + + net.openhft + chronicle-quality-rules + ${chronicle-quality-rules.version} + + + + + checkstyle + + check + + verify + + + + + com.github.spotbugs + spotbugs-maven-plugin + ${spotbugs.version} + + Max + true + src/main/config/spotbugs-exclude.xml + + + com.h3xstream.findsecbugs + findsecbugs-plugin + ${findsecbugs.version} + + + + + + com.h3xstream.findsecbugs + findsecbugs-plugin + ${findsecbugs.version} + + + + + spotbugs + + check + + verify + + + + + org.apache.maven.plugins + maven-pmd-plugin + ${maven-pmd-plugin.version} + + true + true + + src/main/config/pmd-ruleset.xml + + src/main/config/pmd-exclude.properties + + + + pmd + + check + + verify + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-maven-plugin.version} + + + prepare-agent + + prepare-agent + + + + report + + report + + verify + + + check + + check + + verify + + + + BUNDLE + + + LINE + COVEREDRATIO + ${jacoco.line.coverage} + + + BRANCH + COVEREDRATIO + ${jacoco.branch.coverage} + + + + + + + + + + + assertions diff --git a/src/main/config/checkstyle-suppressions.xml b/src/main/config/checkstyle-suppressions.xml new file mode 100644 index 00000000000..bc3dada338a --- /dev/null +++ b/src/main/config/checkstyle-suppressions.xml @@ -0,0 +1,6 @@ + + + + + diff --git a/src/main/config/checkstyle.xml b/src/main/config/checkstyle.xml new file mode 100644 index 00000000000..844dd904bc4 --- /dev/null +++ b/src/main/config/checkstyle.xml @@ -0,0 +1,210 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/main/config/pmd-exclude.properties b/src/main/config/pmd-exclude.properties new file mode 100644 index 00000000000..9cde124be68 --- /dev/null +++ b/src/main/config/pmd-exclude.properties @@ -0,0 +1,36 @@ +# PMD exclusions with justification. Format: filepath=RuleOne,RuleTwo +# CORE-PMD-201: JVM reflective and signal handling fallbacks swallow errors deliberately; revisit under CORE-RISK-201 (EmptyCatch, legacy loop allocations). +.*net/openhft/chronicle/core/Jvm.java$=EmptyCatchBlock,AvoidInstantiatingObjectsInLoops +# CORE-PMD-202: OS PID discovery and temp-dir helpers rely on defensive loops; cleanup tracked via CORE-OPS-202. +.*net/openhft/chronicle/core/OS.java$=EmptyCatchBlock,AvoidInstantiatingObjectsInLoops +# CORE-PMD-203: Annotation scanning tolerates ClassNotFound across modules; convert to structured logging under CORE-DOC-203. +.*net/openhft/chronicle/core/internal/AnnotationFinder.java$=EmptyCatchBlock +# CORE-PMD-204: Bootstrap capability probes rely on empty catches until module layering is simplified (CORE-NF-O-204). +.*net/openhft/chronicle/core/internal/Bootstrap.java$=EmptyCatchBlock +# CORE-PMD-205: PackageNameUtil suppresses ClassNotFound for optional modules; revisit in CORE-NF-O-205. +.*net/openhft/chronicle/core/internal/PackageNameUtil.java$=EmptyCatchBlock +# CORE-PMD-206: IOTools close helpers tolerate suppressed exceptions; redesign being tracked in CORE-OPS-206. +.*net/openhft/chronicle/core/io/IOTools.java$=EmptyCatchBlock +# CORE-PMD-207: ObjectUtils conversion heuristics intentionally swallow format errors; replace with explicit diagnostics under CORE-FN-207. +.*net/openhft/chronicle/core/util/ObjectUtils.java$=EmptyCatchBlock +# CORE-PMD-208: ConversionFunction handles primitive parsing fallbacks; revisit logging strategy in CORE-FN-208. +.*net/openhft/chronicle/core/util/ConversionFunction.java$=EmptyCatchBlock +# CORE-PMD-301: Reference counting cleanup pending; null sentinels retained for backwards compatibility (CORE-NF-O-301). +.*net/openhft/chronicle/core/io/AbstractCloseable.java$=NullAssignment +.*net/openhft/chronicle/core/io/AbstractCloseableReferenceCounted.java$=NullAssignment +.*net/openhft/chronicle/core/io/AbstractReferenceCounted.java$=NullAssignment +.*net/openhft/chronicle/core/io/UnsafeCloseable.java$=NullAssignment +.*net/openhft/chronicle/core/io/VanillaReferenceCounted.java$=NullAssignment +# CORE-PMD-302: Enum/scoped caches keep null sentinels for performance; revisit under CORE-FN-302. +.*net/openhft/chronicle/core/pool/DynamicEnumClass.java$=NullAssignment +.*net/openhft/chronicle/core/scoped/ScopedThreadLocal.java$=NullAssignment +.*net/openhft/chronicle/core/scoped/WeakReferenceScopedResource.java$=NullAssignment +.*net/openhft/chronicle/core/shutdown/PriorityHook.java$=NullAssignment +# CORE-PMD-303: Interner loops intentionally allocate keys during lookups; restructure planned under CORE-FN-303. +.*net/openhft/chronicle/core/pool/ClassAliasPool.java$=AvoidInstantiatingObjectsInLoops +# CORE-PMD-304: Thread dumps build diagnostic snapshots allocating per iteration; revisit under CORE-OPS-304. +.*net/openhft/chronicle/core/threads/ThreadDump.java$=AvoidInstantiatingObjectsInLoops +# CORE-PMD-305: WeakIdentityHashMap uses null resets to clear buckets; replaced map implementation tracked in CORE-FN-305. +.*net/openhft/chronicle/core/util/WeakIdentityHashMap.java$=NullAssignment +# CORE-PMD-306: CloseableUtils relies on looped resource probing; queued for refactor under CORE-OPS-306. +.*net/openhft/chronicle/core/internal/CloseableUtils.java$=AvoidInstantiatingObjectsInLoops diff --git a/src/main/config/pmd-ruleset.xml b/src/main/config/pmd-ruleset.xml new file mode 100644 index 00000000000..ba98297f72d --- /dev/null +++ b/src/main/config/pmd-ruleset.xml @@ -0,0 +1,14 @@ + + + + Baseline Chronicle rule selection used by the code-review profile. Additional rules (NullAssignment, AvoidInstantiatingObjectsInLoops) remain tracked in TODO for follow-up once legacy cleanup lands. + + + + + + + diff --git a/src/main/config/spotbugs-exclude.xml b/src/main/config/spotbugs-exclude.xml new file mode 100644 index 00000000000..1cabab5371e --- /dev/null +++ b/src/main/config/spotbugs-exclude.xml @@ -0,0 +1,235 @@ + + + + + + + + CORE-SB-201: GC retry path required until segmented mapping lands (CORE-NF-P-201). + + + + + + + + CORE-SB-202: XmlDecoder exercise harness; safe harness tracked under CORE-TEST-202. + + + + + + + + CORE-SB-203: Mutable clock retained for deterministic tests (CORE-TEST-203). + + + + + + + + + + + CORE-SB-301: JVM command helpers accept trusted inputs; sanitisation backlog CORE-NF-S-301. + + + + + CORE-SB-301: JVM anonymous helper warnings tolerated until reflection strategy update (CORE-NF-S-301). + + + + + + + + CORE-SB-302: Default OS helpers rely on shell commands; hardening in CORE-NF-S-302. + + + + + + + + + + + + + CORE-SB-302: Default hostname fallback allows plaintext within trusted host boundary; revisit under CORE-NF-S-302. + + + + + CORE-SB-303: Reference counting exposes state for diagnostics; refactor tracked in CORE-FN-303. + + + + + + + CORE-SB-303: TracingReferenceCounted remains mutable for logging until CORE-FN-303. + + + + + CORE-SB-303: Legacy reference counter field retained for debug metrics (CORE-FN-303). + + + + + CORE-SB-303: UnsafeCloseable null-check helpers preserved for backwards compatibility (CORE-FN-303). + + + + + CORE-SB-302: CleaningRandomAccessFile temp lookup tolerated while filesystem helper redesigns under CORE-OPS-302. + + + + + CORE-SB-302: IOTools suppression covers legacy path/URL handling; rewrite logged under CORE-OPS-302. + + + + + + + + + + CORE-SB-302: Language detection still uses plain sockets for localhost probes (CORE-OPS-302). + + + + + CORE-SB-302: Wget default stream uses URL.openStream until refactor (CORE-OPS-302). + + + + + CORE-SB-302: Builder SSRF guard covered by public-endpoint opt-in; cleanup tracked under CORE-OPS-302. + + + + + CORE-SB-304: Histogram exposes counters for monitoring; restructure under CORE-FN-304. + + + + + CORE-SB-304: RecordingHistogram equality/field use deferred to CORE-FN-304. + + + + + + + CORE-SB-305: StringUtils preserves historical allocation patterns (CORE-FN-305). + + + + + + + + CORE-SB-306: Pool/thread constructors throw during validation; safe alternative tracked under CORE-NF-O-306. + + + + + + + CORE-SB-306: CancellableTimer exposes event loop intentionally for diagnostics (CORE-NF-O-306). + + + + + + + CORE-SB-306: InvalidEventHandlerException exposes reusable instance until redesigned (CORE-NF-O-306). + + + + + CORE-SB-306: PriorityHook legacy singleton semantics slated for replacement (CORE-NF-O-306). + + + + + CORE-SB-307: CloseableUtils retains null-handling loop until IO helper rewrite (CORE-OPS-306). + + + + + + + + CORE-SB-307: CpuClass Windows/macOS command parsing persists until platform abstraction redo (CORE-NF-S-302). + + + + + + + + + + CORE-SB-307: UnsafeMemory floating division comparison pending MemorySegment migration (CORE-FN-307). + + + + + CORE-SB-307: ChainedExceptionHandler exposes delegate array for compatibility (CORE-FN-303). + + + + + CORE-SB-307: ExceptionKey leaks message in handler map; remediation under CORE-NF-S-307. + + + + + CORE-SB-307: RecordingExceptionHandler retains captured array for diagnostics (CORE-FN-303). + + + + + CORE-SB-302: Dynamic enum exposes array for performance; revisit in CORE-FN-302. + + + + + CORE-SB-302: Static enum interner exposes map for legacy caller contract (CORE-FN-302). + + + + + CORE-SB-306: ParsingCache constructor throws on unsupported types; redesign planned under CORE-FN-302. + + + + + CORE-SB-302: String interner constructor failure path remains for now (CORE-FN-302). + + + + + CORE-SB-306: VanillaReferenceOwner constructor throws for legacy compatibility (CORE-NF-O-306). + + + + + CORE-SB-306: TypeOf constructor guard retained until reflection refactor (CORE-FN-306). + + + + + + + + + + diff --git a/src/main/docs/common-pitfalls-and-debugging.adoc b/src/main/docs/common-pitfalls-and-debugging.adoc index 7282355a3e1..d8f1216f1b3 100644 --- a/src/main/docs/common-pitfalls-and-debugging.adoc +++ b/src/main/docs/common-pitfalls-and-debugging.adoc @@ -11,103 +11,144 @@ Chronicle Core heavily relies on explicit resource management, especially for of === Frequent Off-Heap Mistakes -* **Failing to `release()` reference-counted objects**: -* **Pitfall**: Forgetting to call `release(owner)` or `releaseLast(owner)` on `ReferenceCounted` objects (like those extending `AbstractReferenceCounted`) leads to native memory leaks, as the underlying resources are not freed until the reference count drops to zero. -* **Debugging**: Enable resource tracing (`-Djvm.resource.tracing=true`).Use `AbstractReferenceCounted.assertReferencesReleased()` in tests. `TracingReferenceCounted` can provide detailed information about who reserved and released a resource. -* **Relying on `warnAndCloseIfNotClosed` or `warnAndReleaseIfNotReleased`**: -* **Pitfall**: These methods (`AbstractCloseable.warnAndCloseIfNotClosed()`, `ReferenceCountedTracer.warnAndReleaseIfNotReleased()`) are safety nets, often called from finalizers or test cleanups. +Failing to `release()` reference-counted objects :: leading to a memory leak +Pitfall :: +Forgetting to call `release(owner)` or `releaseLast(owner)` on `ReferenceCounted` objects (like those extending `AbstractReferenceCounted`) leads to native memory leaks, as the underlying resources are not freed until the reference count drops to zero. +Debugging :: +Enable resource tracing (`-Djvm.resource.tracing=true`).Use `AbstractReferenceCounted.assertReferencesReleased()` in tests. +`TracingReferenceCounted` can provide detailed information about who reserved and released a resource. +Relying on `warnAndCloseIfNotClosed` or `warnAndReleaseIfNotReleased` :: +Pitfall :: +These methods (`AbstractCloseable.warnAndCloseIfNotClosed()`, `ReferenceCountedTracer.warnAndReleaseIfNotReleased()`) are safety nets, often called from finalizers or test cleanups. Relying on them for normal resource cleanup in application logic indicates a design flaw and can lead to unpredictable resource release. -* **Debugging**: Heed the warnings these methods produce. +Debugging :: +Heed the warnings these methods produce. The goal is to have zero such warnings by ensuring explicit and timely `close()` or `release()` calls. -* **Forgetting to `unmap()` a memory-mapped region**: -* **Pitfall**: Memory-mapped regions created with `OS.map()` must be explicitly unmapped using `OS.unmap()`. +Forgetting to `unmap()` a memory-mapped region :: +Pitfall :: +Memory-mapped regions created with `OS.map()` must be explicitly unmapped using `OS.unmap()`. Failure to do so can lead to `ENFILE` (too many open files) errors or exhaustion of virtual memory. -* **Debugging**: Track `OS.memoryMapped()` values. +Debugging :: +Track `OS.memoryMapped()` values. Ensure `unmap` is called in `finally` blocks or via try-with-resources if applicable. -* **Over-allocating direct buffers and relying on GC**: -* **Pitfall**: While `ByteBuffer.allocateDirect()` allocates off-heap, its cleanup relies on a `Cleaner` and garbage collection, which can be unpredictable and lead to `OutOfMemoryError` for direct memory. -* **Solution**: Manage direct buffers explicitly or use Chronicle Core's memory management which provides more control. -* **Misusing `CleanerServiceLocator` or `ByteBufferCleanerService`**: -* **Pitfall**: Forcibly cleaning ByteBuffers using these services without understanding the lifecycle of the associated resource can lead to crashes if the memory is still in use or accessed after cleaning. -* **Best Practice**: Prefer Chronicle Core's `ReferenceCounted` mechanisms for resources wrapping ByteBuffers. -* **Misconfiguring `BackgroundResourceReleaser`**: -* **Pitfall**: If `background.releaser.thread=false` is set, pending resources are queued but not automatically released. `BackgroundResourceReleaser.releasePendingResources()` must be called explicitly by a user thread. -* **Pitfall**: If `background.releaser=false`, releases are synchronous, potentially impacting performance-sensitive paths. -* **Debugging**: Monitor the `background~resource~releaser` thread activity and the queue size if issues are suspected. -* **Not using `CleaningRandomAccessFile`**: -* **Pitfall**: Standard `java.io.RandomAccessFile` might not reliably release its file descriptor on GC if not explicitly closed, leading to resource leaks. -* **Solution**: Use `net.openhft.chronicle.core.io.CleaningRandomAccessFile` which attempts to close the file in its finalizer. +Over-allocating direct buffers and relying on GC :: +Pitfall :: +While `ByteBuffer.allocateDirect()` allocates off-heap, its cleanup relies on a `Cleaner` and garbage collection, which can be unpredictable and lead to `OutOfMemoryError` for direct memory. +Solution :: +Manage direct buffers explicitly or use Chronicle Core's memory management which provides more control. +Misusing `CleanerServiceLocator` or `ByteBufferCleanerService` :: +Pitfall :: +Forcibly cleaning ByteBuffers using these services without understanding the lifecycle of the associated resource can lead to crashes if the memory is still in use or accessed after cleaning. +Best Practice :: +Prefer Chronicle Core's `ReferenceCounted` mechanisms for resources wrapping ByteBuffers. +Misconfiguring `BackgroundResourceReleaser` :: +Pitfall :: +If `background.releaser.thread=false` is set, pending resources are queued but not automatically released. +`BackgroundResourceReleaser.releasePendingResources()` must be called explicitly by a user thread. +Pitfall :: +If `background.releaser=false`, releases are synchronous, potentially impacting performance-sensitive paths. +Debugging :: +Monitor the `background~resource~releaser` thread activity and the queue size if issues are suspected. +Not using `CleaningRandomAccessFile` :: +Pitfall :: +Standard `java.io.RandomAccessFile` might not reliably release its file descriptor on GC if not explicitly closed, leading to resource leaks. +Solution :: +Use `net.openhft.chronicle.core.io.CleaningRandomAccessFile` which attempts to close the file in its finalizer. However, explicit closing is always best. === Diagnosing `ClosedIllegalStateException` A `ClosedIllegalStateException` signals that a resource was used after it was closed or released. -* **Enable Resource Tracing**: Set the system property `jvm.resource.tracing=true`. +Enable Resource Tracing :: +Set the system property `jvm.resource.tracing=true`. This typically enables `AbstractCloseable` to capture `createdHere` and `closedHere` stack traces. -* **Inspect Stack Traces**: When the exception occurs, the `closedHere` (and sometimes `createdHere`) stack trace will be available, often as a suppressed exception or via `getCause()`. +Inspect Stack Traces :: +When the exception occurs, the `closedHere` (and sometimes `createdHere`) stack trace will be available, often as a suppressed exception or via `getCause()`. This helps identify where the resource was closed and where it was created, making it easier to find the use-after-close bug. == Threading and Concurrency -* **`ThreadingIllegalStateException`**: -* **Cause**: Thrown when a component marked for single-threaded access (e.g., implementing `SingleThreadedChecked`) is accessed by multiple threads without proper hand-off. -* **Debugging**: The exception usually includes the stack trace of where the object was first used by a different thread (`usedByThreadHere` in `AbstractCloseable`). +`ThreadingIllegalStateException` :: +Cause :: +Thrown when a component marked for single-threaded access (e.g., implementing `SingleThreadedChecked`) is accessed by multiple threads without proper hand-off. +Debugging :: +The exception usually includes the stack trace of where the object was first used by a different thread (`usedByThreadHere` in `AbstractCloseable`). Ensure objects are not inadvertently shared or use `singleThreadedCheckReset()` after initialization on one thread before handing it off to another. -* **Memory Visibility and Ordering**: -* **Pitfall**: Using plain memory reads/writes (e.g., `UnsafeMemory.writeInt()`) for fields shared between threads without proper volatile semantics (`UnsafeMemory.writeVolatileInt()`, `UnsafeMemory.writeOrderedInt()`) or fences (`storeFence()`, `loadFence()`) can lead to stale data or incorrect instruction reordering. -* **Debugging**: Carefully review shared memory access patterns. +Memory Visibility and Ordering :: +Pitfall :: +Using plain memory reads/writes (e.g., `UnsafeMemory.writeInt()`) for fields shared between threads without proper volatile semantics (`UnsafeMemory.writeVolatileInt()`, `UnsafeMemory.writeOrderedInt()`) or fences (`storeFence()`, `loadFence()`) can lead to stale data or incorrect instruction reordering. +Debugging :: +Carefully review shared memory access patterns. Use JOL (Java Object Layout) to understand memory layouts. Consider concurrency testing tools. -* **`UniqueMicroTimeProvider` and Testing**: -* **Pitfall**: When testing code that uses `UniqueMicroTimeProvider`, if you replace its underlying provider with a `SetTimeProvider` for deterministic testing, ensure the `SetTimeProvider`'s clock is also controlled (e.g., frozen using `autoIncrement(0, NANOSECONDS)`) to avoid unexpected time jumps during the CAS loop. +`UniqueMicroTimeProvider` and Testing :: +Pitfall :: +When testing code that uses `UniqueMicroTimeProvider`, if you replace its underlying provider with a `SetTimeProvider` for deterministic testing, ensure the `SetTimeProvider`'s clock is also controlled (e.g., frozen using `autoIncrement(0, NANOSECONDS)`) to avoid unexpected time jumps during the CAS loop. == Low-Level Memory Operations -* **Misaligned Access**: -* **Pitfall**: Accessing multi-byte types (short, int, long, float, double) at memory addresses that are not aligned to their natural boundaries can cause `MisAlignedAssertionError` on some architectures (especially ARM) or lead to performance penalties. -* **Solution**: Ensure addresses and offsets are properly aligned. +Misaligned Access :: +Pitfall :: +Accessing multi-byte types (short, int, long, float, double) at memory addresses that are not aligned to their natural boundaries can cause `MisAlignedAssertionError` on some architectures (especially ARM) or lead to performance penalties. +Solution :: +Ensure addresses and offsets are properly aligned. Use `UnsafeMemory.safeAlignedInt()` or `UnsafeMemory.safeAlignedLong()` to check alignment if unsure, though these checks are themselves specific to certain implementations (like ARM). -* **Bounds Checking**: -* **Pitfall**: Direct memory access (e.g., via `OS.memory()` or `UnsafeMemory`) bypasses Java's array bounds checks. +Bounds Checking :: +Pitfall :: +Direct memory access (e.g., via `OS.memory()` or `UnsafeMemory`) bypasses Java's array bounds checks. Incorrect offset or length calculations can lead to reading/writing unintended memory, causing data corruption or JVM crashes. -* **Debugging**: Rigorous testing, assertions, and careful review of offset/length calculations are crucial. +Debugging :: +Rigorous testing, assertions, and careful review of offset/length calculations are crucial. Consider using boundary condition tests. == System Properties and Initialization -* **Properties Not Taking Effect**: -* **Pitfall**: System properties intended to configure Chronicle Core might not have the desired effect if they are set *after* the relevant Chronicle Core classes (like `Jvm.java`) have been initialized. -* **Solution**: Set system properties via the command line (`-Dproperty=value`) or use the `ChronicleInit` mechanism (`chronicle.init.runnable`) to programmatically set properties before Chronicle's static initializers run. -* **Boolean Property Interpretation**: -* **Pitfall**: Misunderstanding how `Jvm.getBoolean(String)` works. +Properties Not Taking Effect :: +Pitfall :: +System properties intended to configure Chronicle Core might not have the desired effect if they are set _after_ the relevant Chronicle Core classes (like `Jvm.java`) have been initialized. +Solution :: +Set system properties via the command line (`-Dproperty=value`) or use the `ChronicleInit` mechanism (`chronicle.init.runnable`) to programmatically set properties before Chronicle's static initializers run. +Boolean Property Interpretation :: +Pitfall :: +Misunderstanding how `Jvm.getBoolean(String)` works. It typically interprets the absence of a property as `false` (or the specified default), and the presence of the property (e.g., `-Dmy.flag`) or values like "true" or "yes" as `true`. -* **Debugging**: Log the actual property values at startup using `Jvm.getProperty()` to verify. +Debugging :: +Log the actual property values at startup using `Jvm.getProperty()` to verify. == Serialization and Data Integrity -* **`InvalidMarshallableException`**: -* **Cause**: Thrown when an object implementing `Validatable` fails its `validate()` method, typically before marshalling. -* **Debugging**: The exception message should indicate the validation rule that failed. +`InvalidMarshallableException` :: +Cause :: +Thrown when an object implementing `Validatable` fails its `validate()` method, typically before marshalling. +Debugging :: +The exception message should indicate the validation rule that failed. Ensure objects are in a consistent and complete state before attempting to serialize them. Implement comprehensive `validate()` methods. == Loop Block Monitor The Loop Block Monitor helps identify threads that might be stuck or performing unexpectedly long operations within a loop. -* **Usage**: Run `net.openhft.chronicle.core.threads.MonitorProfileAnalyserMain` with appropriate arguments to analyze thread dumps and summarize stack traces, highlighting potential bottlenecks. +Usage :: +Run `net.openhft.chronicle.core.threads.MonitorProfileAnalyserMain` with appropriate arguments to analyze thread dumps and summarize stack traces, highlighting potential bottlenecks. == Performance Analysis Tips -* **Targeted Profiling**: Capture short bursts of profiling data using standard Java profilers (like JFR, async-profiler) rather than overly long traces which can be hard to analyze. -* **`Histogram` Utility**: Use `net.openhft.chronicle.core.util.Histogram` or `RecordingHistogram` to record event latencies, especially for low-latency paths. +Targeted Profiling :: +Capture short bursts of profiling data using standard Java profilers (like JFR, async-profiler) rather than overly long traces which can be hard to analyze. +`Histogram` Utility :: +Use `net.openhft.chronicle.core.util.Histogram` or `RecordingHistogram` to record event latencies, especially for low-latency paths. This helps in understanding the distribution of latencies and spotting outliers. -* **Off-Heap Memory Usage**: Track off-heap memory usage with `Jvm.usedDirectMemory()` (for `ByteBuffer.allocateDirect`) and `Jvm.usedNativeMemory()` (for `UnsafeMemory.allocate()`). +Off-Heap Memory Usage :: +Track off-heap memory usage with `Jvm.usedDirectMemory()` (for `ByteBuffer.allocateDirect`) and `Jvm.usedNativeMemory()` (for `UnsafeMemory.allocate()`). Compare these values against expected levels to detect potential leaks or excessive usage. -* **`StackTrace` Overhead**: Be mindful that creating `StackTrace` objects frequently in performance-critical code can be expensive. +`StackTrace` Overhead :: +Be mindful that creating `StackTrace` objects frequently in performance-critical code can be expensive. Use it judiciously for debugging or when explicitly enabled. -* **Caching Utilities**: For `StringInterner` and `ParsingCache`, ensure their capacity is appropriately sized for the workload to avoid excessive hash collisions or evictions, which can degrade performance. +Caching Utilities :: +For `StringInterner` and `ParsingCache`, ensure their capacity is appropriately sized for the workload to avoid excessive hash collisions or evictions, which can degrade performance. == Build and Test Considerations -* **Assertions**: Chronicle Core supports zero-cost assertions. +Assertions :: +Chronicle Core supports zero-cost assertions. Ensure you understand the difference between standard Java assertions (`-ea`) and Chronicle's `AssertUtil.SKIP_ASSERTIONS` mechanism to verify code correctly during testing without performance impact in production if assertions are compiled out. diff --git a/src/main/docs/decision-log.adoc b/src/main/docs/decision-log.adoc index d15cfd866ae..c1579fcd8f6 100644 --- a/src/main/docs/decision-log.adoc +++ b/src/main/docs/decision-log.adoc @@ -3,9 +3,9 @@ :lang: en-GB :source-highlighter: rouge -This file captures *component-specific* architectural decisions for the Chronicle Core library. +This file captures _component-specific_ architectural decisions for the Chronicle Core library. Identifiers follow the -`--xxx` pattern mandated by the company-wide guidelines; numbers are unique within the *CORE* scope. +`--xxx` pattern mandated by the company-wide guidelines; numbers are unique within the _CORE_ scope. == Decision Index @@ -22,10 +22,10 @@ Context:: * Numerous Chronicle projects expressed difficulty tracing requirements to code and to ADRs. * A single, cross-project taxonomy is required for tooling (link-checks, changelogs). Decision Statement:: -* Chronicle Core **adopts** the Nine-Box tag set (`FN`, `NF-P`, `NF-S`, `NF-O`, `TEST`, `DOC`, `OPS`, `UX`, `RISK`) for all requirement and decision identifiers. +* Chronicle Core *adopts* the Nine-Box tag set (`FN`, `NF-P`, `NF-S`, `NF-O`, `TEST`, `DOC`, `OPS`, `UX`, `RISK`) for all requirement and decision identifiers. Alternatives Considered:: -* *Free-form tags* - flexible but unsearchable; high risk of duplication. -* *SAFe epic/capability IDs* - too coarse for low-level library work. +* _Free-form tags_ - flexible but unsearchable; high risk of duplication. +* _SAFe epic/capability IDs_ - too coarse for low-level library work. Rationale for Decision:: * Provides immediate traceability across projects with minimal ceremony. * Aligns with company-wide governance already approved under `ALL-DOC-100`. @@ -42,11 +42,11 @@ Context:: * Off-heap memory and mmap handles are invisible to the JVM GC. * Production leaks previously caused kernel `ENFILE` (too many open files) errors. Decision Statement:: -* All classes that manage native resources **MUST** extend `AbstractCloseable` -*and* implement `ReferenceCounted` when those resources are shareable. +* All classes that manage native resources *MUST* extend `AbstractCloseable` +_and_ implement `ReferenceCounted` when those resources are shareable. Alternatives Considered:: -* *Rely on `finalize()`* - deprecated in JDK 18; unpredictable latency. -* *Cleaner API* - safer, but incurs allocation-heavy reachability fences. +* _Rely on `finalize()`_ - deprecated in JDK 18; unpredictable latency. +* _Cleaner API_ - safer, but incurs allocation-heavy reachability fences. Rationale for Decision:: * `AbstractCloseable` already proven in Queue/Bytes with micros-level overhead. * Reference counting guarantees orderly teardown in multi-thread/process use cases. @@ -66,10 +66,10 @@ Context:: Decision Statement:: * Introduce optional system property `chronicle.core.mmap.allowedDirs` (comma-separated list). -When set, **any** attempt to memory-map a file outside these directories fails fast with a `SecurityException`. +When set, *any* attempt to memory-map a file outside these directories fails fast with a `SecurityException`. Alternatives Considered:: -* *Do nothing* - cheaper but fails compliance for tier-1 banks. -* *Java SecurityManager checks* - SecurityManager removed in JDK 18. +* _Do nothing_ - cheaper but fails compliance for tier-1 banks. +* _Java SecurityManager checks_ - SecurityManager removed in JDK 18. Rationale for Decision:: * Simple, zero-allocation check; satisfies audit without penalising normal users (property unset = current behaviour). Impact & Consequences:: @@ -90,8 +90,8 @@ Decision Statement:: When system property `chronicle.core.backgroundRelease=true` is set, `AbstractCloseable.performClose()` submits the heavy deallocation to this low-priority thread, returning immediately. Alternatives Considered:: -* *Always release synchronously* - simplest, but violates latency SLOs. -* *Thread-pool per caller* - avoids single bottleneck but over-complex. +* _Always release synchronously_ - simplest, but violates latency SLOs. +* _Thread-pool per caller_ - avoids single bottleneck but over-complex. Rationale for Decision:: * Keeps hot path predictable; opt-in preserves zero-thread baseline for most users. Impact & Consequences:: @@ -112,8 +112,8 @@ Decision Statement:: `chronicle.init.runnable` and `chronicle.postinit.runnable` system properties. * Support `ChronicleInitRunnable` implementations discovered via `ServiceLoader`. Alternatives Considered:: -* *Static blocks in `Jvm`* - simple but hard to extend by user code. -* *External bootstrap scripts* - violate embedability and complicate deployment. +* _Static blocks in `Jvm`_ - simple but hard to extend by user code. +* _External bootstrap scripts_ - violate embedability and complicate deployment. Rationale for Decision:: * Gives applications controlled extension points without requiring code changes. Impact & Consequences:: diff --git a/src/main/docs/deterministic-resource-management.adoc b/src/main/docs/deterministic-resource-management.adoc index 168b9d838f1..5afb30b6612 100644 --- a/src/main/docs/deterministic-resource-management.adoc +++ b/src/main/docs/deterministic-resource-management.adoc @@ -15,10 +15,13 @@ This is achieved through a combination of `Closeable` and `ReferenceCounted` pat === 1.1. Resource Tracing Resource tracing helps diagnose leaks and incorrect resource usage by tracking resource creation and release sites. -* **Enabling**: Set system property `jvm.resource.tracing=true`. +Enabling :: +Set system property `jvm.resource.tracing=true`. Alternatively, call `AbstractCloseable.enableCloseableTracing()` and `AbstractReferenceCounted.enableReferenceTracing()`. -* **Debugging**: In tests, use `CloseableUtils.assertCloseablesClosed()` and `ReferenceCountedUtils.assertReferencesReleased()` to verify all tracked resources are properly released. -* **Tracking Mechanism**: Utilizes `WeakIdentityHashMap` to store references to managed resources without preventing them from being garbage collected if all other references are gone. +Debugging :: +In tests, use `CloseableUtils.assertCloseablesClosed()` and `ReferenceCountedUtils.assertReferencesReleased()` to verify all tracked resources are properly released. +Tracking Mechanism :: +Utilizes `WeakIdentityHashMap` to store references to managed resources without preventing them from being garbage collected if all other references are gone. == 2. `Closeable` Resources with `AbstractCloseable` @@ -27,23 +30,29 @@ The `net.openhft.chronicle.core.io.Closeable` interface extends `java.io.Closeab === 2.1. Lifecycle and Features -* **Creation Tracking**: If resource tracing is enabled, `AbstractCloseable` records a stack trace where the resource was created (`createdHere()`). -* **Single-Threaded Checks**: Implements `SingleThreadedChecked`. +Creation Tracking :: +If resource tracing is enabled, `AbstractCloseable` records a stack trace where the resource was created (`createdHere()`). +Single-Threaded Checks :: +Implements `SingleThreadedChecked`. It checks that operations (especially `close()`) are performed by the same thread that first used the resource, or if the check is disabled/reset. This helps prevent concurrent modification issues. * Use `singleThreadedCheckReset()` to hand off an object to another thread after its initial setup. * Disable checks via system property `disable.single.threaded.check=true` or instance method `singleThreadedCheckDisabled(true)`. -* **`performClose()`**: Subclasses *must* implement `protected abstract void performClose()` for their specific cleanup logic. +`performClose()` :: +Subclasses _must_ implement `protected abstract void performClose()` for their specific cleanup logic. This method is guaranteed to be called exactly once. -* **State Management**: +State Management :: * `isClosed()`: Returns `true` if `performClose()` has completed. * `isClosing()`: Returns `true` if the close process has started. * `closedHere`: If tracing is enabled, a stack trace of the `close()` call site is recorded. -* **`throwExceptionIfClosed()`**: Throws a `ClosedIllegalStateException` if the resource is closed or closing, providing `closedHere` details. -* **`warnAndCloseIfNotClosed()`**: A safety net, typically called from finalizers. +`throwExceptionIfClosed()` :: +Throws a `ClosedIllegalStateException` if the resource is closed or closing, providing `closedHere` details. +`warnAndCloseIfNotClosed()` :: +A safety net, typically called from finalizers. If the resource wasn't closed, it logs a warning (with `createdHere` trace if available) and closes it. Controlled by system property `disable.discard.warning` (default `true`, meaning warnings are off by default unless resource tracing explicitly enables them in some contexts). -* **Reference ID**: `referenceId()` provides a unique ID for the instance, useful for logging and tracking. +Reference ID :: +`referenceId()` provides a unique ID for the instance, useful for logging and tracking. === 2.2. Controlling Close Behavior @@ -63,27 +72,40 @@ The `ReferenceCounted` interface provides a contract for managing shared resourc === 3.1. Core Operations -* **`reserve(ReferenceOwner owner)`**: Increments the reference count. +`reserve(ReferenceOwner owner)` :: +Increments the reference count. The `owner` is an identifier for who holds the reservation. -* **`release(ReferenceOwner owner)`**: Decrements the reference count. +`release(ReferenceOwner owner)` :: +Decrements the reference count. When the count reaches zero, `performRelease()` (in `AbstractReferenceCounted`) is triggered. -* **`releaseLast(ReferenceOwner owner)`**: Decrements the count and asserts it becomes zero. +`releaseLast(ReferenceOwner owner)` :: +Decrements the count and asserts it becomes zero. Throws `IllegalStateException` if not the last release. -* **`tryReserve(ReferenceOwner owner)`**: Attempts to reserve; returns `false` if the resource is already released, rather than throwing. -* **`reserveTransfer(ReferenceOwner from, ReferenceOwner to)`**: Atomically moves a reservation from one owner to another without changing the total reference count. -* **`refCount()`**: Returns the current number of active reservations. -* **`ReferenceOwner`**: Identifies the entity holding a reference. Common instances include `ReferenceOwner.INIT` (for the initial reference held by the creator) and `ReferenceOwner.TMP` (or `ReferenceOwner.temporary("name")`) for short-lived, identifiable reservations. `referenceName()` and `referenceId()` help in debugging. +`tryReserve(ReferenceOwner owner)` :: +Attempts to reserve; returns `false` if the resource is already released, rather than throwing. +`reserveTransfer(ReferenceOwner from, ReferenceOwner to)` :: +Atomically moves a reservation from one owner to another without changing the total reference count. +`refCount()` :: +Returns the current number of active reservations. +`ReferenceOwner` :: +Identifies the entity holding a reference. +Common instances include `ReferenceOwner.INIT` (for the initial reference held by the creator) and `ReferenceOwner.TMP` (or `ReferenceOwner.temporary("name")`) for short-lived, identifiable reservations. +`referenceName()` and `referenceId()` help in debugging. === 3.2. `AbstractReferenceCounted` Implementation Details -* **Integration with Tracing**: If enabled, tracks creation site (`createdHere()`) and reservation owners. -* **`performRelease()`**: Subclasses implement this for actual resource deallocation. -* **Background Release**: Can offload `performRelease()` to `BackgroundResourceReleaser` by overriding `canReleaseInBackground()`. -* **Diagnostics**: +Integration with Tracing :: +If enabled, tracks creation site (`createdHere()`) and reservation owners. +`performRelease()` :: +Subclasses implement this for actual resource deallocation. +Background Release :: +Can offload `performRelease()` to `BackgroundResourceReleaser` by overriding `canReleaseInBackground()`. +Diagnostics :: * `throwExceptionIfNotReleased()`: Checks if `refCount() > 0`. * `warnAndReleaseIfNotReleased()`: Releases if not already, and warns if tracing is enabled. * System properties `reference.warn.secs` and `reference.warn.count` control warnings for long release times or high reservation counts. -* **Listeners**: `addReferenceChangeListener(ReferenceChangeListener)` allows tracking reservation changes for diagnostic purposes. +Listeners :: +`addReferenceChangeListener(ReferenceChangeListener)` allows tracking reservation changes for diagnostic purposes. === 3.3. `AbstractCloseableReferenceCounted` @@ -97,7 +119,7 @@ The actual resource cleanup in `performRelease()` happens when the reference cou For resources where deallocation (e.g., `munmap`) can take a significant time (milliseconds) and impact performance-sensitive threads, Chronicle Core provides an optional background releaser thread. -* **Configuration via System Properties**: +Configuration via System Properties :: * `background.releaser=true` (default): Enables queuing of resource deallocation. * `background.releaser.thread=true` (default): A dedicated daemon thread (`background~resource~releaser`) is started to process the queue. * `background.releaser.thread=false`: No dedicated thread is started. @@ -105,7 +127,7 @@ Resources are queued, but `BackgroundResourceReleaser.releasePendingResources()` * `background.releaser=false`: Resources are deallocated synchronously on the calling thread. The `BackgroundResourceReleaser` queue is not used. -* **Usage**: +Usage :: * `AbstractCloseable` instances opt-in by overriding `shouldPerformCloseInBackground()` to return `true`. * `AbstractReferenceCounted` instances opt-in by overriding `canReleaseInBackground()` to return `true`. * `BackgroundResourceReleaser.releasePendingResources()` can be called to process the queue synchronously if needed, especially in tests or specific shutdown sequences. diff --git a/src/main/docs/exception-overview.adoc b/src/main/docs/exception-overview.adoc index 693fc5d1e3f..cf7cd96d28c 100644 --- a/src/main/docs/exception-overview.adoc +++ b/src/main/docs/exception-overview.adoc @@ -21,7 +21,8 @@ if (isClosed()) { == ClosedIORuntimeException -Raised when an I/O operation targets a closed file or socket. It is an +Raised when an I/O operation targets a closed file or socket. +It is an unchecked variant built on link:../src/main/java/net/openhft/chronicle/core/io/IORuntimeException.java[IORuntimeException]. Often returned from `IORuntimeException.newIORuntimeException` when the underlying `IOException` indicates a closed resource. @@ -38,14 +39,17 @@ try { == IORuntimeException -Generic wrapper for `IOException` failures. Many utility methods rethrow -checked I/O errors using this unchecked form. Its helper method can return a +Generic wrapper for `IOException` failures. +Many utility methods rethrow +checked I/O errors using this unchecked form. +Its helper method can return a `ClosedIORuntimeException` when appropriate. == InvalidMarshallableException Signifies that an object being marshalled or unmarshalled violates its -constraints. Commonly thrown by implementations of +constraints. +Commonly thrown by implementations of link:../src/main/java/net/openhft/chronicle/core/io/Validatable.java[Validatable]. [source,java] @@ -59,7 +63,8 @@ public void validate() { == ThreadingIllegalStateException Indicates an illegal operation due to concurrent access on a single-threaded -component. The stack trace helps identify where the unsafe access occurred. +component. +The stack trace helps identify where the unsafe access occurred. Classes implementing link:../src/main/java/net/openhft/chronicle/core/io/SingleThreadedChecked.java[SingleThreadedChecked] may throw this exception. diff --git a/src/main/docs/jvm-and-os-utilities.adoc b/src/main/docs/jvm-and-os-utilities.adoc index 1ab73999516..53a39f06835 100644 --- a/src/main/docs/jvm-and-os-utilities.adoc +++ b/src/main/docs/jvm-and-os-utilities.adoc @@ -30,17 +30,20 @@ Provides methods to query various aspects of the JVM runtime environment: === 1.2. System Properties `Jvm` enhances system property access and management: -* **Loading from File**: Automatically looks for a file named `system.properties` in the classpath or current/parent directory at startup. +Loading from File :: +Automatically looks for a file named `system.properties` in the classpath or current/parent directory at startup. Values from this file are loaded unless already set via `-D` command-line arguments. The file to load can be overridden with `-Dsystem.properties=my.file`. -* **Typed Getters**: Provides type-safe getters like: -** `Jvm.getProperty(String key)` / `Jvm.getProperty(String key, String defaultValue)` -** `Jvm.getBoolean(String key)` / `Jvm.getBoolean(String key, boolean defaultValue)` (interprets "true", "yes", or empty as true) -** `Jvm.getLong(String key, Long defaultValue)` -** `Jvm.getInteger(String key, Integer defaultValue)` -** `Jvm.getDouble(String key, double defaultValue)` -** `Jvm.getSize(String key, long defaultValue)`: Parses memory sizes with suffixes (k, M, G, T). -* **Initialization Guarantee**: Using these `Jvm` getters ensures that the `Jvm` class (and its property loading) is initialized before the property is read. +Typed Getters :: +Provides type-safe getters like: +* `Jvm.getProperty(String key)` / `Jvm.getProperty(String key, String defaultValue)` +* `Jvm.getBoolean(String key)` / `Jvm.getBoolean(String key, boolean defaultValue)` (interprets "true", "yes", or empty as true) +* `Jvm.getLong(String key, Long defaultValue)` +* `Jvm.getInteger(String key, Integer defaultValue)` +* `Jvm.getDouble(String key, double defaultValue)` +* `Jvm.getSize(String key, long defaultValue)`: Parses memory sizes with suffixes (k, M, G, T). +Initialization Guarantee :: +Using these `Jvm` getters ensures that the `Jvm` class (and its property loading) is initialized before the property is read. === 1.3. Memory Information @@ -54,13 +57,14 @@ The file to load can be overridden with `-Dsystem.properties=my.file`. * `Jvm.rethrow(Throwable t)`: Throws any checked exception as an unchecked `RuntimeException` without wrapping it, using a vacuous cast. The method never actually returns. -* **Centralised Handlers**: Provides static methods to access default or thread-local exception handlers for different severity levels: +Centralised Handlers :: +Provides static methods to access default or thread-local exception handlers for different severity levels: ** `Jvm.error()`: For critical errors. ** `Jvm.warn()`: For warnings. ** `Jvm.perf()`: For performance-related logging (often INFO level). ** `Jvm.debug()`: For debug messages. ** `Jvm.startup()`: For startup messages (often INFO level). -* **Configuration**: +Configuration :: ** `Jvm.setExceptionHandlers(...)`: Sets default handlers for various levels. ** `Jvm.setThreadLocalExceptionHandlers(...)`: Sets handlers specific to the current thread. ** `Jvm.resetExceptionHandlers()`: Restores default handlers. @@ -133,7 +137,7 @@ The `OS` class provides methods to interact with the operating system for inform * `OS.pageAlign(long size, int pageSize)` / `OS.pageAlign(long size)`: Aligns a size to a page boundary. * `OS.mapAlignment()`: Returns the required alignment for memory map offsets (usually `defaultOsPageSize()`). * `OS.mapAlign(long offset, int pageAlignment)` / `OS.mapAlign(long offset)`: Aligns an offset for memory mapping. -* **Memory Mapping**: +Memory Mapping :: ** `OS.map(FileChannel, MapMode, long start, long size, int pageSize)` / `OS.map(FileChannel, MapMode, long start, long size)`: Maps a region of a file into memory, returning the native address. ** `OS.unmap(long address, long size, int pageSize)` / `OS.unmap(long address, long size)`: Unmaps a previously mapped memory region. ** `OS.memoryMapped()`: Returns the total number of bytes currently memory-mapped by Chronicle Core through these methods. @@ -151,13 +155,20 @@ The `OS` class provides methods to interact with the operating system for inform The `OS.memory()` method returns an instance of the `net.openhft.chronicle.core.Memory` interface (typically `UnsafeMemory`), which provides direct access to memory operations: -* **Fences**: `storeFence()`, `loadFence()` for memory ordering. -* **Allocation/Deallocation**: `allocate(long capacity)`, `freeMemory(long address, long size)` for native memory. -* **Primitive Read/Write**: Methods like `readInt(long address)`, `writeLong(Object o, long offset, long value)` for all primitive types, with both direct address and object+offset variants. -* **Volatile Read/Write**: `readVolatileInt()`, `writeVolatileLong()`, etc., for thread-safe access with memory visibility guarantees. -* **Ordered Writes**: `writeOrderedInt()`, `writeOrderedLong()` for "lazy set" semantics. -* **Atomic Operations**: `compareAndSwapInt()`, `compareAndSwapLong()`, `getAndSetInt()`, `addInt()`, `addLong()` for atomic updates. -* **Copying Memory**: `copyMemory(...)` overloads for various source/destination types (address-to-address, array-to-address, object-to-address, etc.). +Fences :: +`storeFence()`, `loadFence()` for memory ordering. +Allocation/Deallocation :: +`allocate(long capacity)`, `freeMemory(long address, long size)` for native memory. +Primitive Read/Write :: +Methods like `readInt(long address)`, `writeLong(Object o, long offset, long value)` for all primitive types, with both direct address and object+offset variants. +Volatile Read/Write :: +`readVolatileInt()`, `writeVolatileLong()`, etc., for thread-safe access with memory visibility guarantees. +Ordered Writes :: +`writeOrderedInt()`, `writeOrderedLong()` for "lazy set" semantics. +Atomic Operations :: +`compareAndSwapInt()`, `compareAndSwapLong()`, `getAndSetInt()`, `addInt()`, `addLong()` for atomic updates. +Copying Memory :: +`copyMemory(...)` overloads for various source/destination types (address-to-address, array-to-address, object-to-address, etc.). _For detailed usage of these memory operations, refer to the specific Javadoc of `net.openhft.chronicle.core.Memory` and `net.openhft.chronicle.core.UnsafeMemory`._ @@ -166,11 +177,15 @@ _For detailed usage of these memory operations, refer to the specific Javadoc of `IOTools` provides higher-level utilities for common I/O tasks: * `IOTools.isClosedException(Exception e)`: Heuristic to check if an `IOException` likely indicates a closed connection/stream. -* **File/Directory Deletion**: `deleteDirWithFiles(...)`, `shallowDeleteDirWithFiles(...)` for recursive or shallow directory cleanup. -* **File Reading/Writing**: -** `IOTools.readFile(Class, String name)`: Reads a file from classpath or filesystem, supports `.gz` transparently. -** `IOTools.readAsBytes(InputStream is)`: Reads an entire InputStream into a byte array. -** `IOTools.writeFile(String filename, byte[] bytes)`: Writes bytes to a file, supports `.gz` transparently. -* **Temporary Files**: `IOTools.createTempFile(String name)`, `IOTools.createTempDirectory(String name)` create temporary files/directories typically under the build target directory. -* **Buffer Cleaning**: `IOTools.clean(ByteBuffer bb)`: Utility to clean direct ByteBuffers using `CleanerServiceLocator`. -* **URL Handling**: `IOTools.urlFor(Class/ClassLoader, String name)` to locate resources, `IOTools.open(URL url)` to open streams (handles `.gz`). +File/Directory Deletion :: +`deleteDirWithFiles(...)`, `shallowDeleteDirWithFiles(...)` for recursive or shallow directory cleanup. +File Reading/Writing :: +* `IOTools.readFile(Class, String name)`: Reads a file from classpath or filesystem, supports `.gz` transparently. +* `IOTools.readAsBytes(InputStream is)`: Reads an entire InputStream into a byte array. +* `IOTools.writeFile(String filename, byte[] bytes)`: Writes bytes to a file, supports `.gz` transparently. +Temporary Files :: +`IOTools.createTempFile(String name)`, `IOTools.createTempDirectory(String name)` create temporary files/directories typically under the build target directory. +Buffer Cleaning :: +`IOTools.clean(ByteBuffer bb)`: Utility to clean direct ByteBuffers using `CleanerServiceLocator`. +URL Handling :: +`IOTools.urlFor(Class/ClassLoader, String name)` to locate resources, `IOTools.open(URL url)` to open streams (handles `.gz`). diff --git a/src/main/docs/memory-management-guide.adoc b/src/main/docs/memory-management-guide.adoc index 0e36603476f..0425d6cadb8 100644 --- a/src/main/docs/memory-management-guide.adoc +++ b/src/main/docs/memory-management-guide.adoc @@ -15,18 +15,19 @@ Chronicle Core provides mechanisms for allocating and managing memory outside th The primary way to interact with raw off-heap memory is via `OS.memory()`, which returns an instance of the `net.openhft.chronicle.core.Memory` interface. This is typically backed by `net.openhft.chronicle.core.UnsafeMemory`. -* **Allocation and Deallocation**: -** `long address = memory.allocate(long capacity)`: Allocates a block of native memory. -** `memory.freeMemory(long address, long size)`: Releases the allocated memory. *It is crucial to manually free this memory to prevent leaks.* -* **Tracking**: -** `memory.nativeMemoryUsed()`: Returns the total bytes allocated via `memory.allocate()` that haven't been freed. -* **Operations**: The `Memory` interface provides a rich set of methods for: -** Reading and writing all Java primitive types (e.g., `readInt(long address)`, `writeLong(Object obj, long offset, long value)`). -** Volatile reads/writes for thread safety (e.g., `readVolatileInt(long address)`, `writeVolatileLong(Object obj, long offset, long value)`). -** Ordered writes for specific memory ordering guarantees (e.g., `writeOrderedInt(long address)`). -** Compare-and-Swap (CAS) operations (e.g., `compareAndSwapLong(long address, long expected, long value)`). -** Memory copying (`copyMemory(...)` overloads). -** Memory fences (`storeFence()`, `loadFence()`). +Allocation and Deallocation :: +* `long address = memory.allocate(long capacity)`: Allocates a block of native memory. +* `memory.freeMemory(long address, long size)`: Releases the allocated memory. _It is crucial to manually free this memory to prevent leaks._ +Tracking :: +* `memory.nativeMemoryUsed()`: Returns the total bytes allocated via `memory.allocate()` that haven't been freed. +Operations :: +The `Memory` interface provides a rich set of methods for: +* Reading and writing all Java primitive types (e.g., `readInt(long address)`, `writeLong(Object obj, long offset, long value)`). +* Volatile reads/writes for thread safety (e.g., `readVolatileInt(long address)`, `writeVolatileLong(Object obj, long offset, long value)`). +* Ordered writes for specific memory ordering guarantees (e.g., `writeOrderedInt(long address)`). +* Compare-and-Swap (CAS) operations (e.g., `compareAndSwapLong(long address, long expected, long value)`). +* Memory copying (`copyMemory(...)` overloads). +* Memory fences (`storeFence()`, `loadFence()`). [source,java] ---- @@ -65,7 +66,7 @@ Memory-mapped files allow sections of a file to be treated as if they were in-me ** `mode`: Can be `MapMode.READ_ONLY`, `MapMode.READ_WRITE`, or `MapMode.PRIVATE`. ** `start` and `size` define the region of the file to map. These may be adjusted by page alignment requirements. -* `OS.unmap(long address, long size)`: Releases the mapping. *This is crucial to avoid resource leaks (virtual memory and file handles).* +* `OS.unmap(long address, long size)`: Releases the mapping. _This is crucial to avoid resource leaks (virtual memory and file handles)._ [source,java] ---- @@ -135,25 +136,32 @@ System.out.println("Requested offset " + fileOffset + ", Aligned offset: " + ali While Chronicle Core often uses its own `Memory` abstraction, you might interact with direct `ByteBuffer`s from other libraries. -* **Creation**: `ByteBuffer.allocateDirect(capacity)` allocates off-heap memory. -* **Cleanup**: Relies on `sun.misc.Cleaner` and GC, which can be unpredictable. -* **Chronicle Core Utilities for Direct `ByteBuffer`s**: -** `IOTools.clean(ByteBuffer bb)`: Attempts to invoke the cleaner of a direct `ByteBuffer` more promptly using `CleanerServiceLocator`. -** `CleanerServiceLocator`: Dynamically finds a `ByteBufferCleanerService` (e.g., `Jdk9ByteBufferCleanerService` or `ReflectionBasedByteBufferCleanerService`) to invoke the `clean()` method of a `DirectBuffer`'s cleaner. -** `DirectBufferUtil`: Provides utility methods to interact with `sun.nio.ch.DirectBuffer` like getting its address or cleaning it. -* **Caution**: Manually cleaning `ByteBuffer`s can be dangerous if the buffer (or its memory region) is still in use. +Creation :: +`ByteBuffer.allocateDirect(capacity)` allocates off-heap memory. +Cleanup :: +Relies on `sun.misc.Cleaner` and GC, which can be unpredictable. +Chronicle Core Utilities for Direct `ByteBuffer`s :: +* `IOTools.clean(ByteBuffer bb)`: Attempts to invoke the cleaner of a direct `ByteBuffer` more promptly using `CleanerServiceLocator`. +* `CleanerServiceLocator`: Dynamically finds a `ByteBufferCleanerService` (e.g., `Jdk9ByteBufferCleanerService` or `ReflectionBasedByteBufferCleanerService`) to invoke the `clean()` method of a `DirectBuffer`'s cleaner. +* `DirectBufferUtil`: Provides utility methods to interact with `sun.nio.ch.DirectBuffer` like getting its address or cleaning it. +Caution :: +Manually cleaning `ByteBuffer`s can be dangerous if the buffer (or its memory region) is still in use. This should be done with extreme care. == 5. Freeing Memory and Debugging Issues -* **Rule 1**: Always call `OS.memory().freeMemory(address, size)` for each corresponding `OS.memory().allocate(capacity)` call. -* **Rule 2**: Always call `OS.unmap(address, size)` for every successful `OS.map()` call. -* **Leak Diagnosis**: -** Monitor `OS.memoryMapped()` to track mapped regions. -** Monitor `OS.memory().nativeMemoryUsed()` for `UnsafeMemory` allocations. -** Monitor `Jvm.usedDirectMemory()` for `ByteBuffer.allocateDirect()` usage. -* **Resource Tracing**: Enable `jvm.resource.tracing=true` (see `systemProperties.adoc`). +Rule 1 :: +Always call `OS.memory().freeMemory(address, size)` for each corresponding `OS.memory().allocate(capacity)` call. +Rule 2 :: +Always call `OS.unmap(address, size)` for every successful `OS.map()` call. +Leak Diagnosis :: +* Monitor `OS.memoryMapped()` to track mapped regions. +* Monitor `OS.memory().nativeMemoryUsed()` for `UnsafeMemory` allocations. +* Monitor `Jvm.usedDirectMemory()` for `ByteBuffer.allocateDirect()` usage. +Resource Tracing :: +Enable `jvm.resource.tracing=true` (see `systemProperties.adoc`). This helps `AbstractCloseable` (often a base for classes managing mapped regions or other native resources) to record `createdHere()` and `closedHere()` stack traces, aiding in diagnosing leaks or use-after-close bugs. Refer to the "Deterministic Resource Management" guide for more on this. -* **Alignment Errors**: Direct use of `UnsafeMemory` or `Memory` interface methods can lead to `MisAlignedAssertionError` if addresses are not correctly aligned for the data type being accessed, particularly on ARM architectures. +Alignment Errors :: +Direct use of `UnsafeMemory` or `Memory` interface methods can lead to `MisAlignedAssertionError` if addresses are not correctly aligned for the data type being accessed, particularly on ARM architectures. Ensure proper alignment when performing low-level operations. diff --git a/src/main/docs/object-pooling-and-caching.adoc b/src/main/docs/object-pooling-and-caching.adoc index 040d9bc0903..0606ed93abc 100644 --- a/src/main/docs/object-pooling-and-caching.adoc +++ b/src/main/docs/object-pooling-and-caching.adoc @@ -14,14 +14,16 @@ It maintains an internal hashed array to store recently encountered strings. When a `CharSequence` is "interned," if an equal `String` is already in the cache, the cached instance is returned, otherwise, the new `String` is cached and returned. This can save memory by ensuring that identical string values share the same `String` object. -* **Key Methods**: -** `intern(@Nullable CharSequence cs)`: Returns a cached `String` equivalent to the input `CharSequence`, or caches and returns `cs.toString()` if not already present. -** `index(@Nullable CharSequence cs, @Nullable Changed onChanged)`: Returns the internal array index for the interned string. If the string is new to the cache and `onChanged` is provided, the callback is invoked with the index and the new string value. This is useful for associating other cached data with the interned string. -** `get(int index)`: Retrieves an interned string by its index. -** `capacity()`: Returns the size of the internal array used for caching. -* **Thread Safety**: `StringInterner` is designed for high performance and uses a "best-effort" approach to thread safety for its cache updates, relying on the thread safety of `String` itself. +Key Methods :: +* `intern(@Nullable CharSequence cs)`: Returns a cached `String` equivalent to the input `CharSequence`, or caches and returns `cs.toString()` if not already present. +* `index(@Nullable CharSequence cs, @Nullable Changed onChanged)`: Returns the internal array index for the interned string. If the string is new to the cache and `onChanged` is provided, the callback is invoked with the index and the new string value. This is useful for associating other cached data with the interned string. +* `get(int index)`: Retrieves an interned string by its index. +* `capacity()`: Returns the size of the internal array used for caching. +Thread Safety :: +`StringInterner` is designed for high performance and uses a "best-effort" approach to thread safety for its cache updates, relying on the thread safety of `String` itself. It doesn't guarantee that all threads will always see the exact same cached `String` object for identical inputs if interning concurrently, but it ensures functional correctness (i.e., the returned string will be equal to the input). -* **Cache Mechanism**: Uses a simple hashing mechanism with a toggle for placing new entries, aiming to balance cache utilization. +Cache Mechanism :: +Uses a simple hashing mechanism with a toggle for placing new entries, aiming to balance cache utilization. [source,java] ---- @@ -41,11 +43,12 @@ assert internedS1 == internedCS; // Both point to the same String object in the It uses an internal cache (an array) to store enum values, making subsequent lookups for the same name very fast. This is particularly useful when deserializing enums or parsing them from external string sources. -* **Key Methods**: -** `EnumInterner(Class eClass)` / `EnumInterner(Class eClass, int capacity)`: Constructors to create an interner for a specific enum class, optionally with a suggested initial capacity for the internal cache. -** `intern(@NotNull CharSequence cs)`: Takes a `CharSequence` representing an enum name and returns the corresponding enum constant. +Key Methods :: +* `EnumInterner(Class eClass)` / `EnumInterner(Class eClass, int capacity)`: Constructors to create an interner for a specific enum class, optionally with a suggested initial capacity for the internal cache. +* `intern(@NotNull CharSequence cs)`: Takes a `CharSequence` representing an enum name and returns the corresponding enum constant. If the name is encountered for the first time, it resolves the enum constant (using an underlying `EnumCache`) and caches it. -* **Static Factory**: `EnumInterner.ENUM_INTERNER` is a `ClassLocal` instance that provides a global, per-enum-type cache of `EnumInterner` instances. +Static Factory :: +`EnumInterner.ENUM_INTERNER` is a `ClassLocal` instance that provides a global, per-enum-type cache of `EnumInterner` instances. [source,java] ---- @@ -65,16 +68,17 @@ assert rm == RoundingMode.HALF_UP; `EnumCache` is an abstraction for caching enum instances, providing efficient access by name or ordinal index. It supports both standard Java (static) enums and dynamic enums (which can have instances defined at runtime). -* **Factory Method**: `EnumCache.of(Class eClass)` is the primary way to obtain an `EnumCache` instance for a given enum type. +Factory Method :: +`EnumCache.of(Class eClass)` is the primary way to obtain an `EnumCache` instance for a given enum type. It automatically provides the correct cache implementation (`StaticEnumClass` for standard enums, `DynamicEnumClass` for enums implementing `CoreDynamicEnum`). -* **Key Methods**: -** `valueOf(String name)`: Returns the enum instance for the given name. For `DynamicEnumClass`, it can create new instances if the name is not found. -** `get(String name)`: Similar to `valueOf`, often an alias. -** `forIndex(int index)`: Retrieves an enum instance by its ordinal (for static enums) or creation index (for dynamic enums). -** `asArray()`: Returns an array containing all known enum instances. -** `size()`: Returns the number of enum instances in the cache. -** `type()`: Returns the enum `Class` object managed by this cache. -** `createMap()` / `createSet()`: Utility methods to create `EnumMap`-like or `EnumSet`-like collections. +Key Methods :: +* `valueOf(String name)`: Returns the enum instance for the given name. For `DynamicEnumClass`, it can create new instances if the name is not found. +* `get(String name)`: Similar to `valueOf`, often an alias. +* `forIndex(int index)`: Retrieves an enum instance by its ordinal (for static enums) or creation index (for dynamic enums). +* `asArray()`: Returns an array containing all known enum instances. +* `size()`: Returns the number of enum instances in the cache. +* `type()`: Returns the enum `Class` object managed by this cache. +* `createMap()` / `createSet()`: Utility methods to create `EnumMap`-like or `EnumSet`-like collections. [source,java] ---- @@ -95,9 +99,10 @@ assert timeUnitCache.forIndex(TimeUnit.MILLISECONDS.ordinal()) == TimeUnit.MILLI It's particularly useful for caching class-specific information or helper objects. The value for each class is computed only once, the first time it's requested for that class. -* **Factory Method**: `ClassLocal.withInitial(Function, V> classVFunction)` creates a `ClassLocal` where the function provides the initial value for each class. -* **Usage**: -** `get(Class type)`: Retrieves the value associated with the given class, computing it via the initial-value function if it's the first access for this class. +Factory Method :: +`ClassLocal.withInitial(Function, V> classVFunction)` creates a `ClassLocal` where the function provides the initial value for each class. +Usage :: +* `get(Class type)`: Retrieves the value associated with the given class, computing it via the initial-value function if it's the first access for this class. [source,java] ---- @@ -119,9 +124,12 @@ The example from `EnumInterner` itself, `public static final ClassLocal sr = pool.get()) { `ParsingCache` is designed to cache the results of parsing strings into objects of type `E`, particularly for types where parsing is expensive (e.g., `BigDecimal`). It uses a fixed-size array and a hash function for quick lookups. -* **Constructor**: `ParsingCache(int capacity, Function eFunction)` where `eFunction` is used to create new objects when a string is not found in the cache. -* **Key Method**: -** `intern(@Nullable CharSequence cs)`: Returns the cached object for `cs`. +Constructor :: +`ParsingCache(int capacity, Function eFunction)` where `eFunction` is used to create new objects when a string is not found in the cache. +Key Method :: +* `intern(@Nullable CharSequence cs)`: Returns the cached object for `cs`. If `cs` is not in the cache, it's parsed using `eFunction`, the result is cached, and then returned. -* **Capacity**: The cache has a fixed capacity, determined at construction (rounded up to the next power of 2, min 128). -* `valueCount()`: Returns the number of items currently in the cache. +Capacity :: +The cache has a fixed capacity, determined at construction (rounded up to the next power of 2, min 128). +** `valueCount()`: Returns the number of items currently in the cache. [source,java] ---- diff --git a/src/main/docs/thread-safety-guarantees.adoc b/src/main/docs/thread-safety-guarantees.adoc index 58df70613c2..3da1adf8796 100644 --- a/src/main/docs/thread-safety-guarantees.adoc +++ b/src/main/docs/thread-safety-guarantees.adoc @@ -11,24 +11,25 @@ This document outlines Chronicle Core's mechanisms for helping developers manage `SingleThreadedChecked` is an interface designed for components that are intended for use by only one thread at a time. This is a common pattern for resources that are not inherently thread-safe or where the overhead of full concurrency control is undesirable. -* **Mechanism**: -** Implementations typically record the `Thread` that first accesses or "touches" the object. -** Subsequent accesses from different threads will result in a `net.openhft.chronicle.core.io.ThreadingIllegalStateException` being thrown. This exception usually includes details about the original owning thread and where it first accessed the object, aiding in debugging. -* **Disabling Checks**: -** Globally: Thread checking can be disabled for all instances by setting the system property `disable.single.threaded.check=true`. (See `systemProperties.adoc` for more details). -** Per Instance: Individual instances can have this check disabled by calling the `singleThreadedCheckDisabled(true)` method. -* **Benefits**: This mechanism provides an early warning system during development and testing, helping to catch unintended concurrent access to stateful objects that should be confined to a single thread. +Mechanism :: +* Implementations typically record the `Thread` that first accesses or "touches" the object. +* Subsequent accesses from different threads will result in a `net.openhft.chronicle.core.io.ThreadingIllegalStateException` being thrown. This exception usually includes details about the original owning thread and where it first accessed the object, aiding in debugging. +Disabling Checks :: +* Globally: Thread checking can be disabled for all instances by setting the system property `disable.single.threaded.check=true`. (See `systemProperties.adoc` for more details). +* Per Instance: Individual instances can have this check disabled by calling the `singleThreadedCheckDisabled(true)` method. +Benefits :: +This mechanism provides an early warning system during development and testing, helping to catch unintended concurrent access to stateful objects that should be confined to a single thread. == 2. The "Initialise, Reset, Hand-off" Pattern A common and supported pattern for using `SingleThreadedChecked` objects across different threads in a controlled manner involves the following steps: -. **Initialise**: The object is created and fully initialized by a "creating" thread. +. *Initialise*: The object is created and fully initialized by a "creating" thread. The first use of a method that performs a thread-safety check will typically set the initial "owner" thread. -. **Reset**: Before passing the object to another thread for subsequent use, the creating thread calls `singleThreadedCheckReset()` on the object. +. *Reset*: Before passing the object to another thread for subsequent use, the creating thread calls `singleThreadedCheckReset()` on the object. This clears the recorded "owner" thread. -. **Hand-off**: The object is then safely passed to a "worker" or "event loop" thread. -. **Worker Usage**: The first use of the object by the worker thread will then establish it as the new "owner" thread for subsequent checks. +. *Hand-off*: The object is then safely passed to a "worker" or "event loop" thread. +. *Worker Usage*: The first use of the object by the worker thread will then establish it as the new "owner" thread for subsequent checks. This pattern allows, for example, configuration on a main thread and operational use on a dedicated event loop thread, while still protecting against accidental concurrent access from other, unintended threads. @@ -52,32 +53,46 @@ resource.doMoreWork(); // Subsequent calls are fine from WorkerThread Many core Chronicle classes that manage resources or state that is not inherently thread-safe extend `SingleThreadedChecked`. -* **`AbstractCloseable` and `AbstractReferenceCounted`**: Both these foundational classes implement `SingleThreadedChecked`. -** Their internal operations, such as `close()`, `reserve()`, `release()`, and often methods that modify state, call `threadSafetyCheck(boolean isUsed)`. -** The `isUsed` parameter in `threadSafetyCheck(boolean isUsed)` indicates if the operation is actively using/modifying the resource (true) or just querying/preparing (false), potentially leading to stricter checks for active use. -** Developers extending these classes automatically inherit this thread-checking capability. -* **Rationale**: These classes often manage native resources (like memory pointers or file handles) or maintain complex internal state. Making them fully concurrent would often introduce significant performance overhead (e.g., locks, CAS loops on many fields) or complexity. The `SingleThreadedChecked` mechanism provides a lighter-weight safety net for their intended use patterns. +`AbstractCloseable` and `AbstractReferenceCounted` :: +Both these foundational classes implement `SingleThreadedChecked`. +* Their internal operations, such as `close()`, `reserve()`, `release()`, and often methods that modify state, call `threadSafetyCheck(boolean isUsed)`. +* The `isUsed` parameter in `threadSafetyCheck(boolean isUsed)` indicates if the operation is actively using/modifying the resource (true) or just querying/preparing (false), potentially leading to stricter checks for active use. +* Developers extending these classes automatically inherit this thread-checking capability. +Rationale :: +These classes often manage native resources (like memory pointers or file handles) or maintain complex internal state. +Making them fully concurrent would often introduce significant performance overhead (e.g., locks, CAS loops on many fields) or complexity. +The `SingleThreadedChecked` mechanism provides a lighter-weight safety net for their intended use patterns. == 4. When and Why Use `SingleThreadedChecked`? -* **Purpose**: To provide a development and testing safeguard for components that are not designed for concurrent access but might be used in environments where such access could accidentally occur. It's about ensuring thread confinement. -* **Use Cases**: -** Stateful objects managed by a single event loop (e.g., Chronicle Queue's `ExcerptAppender` or `ExcerptTailer`). -** Resources that have an explicit lifecycle tied to a specific thread for a phase of their operation. -* **Contrast with Thread-Safe Components**: Fully thread-safe components would use internal synchronization mechanisms (locks, atomic operations, concurrent data structures) and would not typically need `SingleThreadedChecked`. This check is for components that *avoid* such heavy concurrency mechanisms for performance reasons, relying instead on disciplined single-threaded access. +Purpose :: +To provide a development and testing safeguard for components that are not designed for concurrent access but might be used in environments where such access could accidentally occur. +It's about ensuring thread confinement. +Use Cases :: +* Stateful objects managed by a single event loop (e.g., Chronicle Queue's `ExcerptAppender` or `ExcerptTailer`). +* Resources that have an explicit lifecycle tied to a specific thread for a phase of their operation. +Contrast with Thread-Safe Components :: +Fully thread-safe components would use internal synchronization mechanisms (locks, atomic operations, concurrent data structures) and would not typically need `SingleThreadedChecked`. +This check is for components that _avoid_ such heavy concurrency mechanisms for performance reasons, relying instead on disciplined single-threaded access. == 5. Debugging `ThreadingIllegalStateException` When a `ThreadingIllegalStateException` is thrown: -* **Examine the Exception Message**: It usually indicates which thread currently owns the resource and which thread attempted to access it. -* **Inspect `usedByThreadHere`**: If the object is an instance of `AbstractCloseable` or `AbstractReferenceCounted`, and resource tracing is enabled, the exception may be chained with or contain a `StackTrace` (often accessible via `getCause()`) from where the original owning thread first "claimed" the object. This is invaluable for identifying the source of the contention. -* **Check for Common Issues**: -** **Accidental Sharing**: Is an instance of the resource being unintentionally shared across multiple threads without proper synchronization or hand-off? -** **Missing `singleThreadedCheckReset()`**: In a valid hand-off scenario, was `singleThreadedCheckReset()` called by the previous owning thread before the new thread started using the object? -** **Lifecycle Mismatch**: Is a component being accessed by a thread that should no longer have ownership, perhaps due to an incorrect assumption about its lifecycle state? +Examine the Exception Message :: +It usually indicates which thread currently owns the resource and which thread attempted to access it. +Inspect `usedByThreadHere` :: +If the object is an instance of `AbstractCloseable` or `AbstractReferenceCounted`, and resource tracing is enabled, the exception may be chained with or contain a `StackTrace` (often accessible via `getCause()`) from where the original owning thread first "claimed" the object. +This is invaluable for identifying the source of the contention. +Check for Common Issues :: +* *Accidental Sharing*: Is an instance of the resource being unintentionally shared across multiple threads without proper synchronization or hand-off? +* *Missing `singleThreadedCheckReset()`*: In a valid hand-off scenario, was `singleThreadedCheckReset()` called by the previous owning thread before the new thread started using the object? +* *Lifecycle Mismatch*: Is a component being accessed by a thread that should no longer have ownership, perhaps due to an incorrect assumption about its lifecycle state? == 6. Related Requirements and Properties -* **Project Requirements**: `CORE-FN-040`, `CORE-FN-041`, `CORE-FN-042`, `CORE-FN-043` detail these features. -* **System Property**: `disable.single.threaded.check` can disable these checks globally (see `systemProperties.adoc`). -* **Further Reading**: See the `README.adoc` for an initial overview of thread safety checks. +Project Requirements :: +`CORE-FN-040`, `CORE-FN-041`, `CORE-FN-042`, `CORE-FN-043` detail these features. +System Property :: +`disable.single.threaded.check` can disable these checks globally (see `systemProperties.adoc`). +Further Reading :: +See the `README.adoc` for an initial overview of thread safety checks. diff --git a/src/main/docs/unique-micro-time-provider-cas-loop.adoc b/src/main/docs/unique-micro-time-provider-cas-loop.adoc index 4f20cec24fc..66ed051c114 100644 --- a/src/main/docs/unique-micro-time-provider-cas-loop.adoc +++ b/src/main/docs/unique-micro-time-provider-cas-loop.adoc @@ -22,52 +22,54 @@ flowchart TD == 1. Quick Overview `UniqueMicroTimeProvider` wraps another `TimeProvider` (by default `SystemTimeProvider`) and -ensures *monotonic micro-second* timestamps within a single JVM by holding the last value in an +ensures _monotonic micro-second_ timestamps within a single JVM by holding the last value in an `AtomicLong`. -The CAS loop guarantees **(a)** forward-only progression and **(b)** at most one micro-second of +The CAS loop guarantees *(a)* forward-only progression and *(b)* at most one micro-second of artificial skew, even when the underlying provider regresses or multiple threads call the method concurrently. == 2. Algorithm Walk-Through -. *Fetch wall-clock* – read `provider.currentTimeMicros()` into `proposed`. -. *Read* `lastIssuedTimeMicros` **atomically** to obtain `last`. -. *Compare:* - * If `last >= proposed` the wall clock did not advance (or went backwards). +. _Fetch wall-clock_ – read `provider.currentTimeMicros()` into `proposed`. +. _Read_ `lastIssuedTimeMicros` *atomically* to obtain `last`. +. _Compare:_ +* If `last >= proposed` the wall clock did not advance (or went backwards). `proposed` is bumped to `last + 1` to preserve uniqueness. - * Otherwise keep the original `proposed`. -. *CAS attempt* – `compareAndSet(last, proposed)`. -. *Success?* - * **Yes:** return `proposed`. - * **No:** some other thread inserted a newer value; execute a short +* Otherwise keep the original `proposed`. +. _CAS attempt_ – `compareAndSet(last, proposed)`. +. _Success?_ +* *Yes:* return `proposed`. +* *No:* some other thread inserted a newer value; execute a short `Jvm.nanoPause()` spin and retry. -NOTE: The *nanosecond* and *millisecond* methods follow the same logic; the ns flavour multiplies +NOTE: The _nanosecond_ and _millisecond_ methods follow the same logic; the ns flavour multiplies or divides as needed to stay aligned to the µs mesh. == 3. Formal Guarantees -* **Uniqueness** – Two successive calls (`t1 < t2`) in any thread order yield `value(t1) < value(t2)`. -* **Bounded Drift** – The timestamp can be ahead of the real wall clock by < 1 µs × +* *Uniqueness* – Two successive calls (`t1 < t2`) in any thread order yield `value(t1) < value(t2)`. +* *Bounded Drift* – The timestamp can be ahead of the real wall clock by < 1 µs × `MAX_CONCURRENT_CALLS_PER_MICROSECOND` (validated with assertions at runtime). -* **Lock-free** – Only uses a single compare-and-swap; no blocking locks. +* *Lock-free* – Only uses a single compare-and-swap; no blocking locks. == 4. Performance -* *Typical latency:* 45-80 ns on a modern x86_64 CPU (CAS hit + cache-hit). -* *Worst case:* < 200 ns under heavy contention (retry loop adds an extra CAS and a nanospin). -* *GC pressure:* one `AtomicLong` field, no heap allocations. +* _Typical latency:_ 45-80 ns on a modern x86_64 CPU (CAS hit + cache-hit). +* _Worst case:_ < 200 ns under heavy contention (retry loop adds an extra CAS and a nanospin). +* _GC pressure:_ one `AtomicLong` field, no heap allocations. == 5. Common Pitfalls -* **Micro vs Nano confusion** – `currentTimeNanos()` *may* return a value that is *not* strictly - greater than the previous nano call if both happen inside the same microsecond. If you need - monotonic **ns** resolution, add another CAS layer or use `DistributedUniqueTimeProvider`. -* **Clock leap-back** – When the system clock jumps backwards by > 1 s the provider will “run - hot” (issue future-looking timestamps) until real time catches up. This is usually harmless but +* *Micro vs Nano confusion* – `currentTimeNanos()` _may_ return a value that is _not_ strictly +greater than the previous nano call if both happen inside the same microsecond. +If you need + monotonic *ns* resolution, add another CAS layer or use `DistributedUniqueTimeProvider`. +* *Clock leap-back* – When the system clock jumps backwards by > 1 s the provider will “run +hot” (issue future-looking timestamps) until real time catches up. +This is usually harmless but can surprise log readers. -* **Mis-configuring tests** – Using `SetTimeProvider` under `UniqueMicroTimeProvider` can break +* *Mis-configuring tests* – Using `SetTimeProvider` under `UniqueMicroTimeProvider` can break determinism unless you freeze the backing clock (`autoIncrement(0, NANOSECONDS)`). == 6. Usage Examples @@ -89,8 +91,8 @@ SingleChronicleQueue queue = SingleChronicleQueueBuilder == 7. Testing Strategy -* **Unit tests** – Chronicle Core already contains `UniqueMicroTimeProviderTest` which spawns 50×50 +* *Unit tests* – Chronicle Core already contains `UniqueMicroTimeProviderTest` which spawns 50×50 threads and checks for uniqueness across 1 000 000 calls. -* **Stress** – Use JLBH with a synthetic workload to assert that 99.99-ile latency ≤ 150 ns on your hardware. -* **Fault injection** – Wrap a `SetTimeProvider` that deliberately moves backwards every 10 µs and +* *Stress* – Use JLBH with a synthetic workload to assert that 99.99-ile latency ≤ 150 ns on your hardware. +* *Fault injection* – Wrap a `SetTimeProvider` that deliberately moves backwards every 10 µs and verify the returned sequence is still strictly increasing. diff --git a/src/main/docs/value-interfaces-guide.adoc b/src/main/docs/value-interfaces-guide.adoc index ea5c4b19867..7584fe68fa9 100644 --- a/src/main/docs/value-interfaces-guide.adoc +++ b/src/main/docs/value-interfaces-guide.adoc @@ -33,9 +33,9 @@ These interfaces represent a single mutable value. === 2.1. `BooleanValue` Represents a single boolean value. -* **Key Methods**: -** `boolean getValue()` -** `void setValue(boolean value)` +Key Methods :: +* `boolean getValue()` +* `void setValue(boolean value)` [source,java] ---- @@ -47,54 +47,54 @@ Represents a single boolean value. === 2.2. `ByteValue` Represents a single 8-bit byte value. -* **Key Methods**: -** `byte getValue()` -** `void setValue(byte value)` -** `byte addValue(byte delta)` +Key Methods :: +* `byte getValue()` +* `void setValue(byte value)` +* `byte addValue(byte delta)` === 2.3. `CharValue` Represents a single 16-bit char value. -* **Key Methods**: -** `char getValue()` -** `void setValue(char value)` +Key Methods :: +* `char getValue()` +* `void setValue(char value)` === 2.4. `ShortValue` Represents a single 16-bit short value. -* **Key Methods**: -** `short getValue()` -** `void setValue(short value)` -** `short addValue(short delta)` +Key Methods :: +* `short getValue()` +* `void setValue(short value)` +* `short addValue(short delta)` === 2.5. `IntValue` Represents a single 32-bit integer value. Implements `java.io.Closeable`. -* **Key Methods**: -** `int getValue()` -** `void setValue(int value)` -** `int getVolatileValue()` -** `void setOrderedValue(int value)` -** `int addValue(int delta)` -** `int addAtomicValue(int delta)` -** `boolean compareAndSwapValue(int expected, int value)` +Key Methods :: +* `int getValue()` +* `void setValue(int value)` +* `int getVolatileValue()` +* `void setOrderedValue(int value)` +* `int addValue(int delta)` +* `int addAtomicValue(int delta)` +* `boolean compareAndSwapValue(int expected, int value)` === 2.6. `LongValue` `LongValue` represents a single 64-bit value. Implements `java.io.Closeable`. In addition to normal get and set methods, it provides atomic operations and ordered writes for concurrent access scenarios. -* **Key Methods**: -** `long getValue()` -** `void setValue(long value)` -** `long getVolatileValue()` / `getVolatileValue(long closedValue)` -** `void setVolatileValue(long value)` -** `void setOrderedValue(long value)` -** `long addValue(long delta)` -** `long addAtomicValue(long delta)` (typically delegates to `addValue` or provides stronger atomicity) -** `boolean compareAndSwapValue(long expected, long value)` -** `void setMaxValue(long value)` / `void setMinValue(long value)` +Key Methods :: +* `long getValue()` +* `void setValue(long value)` +* `long getVolatileValue()` / `getVolatileValue(long closedValue)` +* `void setVolatileValue(long value)` +* `void setOrderedValue(long value)` +* `long addValue(long delta)` +* `long addAtomicValue(long delta)` (typically delegates to `addValue` or provides stronger atomicity) +* `boolean compareAndSwapValue(long expected, long value)` +* `void setMaxValue(long value)` / `void setMinValue(long value)` [source,java] ---- @@ -108,31 +108,31 @@ boolean swapped = value.compareAndSwapValue(42L, 100L); === 2.7. `FloatValue` Represents a single 32-bit float value. -* **Key Methods**: -** `float getValue()` -** `void setValue(float value)` -** `float getVolatileValue()` -** `void setOrderedValue(float value)` -** `float addValue(float delta)` -** `float addAtomicValue(float delta)` +Key Methods :: +* `float getValue()` +* `void setValue(float value)` +* `float getVolatileValue()` +* `void setOrderedValue(float value)` +* `float addValue(float delta)` +* `float addAtomicValue(float delta)` === 2.8. `DoubleValue` Represents a single 64-bit double value. -* **Key Methods**: -** `double getValue()` -** `void setValue(double value)` -** `double getVolatileValue()` -** `void setOrderedValue(double value)` -** `double addValue(double delta)` -** `double addAtomicValue(double delta)` +Key Methods :: +* `double getValue()` +* `void setValue(double value)` +* `double getVolatileValue()` +* `void setOrderedValue(double value)` +* `double addValue(double delta)` +* `double addAtomicValue(double delta)` === 2.9. `StringValue` `StringValue` models a mutable String, often stored in direct memory. -* **Key Methods**: -** `String getValue()` -** `void setValue(@MaxBytes CharSequence value)`: Sets the string. The `@MaxBytes` annotation can be used by implementations to enforce a maximum encoded size (see section below). -** `StringBuilder getUsingValue(StringBuilder stringBuilder)`: Copies the value into a supplied `StringBuilder` to potentially avoid new String allocations. +Key Methods :: +* `String getValue()` +* `void setValue(@MaxBytes CharSequence value)`: Sets the string. The `@MaxBytes` annotation can be used by implementations to enforce a maximum encoded size (see section below). +* `StringBuilder getUsingValue(StringBuilder stringBuilder)`: Copies the value into a supplied `StringBuilder` to potentially avoid new String allocations. [source,java] ---- @@ -148,15 +148,15 @@ Represents a single 64-bit double value. `TwoLongValue` extends `LongValue` to represent a pair of 64-bit values. It inherits all methods for the first long value and adds corresponding methods for the second. -* **Key Additional Methods**: -** `long getValue2()` -** `void setValue2(long value2)` -** `long getVolatileValue2()` -** `void setVolatileValue2(long value)` -** `void setOrderedValue2(long value)` -** `long addValue2(long delta)` -** `long addAtomicValue2(long delta)` -** `boolean compareAndSwapValue2(long expected, long value)` +Key Additional Methods :: +* `long getValue2()` +* `void setValue2(long value2)` +* `long getVolatileValue2()` +* `void setVolatileValue2(long value)` +* `void setOrderedValue2(long value)` +* `long addValue2(long delta)` +* `long addAtomicValue2(long delta)` +* `boolean compareAndSwapValue2(long expected, long value)` == 3. Array Value Interfaces @@ -166,32 +166,32 @@ They implement `java.io.Closeable`. === 3.1. `IntArrayValues` Represents an array of 32-bit integer values. -* **Key Methods**: -** `long getCapacity()` -** `long getUsed()` / `void setMaxUsed(long usedAtLeast)` -** `int getValueAt(long index)` -** `void setValueAt(long index, int value)` -** `int getVolatileValueAt(long index)` -** `void setOrderedValueAt(long index, int value)` -** `boolean compareAndSet(long index, int expected, int value)` -** `void bindValueAt(long index, IntValue value)` (binds a reusable `IntValue` to an element) -** `long sizeInBytes(long capacity)` -** `boolean isNull()` / `void reset()` +Key Methods :: +* `long getCapacity()` +* `long getUsed()` / `void setMaxUsed(long usedAtLeast)` +* `int getValueAt(long index)` +* `void setValueAt(long index, int value)` +* `int getVolatileValueAt(long index)` +* `void setOrderedValueAt(long index, int value)` +* `boolean compareAndSet(long index, int expected, int value)` +* `void bindValueAt(long index, IntValue value)` (binds a reusable `IntValue` to an element) +* `long sizeInBytes(long capacity)` +* `boolean isNull()` / `void reset()` === 3.2. `LongArrayValues` `LongArrayValues` provides a typed view over an array of 64-bit long values. -* **Key Methods**: -** `long getCapacity()` -** `long getUsed()` / `void setUsed(long used)` / `void setMaxUsed(long usedAtLeast)` -** `long getValueAt(long index)` -** `void setValueAt(long index, long value)` -** `long getVolatileValueAt(long index)` -** `void setOrderedValueAt(long index, long value)` -** `boolean compareAndSet(long index, long expected, long value)` -** `void bindValueAt(long index, LongValue value)` (binds a reusable `LongValue` to an element) -** `long sizeInBytes(long capacity)` -** `boolean isNull()` / `void reset()` +Key Methods :: +* `long getCapacity()` +* `long getUsed()` / `void setUsed(long used)` / `void setMaxUsed(long usedAtLeast)` +* `long getValueAt(long index)` +* `void setValueAt(long index, long value)` +* `long getVolatileValueAt(long index)` +* `void setOrderedValueAt(long index, long value)` +* `boolean compareAndSet(long index, long expected, long value)` +* `void bindValueAt(long index, LongValue value)` (binds a reusable `LongValue` to an element) +* `long sizeInBytes(long capacity)` +* `boolean isNull()` / `void reset()` [source,java] ---- diff --git a/src/main/java/net/openhft/chronicle/core/Jvm.java b/src/main/java/net/openhft/chronicle/core/Jvm.java index 2a53e379efc..6fd2490ced6 100644 --- a/src/main/java/net/openhft/chronicle/core/Jvm.java +++ b/src/main/java/net/openhft/chronicle/core/Jvm.java @@ -42,6 +42,7 @@ import java.net.URL; import java.net.URLClassLoader; import java.nio.ByteBuffer; +import java.nio.charset.Charset; import java.nio.channels.FileChannel; import java.nio.channels.spi.AbstractInterruptibleChannel; import java.nio.file.Paths; @@ -1397,7 +1398,7 @@ private static boolean isProcessAlive0(final long pid, final String command) { try { InputStreamReader isReader = new InputStreamReader( - getRuntime().exec(command).getInputStream()); + getRuntime().exec(command).getInputStream(), Charset.defaultCharset()); final BufferedReader bReader = new BufferedReader(isReader); String strLine; @@ -1715,9 +1716,11 @@ static class ReserveMemoryHolder { } else { reservedMemoryGetter = ThrowingSupplier.asSupplier(() -> f.getLong(null)); } - } catch (Exception e) { - if (MAX_DIRECT_MEMORY > 0) + } catch (ReflectiveOperationException | SecurityException e) { + if (MAX_DIRECT_MEMORY > 0) { + Jvm.warn().on(Jvm.class, "Unable to determine reserved direct memory", e); System.err.println(Jvm.class.getName() + ": Unable to determine the reservedMemory value, will always report 0"); + } reservedMemoryGetter = () -> 0L; } reservedMemory = reservedMemoryGetter; diff --git a/src/main/java/net/openhft/chronicle/core/LicenceCheck.java b/src/main/java/net/openhft/chronicle/core/LicenceCheck.java index 61dd8540deb..5bf1fb9d729 100644 --- a/src/main/java/net/openhft/chronicle/core/LicenceCheck.java +++ b/src/main/java/net/openhft/chronicle/core/LicenceCheck.java @@ -21,6 +21,7 @@ import net.openhft.chronicle.core.io.IOTools; import javax.naming.TimeLimitExceededException; +import java.nio.charset.StandardCharsets; import java.time.LocalDate; import java.time.temporal.ChronoUnit; import java.util.function.BiConsumer; @@ -79,7 +80,7 @@ static void licenceExpiry(String product, Class caller, BiConsumer line.startsWith("model name")) .map(removingTag()) .findFirst().orElse(model); @@ -48,7 +50,7 @@ public final class CpuClass { Process process = new ProcessBuilder(cmd.split(" ")) .redirectErrorStream(true) .start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.defaultCharset()))) { model = reader.lines() .map(String::trim) .filter(s -> !"Name".equals(s) && !s.isEmpty()) @@ -71,7 +73,7 @@ public final class CpuClass { Process process = new ProcessBuilder(cmd.split(" ")) .redirectErrorStream(true) .start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charset.defaultCharset()))) { model = reader.lines() .map(String::trim) .filter(s -> s.startsWith("machdep.cpu.brand_string")) diff --git a/src/main/java/net/openhft/chronicle/core/io/Wget.java b/src/main/java/net/openhft/chronicle/core/io/Wget.java index c8f010f8dcd..3e6db02716b 100644 --- a/src/main/java/net/openhft/chronicle/core/io/Wget.java +++ b/src/main/java/net/openhft/chronicle/core/io/Wget.java @@ -22,9 +22,11 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.HttpURLConnection; +import java.net.InetAddress; import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; +import java.net.UnknownHostException; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.Objects; @@ -55,6 +57,7 @@ public static final class Builder { private int connectTimeoutMs = 10_000; private int readTimeoutMs = 10_000; private long maxResponseBytes = 10L << 20; // 10 MiB + private boolean requirePublicEndpoints = false; public Builder connectionProvider(final ConnectionProvider p) { this.connectionProvider = Objects.requireNonNull(p); return this; } public Builder charsetDetector (final CharsetDetector d) { this.charsetDetector = Objects.requireNonNull(d); return this; } @@ -63,9 +66,29 @@ public static final class Builder { public Builder maxResponseBytes (final long v) { if (v < 0) throw new IllegalArgumentException("maxResponseBytes must be >= 0"); this.maxResponseBytes = v; return this; } /** Creates a {@link Wget} with defaults or caller-supplied overrides. */ + /** + * @deprecated Local hosts are now permitted by default. Use {@link #requirePublicEndpoints()} + * if you need to restrict connections to non-local addresses. + */ + @Deprecated + public Builder allowLocalHosts() { + this.requirePublicEndpoints = false; + return this; + } + + /** + * Instructs the resulting {@link Wget} instance to reject loopback, link-local and site-local + * hosts whenever the default connection provider is used. + */ + public Builder requirePublicEndpoints() { + this.requirePublicEndpoints = true; + return this; + } + public Wget build() { + final boolean usingDefaultProvider = this.connectionProvider == DEFAULT_PROVIDER; ConnectionProvider cp = this.connectionProvider; - if (cp == DEFAULT_PROVIDER) { // wrap default provider to apply time-outs + if (usingDefaultProvider) { // wrap default provider to apply time-outs final int ct = connectTimeoutMs; final int rt = readTimeoutMs; cp = url -> { @@ -77,20 +100,24 @@ public Wget build() { return conn.getInputStream(); }; } - return new Wget(cp, charsetDetector, maxResponseBytes); + final boolean enforce = usingDefaultProvider && requirePublicEndpoints; + return new Wget(cp, charsetDetector, maxResponseBytes, enforce); } } private final ConnectionProvider connectionProvider; private final CharsetDetector charsetDetector; private final long maxResponseBytes; + private final boolean enforcePublicEndpoints; private Wget(final ConnectionProvider cp, final CharsetDetector cd, - final long maxBytes) { + final long maxBytes, + final boolean enforcePublicEndpoints) { this.connectionProvider = cp; this.charsetDetector = cd; this.maxResponseBytes = maxBytes; + this.enforcePublicEndpoints = enforcePublicEndpoints; } private static final int MAX_URL_LENGTH = 2_048; @@ -109,6 +136,8 @@ public void fetch(final String url, final Appendable out) throws IOException { final String scheme = u.getProtocol(); if (!"http".equals(scheme) && !"https".equals(scheme)) throw new MalformedURLException("Only http/https allowed, not " + scheme); + if (enforcePublicEndpoints) + enforcePublicEndpoint(u); try (InputStream raw = connectionProvider.open(u); InputStream limited = new LimitedInputStream(raw, maxResponseBytes)) { @@ -121,4 +150,25 @@ public void fetch(final String url, final Appendable out) throws IOException { out.append((char) ch); } } + + /** + * Rejects URLs that resolve to loopback, link-local, site-local or wildcard addresses. + * Callers that genuinely need to talk to such endpoints can supply a custom + * {@link ConnectionProvider}. + */ + private static void enforcePublicEndpoint(final URL url) throws MalformedURLException { + final String host = url.getHost(); + if (host == null || host.isEmpty()) + throw new MalformedURLException("URL must include a host"); + try { + final InetAddress resolved = InetAddress.getByName(host); + if (resolved.isAnyLocalAddress() || + resolved.isLoopbackAddress() || + resolved.isLinkLocalAddress() || + resolved.isSiteLocalAddress()) + throw new MalformedURLException("Refusing to connect to non-public host: " + host); + } catch (UnknownHostException e) { + throw new MalformedURLException("Unable to resolve host: " + host); + } + } } diff --git a/src/test/java/net/openhft/chronicle/core/JvmUtilitiesTest.java b/src/test/java/net/openhft/chronicle/core/JvmUtilitiesTest.java new file mode 100644 index 00000000000..eac19fb468b --- /dev/null +++ b/src/test/java/net/openhft/chronicle/core/JvmUtilitiesTest.java @@ -0,0 +1,101 @@ +package net.openhft.chronicle.core; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class JvmUtilitiesTest { + + private static final String SIZE_PROPERTY = "chronicle.core.test.size"; + private static final String FLAG_PROPERTY = "chronicle.core.test.flag"; + + @AfterEach + void clearTestProperties() { + System.clearProperty(SIZE_PROPERTY); + System.clearProperty(FLAG_PROPERTY); + } + + @Test + void majorVersionMatchesSpecification() { + assertEquals(detectMajorVersionFromSpecification(), Jvm.majorVersion()); + } + + @Test + void getBooleanUnderstandsEmptyAndFalseValues() { + System.clearProperty(FLAG_PROPERTY); + assertFalse(Jvm.getBoolean(FLAG_PROPERTY)); + + System.setProperty(FLAG_PROPERTY, ""); + assertTrue(Jvm.getBoolean(FLAG_PROPERTY)); + + System.setProperty(FLAG_PROPERTY, "true"); + assertTrue(Jvm.getBoolean(FLAG_PROPERTY)); + + System.setProperty(FLAG_PROPERTY, "false"); + assertFalse(Jvm.getBoolean(FLAG_PROPERTY, true)); + } + + @Test + void parseSizeSupportsBinarySuffixes() { + assertEquals(1024L, Jvm.parseSize("1k")); + assertEquals(1536L, Jvm.parseSize("1.5K")); + assertEquals(1_048_576L, Jvm.parseSize("1MiB")); + assertEquals(1L, Jvm.parseSize("1")); + } + + @Test + void parseSizeRejectsUnknownSuffix() { + assertThrows(IllegalArgumentException.class, () -> Jvm.parseSize("10XB")); + } + + @Test + void getSizeReturnsParsedValueOrDefault() { + System.clearProperty(SIZE_PROPERTY); + assertEquals(512L, Jvm.getSize(SIZE_PROPERTY, 512L)); + + System.setProperty(SIZE_PROPERTY, "2M"); + assertEquals(2 * (1L << 20), Jvm.getSize(SIZE_PROPERTY, 0L)); + + System.setProperty(SIZE_PROPERTY, "notanumber"); + assertEquals(256L, Jvm.getSize(SIZE_PROPERTY, 256L)); + } + + @Test + void getPropertyFallsBackToDefault() { + System.clearProperty("chronicle.test.prop"); + assertEquals("fallback", Jvm.getProperty("chronicle.test.prop", "fallback")); + + System.setProperty("chronicle.test.prop", "value"); + assertEquals("value", Jvm.getProperty("chronicle.test.prop", "fallback")); + } + + @Test + void getLongReturnsParsedValueOrDefault() { + System.clearProperty("chronicle.test.long"); + assertEquals(Long.valueOf(42L), Jvm.getLong("chronicle.test.long", 42L)); + + System.setProperty("chronicle.test.long", "1234"); + assertEquals(Long.valueOf(1234L), Jvm.getLong("chronicle.test.long", 0L)); + } + + @Test + void resourceTracingToggles() { + boolean original = Jvm.isResourceTracing(); + try { + Jvm.setResourceTracing(!original); + assertEquals(!original, Jvm.isResourceTracing()); + } finally { + Jvm.setResourceTracing(original); + } + } + + private int detectMajorVersionFromSpecification() { + String specVersion = System.getProperty("java.specification.version", "8"); + int dotIndex = specVersion.indexOf('.'); + if (dotIndex >= 0) { + return Integer.parseInt(specVersion.substring(dotIndex + 1)); + } + return Integer.parseInt(specVersion); + } +} diff --git a/src/test/java/net/openhft/chronicle/core/internal/invariant/longs/LongConditionTest.java b/src/test/java/net/openhft/chronicle/core/internal/invariant/longs/LongConditionTest.java new file mode 100644 index 00000000000..7932612403f --- /dev/null +++ b/src/test/java/net/openhft/chronicle/core/internal/invariant/longs/LongConditionTest.java @@ -0,0 +1,65 @@ +package net.openhft.chronicle.core.internal.invariant.longs; + +import org.junit.jupiter.api.Test; + +import java.util.function.LongPredicate; + +import static org.junit.jupiter.api.Assertions.*; + +class LongConditionTest { + + @Test + void evaluatesBasicPredicates() { + assertTrue(LongCondition.POSITIVE.test(5)); + assertFalse(LongCondition.POSITIVE.test(0)); + + assertTrue(LongCondition.NEGATIVE.test(-3)); + assertFalse(LongCondition.NEGATIVE.test(1)); + + assertTrue(LongCondition.ZERO.test(0)); + assertFalse(LongCondition.ZERO.test(2)); + } + + @Test + void rangeBasedConditionsRespectBounds() { + assertTrue(LongCondition.BYTE_CONVERTIBLE.test(Byte.MIN_VALUE)); + assertTrue(LongCondition.BYTE_CONVERTIBLE.test(Byte.MAX_VALUE)); + assertFalse(LongCondition.BYTE_CONVERTIBLE.test(Byte.MIN_VALUE - 1L)); + + assertTrue(LongCondition.SHORT_CONVERTIBLE.test(Short.MAX_VALUE)); + assertFalse(LongCondition.SHORT_CONVERTIBLE.test(Short.MAX_VALUE + 1L)); + } + + @Test + void powerOfTwoConditionRecognisesSingleBits() { + assertTrue(LongCondition.EVEN_POWER_OF_TWO.test(1L << 10)); + assertFalse(LongCondition.EVEN_POWER_OF_TWO.test(0)); + assertFalse(LongCondition.EVEN_POWER_OF_TWO.test(3)); + } + + @Test + void alignmentConditionsRequireExpectedMask() { + assertTrue(LongCondition.SHORT_ALIGNED.test(8)); + assertFalse(LongCondition.SHORT_ALIGNED.test(3)); + + assertTrue(LongCondition.INT_ALIGNED.test(16)); + assertFalse(LongCondition.INT_ALIGNED.test(10)); + + assertTrue(LongCondition.LONG_ALIGNED.test(32)); + assertFalse(LongCondition.LONG_ALIGNED.test(18)); + } + + @Test + void negateReturnsMatchingCounterparts() { + assertSame(LongCondition.NON_POSITIVE, LongCondition.POSITIVE.negate()); + assertSame(LongCondition.NON_NEGATIVE, LongCondition.NEGATIVE.negate()); + assertSame(LongCondition.NON_ZERO, LongCondition.ZERO.negate()); + assertSame(LongCondition.POSITIVE, LongCondition.NON_POSITIVE.negate()); + assertSame(LongCondition.NEGATIVE, LongCondition.NON_NEGATIVE.negate()); + assertSame(LongCondition.ZERO, LongCondition.NON_ZERO.negate()); + + LongPredicate negated = LongCondition.EVEN_POWER_OF_TWO.negate(); + assertFalse(negated.test(8)); + assertTrue(negated.test(3)); + } +} diff --git a/src/test/java/net/openhft/chronicle/core/io/WgetTest.java b/src/test/java/net/openhft/chronicle/core/io/WgetTest.java index 080abb8e0de..0e45def765a 100644 --- a/src/test/java/net/openhft/chronicle/core/io/WgetTest.java +++ b/src/test/java/net/openhft/chronicle/core/io/WgetTest.java @@ -4,6 +4,7 @@ import java.io.ByteArrayInputStream; import java.io.IOException; +import java.net.MalformedURLException; import java.nio.charset.StandardCharsets; import static org.junit.jupiter.api.Assertions.*; @@ -200,6 +201,14 @@ void charset_detector_exception_bubbles_up() { assertThrows(RuntimeException.class, () -> wget.fetch("http://x", new StringBuilder())); } + @Test + void require_public_endpoints_blocks_loopback() { + Wget wget = new Wget.Builder() + .requirePublicEndpoints() + .build(); + assertThrows(MalformedURLException.class, () -> wget.fetch("http://127.0.0.1", new StringBuilder())); + } + @Test void default_provider_sets_timeouts() throws Exception { AtomicInteger seenConnect = new AtomicInteger(-1); diff --git a/src/test/java/net/openhft/chronicle/core/pool/StringBuilderPoolTest.java b/src/test/java/net/openhft/chronicle/core/pool/StringBuilderPoolTest.java new file mode 100644 index 00000000000..d7f07c8d900 --- /dev/null +++ b/src/test/java/net/openhft/chronicle/core/pool/StringBuilderPoolTest.java @@ -0,0 +1,74 @@ +package net.openhft.chronicle.core.pool; + +import net.openhft.chronicle.core.scoped.ScopedResource; +import net.openhft.chronicle.core.scoped.ScopedResourcePool; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; + +class StringBuilderPoolTest { + + @Test + void createThreadLocalProvidesPool() { + ScopedResourcePool pool = StringBuilderPool.createThreadLocal(); + assertNotNull(pool); + } + + @Test + void createThreadLocalWithCapacityProvidesPool() { + ScopedResourcePool pool = StringBuilderPool.createThreadLocal(2); + assertNotNull(pool); + } + + @Test + void reusesBuilderWithinThreadAndClearsContent() { + ScopedResourcePool pool = StringBuilderPool.createThreadLocal(1); + + StringBuilder firstBuilder; + try (ScopedResource resource = pool.get()) { + StringBuilder builder = resource.get(); + builder.append("Chronicle"); + firstBuilder = builder; + } + + try (ScopedResource resource = pool.get()) { + StringBuilder builder = resource.get(); + assertSame(firstBuilder, builder); + assertEquals(0, builder.length(), "Builder should be cleared before reuse"); + } + } + + @Test + void suppliesIndependentBuildersPerThread() throws InterruptedException { + ScopedResourcePool pool = StringBuilderPool.createThreadLocal(1); + + AtomicReference mainThreadBuilder = new AtomicReference<>(); + try (ScopedResource resource = pool.get()) { + mainThreadBuilder.set(resource.get()); + } + + CountDownLatch complete = new CountDownLatch(1); + AtomicReference otherThreadBuilder = new AtomicReference<>(); + Thread worker = new Thread(() -> { + try (ScopedResource resource = pool.get()) { + otherThreadBuilder.set(resource.get()); + } finally { + complete.countDown(); + } + }); + worker.start(); + + assertTrue(complete.await(5, TimeUnit.SECONDS), "Worker thread did not finish in time"); + worker.join(1000L); + assertNotNull(otherThreadBuilder.get()); + assertNotSame(mainThreadBuilder.get(), otherThreadBuilder.get(), "Builders must be isolated per thread"); + + try (ScopedResource resource = pool.get()) { + assertSame(mainThreadBuilder.get(), resource.get(), "Main thread should regain its original builder"); + } + } +}