diff --git a/.github/workflows/performance_score_director.yml b/.github/workflows/performance_score_director.yml index 6fc9b93..068789b 100644 --- a/.github/workflows/performance_score_director.yml +++ b/.github/workflows/performance_score_director.yml @@ -50,12 +50,14 @@ jobs: outputs: baseline_solver_version: ${{ steps.step1.outputs.version }} needs_snapshot_built: ${{ steps.step1.outputs.needs_snapshot_built }} + env: + BASELINE: ${{ github.event.inputs.baseline }} steps: - name: Determine the baseline id: step1 shell: bash run: | - if [[ "${{ github.event.inputs.baseline }}" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + if [[ "$BASELINE" =~ ^v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then VERSION="${BASH_REMATCH[1]}" NEEDS_SNAPSHOT_BUILT=false echo "Baseline is a release tag ($VERSION)." @@ -77,9 +79,6 @@ jobs: matrix: # When updating this list, use find-and-replace in the entire file to keep all such lists identical. example: [cloud_balancing, conference_scheduling, curriculum_course, examination, flow_shop, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, vehicle_routing] - env: - MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}' - MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}' steps: - name: Checkout timefold-solver-benchmarks uses: actions/checkout@v6 @@ -94,9 +93,6 @@ jobs: java-version: 25 # Always build with the least recent supported JDK. distribution: 'temurin' cache: 'maven' - server-id: 'timefold-solver-enterprise' - server-username: 'MVN_USERNAME' - server-password: 'MVN_PASSWORD' # Only build the snapshots if determined by the decisions job. - name: Checkout timefold-solver @@ -129,8 +125,9 @@ jobs: if: needs.decisions.outputs.needs_snapshot_built == 'true' working-directory: ./timefold-solver-benchmarks shell: bash + env: + TARGET_BRANCH: ${{ github.event.inputs.baseline }} run: | - TARGET_BRANCH="${{ github.event.inputs.baseline }}" if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD @@ -139,8 +136,10 @@ jobs: - name: Compile the benchmark working-directory: ./timefold-solver-benchmarks shell: bash + env: + VERSION: ${{ needs.decisions.outputs.baseline_solver_version }} run: | - ./mvnw clean install -B -Dquickly -Dversion.ai.timefold.solver=${{ needs.decisions.outputs.baseline_solver_version }} + ./mvnw clean install -B -Dquickly -Dversion.ai.timefold.solver=$VERSION mv target/benchmarks.jar ../benchmarks-baseline.jar - name: Upload the binaries @@ -158,9 +157,6 @@ jobs: matrix: # When updating this list, use find-and-replace in the entire file to keep all such lists identical. example: [cloud_balancing, conference_scheduling, curriculum_course, examination, flow_shop, machine_reassignment, meeting_scheduling, nurse_rostering, patient_admission_scheduling, task_assigning, traveling_tournament, vehicle_routing] - env: - MVN_USERNAME: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_USERNAME }}' - MVN_PASSWORD: '${{ secrets.JFROG_ENTERPRISE_READ_ONLY_ACCESS_TOKEN }}' steps: - name: Checkout timefold-solver-benchmarks uses: actions/checkout@v6 @@ -175,9 +171,6 @@ jobs: java-version: 25 # Always build with the least recent supported JDK. distribution: 'temurin' cache: 'maven' - server-id: 'timefold-solver-enterprise' - server-username: 'MVN_USERNAME' - server-password: 'MVN_PASSWORD' - name: Checkout timefold-solver uses: actions/checkout@v6 @@ -201,8 +194,9 @@ jobs: - name: Switch to correct Enterprise branch if it exists working-directory: ./timefold-solver-enterprise shell: bash + env: + TARGET_BRANCH: ${{ github.event.inputs.branch }} run: | - TARGET_BRANCH="${{ github.event.inputs.branch }}" if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD @@ -218,8 +212,9 @@ jobs: - name: Switch to correct Benchmarks branch if it exists working-directory: ./timefold-solver-benchmarks shell: bash + env: + TARGET_BRANCH: ${{ github.event.inputs.branch }} run: | - TARGET_BRANCH="${{ github.event.inputs.branch }}" if git ls-remote --exit-code --heads origin "$TARGET_BRANCH" > /dev/null; then git fetch --depth=1 origin "$TARGET_BRANCH" git checkout -B "$TARGET_BRANCH" FETCH_HEAD @@ -251,14 +246,22 @@ jobs: steps: - name: Clean results of previous runs shell: bash + env: + BASELINE: ${{ github.event.inputs.baseline }} + BRANCH: ${{ github.event.inputs.branch }} run: | rm -rf timefold-solver-benchmarks # GitHub expressions can't do this, so we need to hack this in shell. # DIRs are different, so that we can run "main" against "main" and have separate results. - echo "SANITIZED_BASELINE=$(echo "${{ github.event.inputs.baseline }}" | sed 's/\//\-/g')" >> $GITHUB_ENV - echo "BASELINE_DIR=baseline_$(echo "${{ github.event.inputs.baseline }}" | sed 's/\//\-/g')" >> $GITHUB_ENV - echo "SANITIZED_BRANCH=$(echo "${{ github.event.inputs.branch }}" | sed 's/\//\-/g')" >> $GITHUB_ENV - echo "SUT_DIR=sut_$(echo "${{ github.event.inputs.branch }}" | sed 's/\//\-/g')" >> $GITHUB_ENV + # Strip CR/LF from inputs before writing to $GITHUB_ENV to prevent env-file injection. + SANITIZED_BASELINE=$(echo "$BASELINE" | tr -d '\r\n' | sed 's/\//\-/g') + SANITIZED_BRANCH=$(echo "$BRANCH" | tr -d '\r\n' | sed 's/\//\-/g') + { + echo "SANITIZED_BASELINE=$SANITIZED_BASELINE" + echo "BASELINE_DIR=baseline_$SANITIZED_BASELINE" + echo "SANITIZED_BRANCH=$SANITIZED_BRANCH" + echo "SUT_DIR=sut_$SANITIZED_BRANCH" + } >> "$GITHUB_ENV" - name: Checkout timefold-solver-benchmarks uses: actions/checkout@v6 @@ -270,13 +273,15 @@ jobs: - name: Configure the benchmark working-directory: ./timefold-solver-benchmarks shell: bash + env: + EXAMPLE: ${{ matrix.example }} run: | echo "forks=20" > scoredirector-benchmark.properties echo "warmup_iterations=5" >> scoredirector-benchmark.properties echo "measurement_iterations=5" >> scoredirector-benchmark.properties echo "relative_score_error_threshold=0.02" >> scoredirector-benchmark.properties echo "score_director_type=cs" >> scoredirector-benchmark.properties - echo "example=${{ matrix.example }}" >> scoredirector-benchmark.properties + echo "example=$EXAMPLE" >> scoredirector-benchmark.properties cat scoredirector-benchmark.properties chmod +x run-scoredirector.sh @@ -309,9 +314,9 @@ jobs: mkdir target cp benchmarks-baseline.jar target/benchmarks.jar ./run-scoredirector.sh - echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT" - echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT" - echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT" - name: (SUT) Setup JDK and Maven uses: actions/setup-java@v5 @@ -331,28 +336,32 @@ jobs: rm target/benchmarks.jar cp benchmarks-sut.jar target/benchmarks.jar ./run-scoredirector.sh - echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT" - echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT" - echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/${{ env.RUN_ID }}/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_START=$(jq '.[0].primaryMetric.scoreConfidence[0]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_END=$(jq '.[0].primaryMetric.scoreConfidence[1]|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT" + echo "RANGE_MID=$(jq '.[0].primaryMetric.score|round' results/scoredirector/$RUN_ID/results.json)" >> "$GITHUB_OUTPUT" - name: Clean up the data working-directory: ./timefold-solver-benchmarks shell: bash + env: + BASELINE_DIR: ${{ env.BASELINE_DIR }} + SUT_DIR: ${{ env.SUT_DIR }} + EXAMPLE: ${{ matrix.example }} run: | - cd results/scoredirector/${{ env.BASELINE_DIR }} - mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../${{ env.BASELINE_DIR }} + cd results/scoredirector/$BASELINE_DIR + mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../$BASELINE_DIR cd ../../../ - mv ${{ env.BASELINE_DIR }}/jfr-cpu.jfr ${{ env.BASELINE_DIR }}/${{ matrix.example }}-${{ env.BASELINE_DIR }}.jfr + mv $BASELINE_DIR/jfr-cpu.jfr $BASELINE_DIR/$EXAMPLE-$BASELINE_DIR.jfr - cd results/scoredirector/${{ env.SUT_DIR }} - mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../${{ env.SUT_DIR }} + cd results/scoredirector/$SUT_DIR + mv $(find . -maxdepth 1 -mindepth 1 -type d -print -quit) ../../../$SUT_DIR cd ../../../ - mv ${{ env.SUT_DIR }}/jfr-cpu.jfr ${{ env.SUT_DIR }}/${{ matrix.example }}-${{ env.SUT_DIR }}.jfr + mv $SUT_DIR/jfr-cpu.jfr $SUT_DIR/$EXAMPLE-$SUT_DIR.jfr - name: Archive benchmark data uses: actions/upload-artifact@v7 with: - name: assets-${{ matrix.example }}-${{ env.SANITIZED_BASELINE }}_vs_${{ env.SANITIZED_BRANCH }} + name: assets-${{ matrix.example }}-${{ env.SANITIZED_BASELINE }}-vs-${{ env.SANITIZED_BRANCH }} path: | ./timefold-solver-benchmarks/scoredirector-benchmark.properties ./timefold-solver-benchmarks/${{ env.BASELINE_DIR }}/*.jfr @@ -365,6 +374,9 @@ jobs: - name: Report results working-directory: ./timefold-solver-benchmarks env: + BASELINE: ${{ github.event.inputs.baseline }} + BRANCH: ${{ github.event.inputs.branch }} + OWNER: ${{ github.event.inputs.branch_owner }} BASELINE_RANGE_START: ${{ steps.benchmark_baseline.outputs.RANGE_START }} BASELINE_RANGE_MID: ${{ steps.benchmark_baseline.outputs.RANGE_MID }} BASELINE_RANGE_END: ${{ steps.benchmark_baseline.outputs.RANGE_END }} @@ -391,17 +403,17 @@ jobs: FAIL=true fi - if [[ "${{ github.event.inputs.baseline }}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/releases/tag/${{ github.event.inputs.baseline }}" + if [[ "$BASELINE" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/releases/tag/$BASELINE" else - BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/tree/${{ github.event.inputs.baseline }}" + BASELINE_URL="https://github.com/TimefoldAI/timefold-solver/tree/$BASELINE" fi - SUT_URL="https://github.com/${{ github.event.inputs.branch_owner }}/timefold-solver/tree/${{ github.event.inputs.branch }}" + SUT_URL="https://github.com/$OWNER/timefold-solver/tree/$BRANCH" echo "| | **Ref** | **Mean** |" >> $GITHUB_STEP_SUMMARY echo "|:------:|:-----------:|:-----------------:|" >> $GITHUB_STEP_SUMMARY - echo "| _Old_ | [TimefoldAI's ${{ github.event.inputs.baseline }}]($BASELINE_URL) | $BASELINE_RANGE_MID ± $BASELINE_DEV % |" >> $GITHUB_STEP_SUMMARY - echo "| _New_ | [${{ github.event.inputs.branch_owner }}'s ${{ github.event.inputs.branch }}]($SUT_URL) | $SUT_RANGE_MID ± $SUT_DEV % |" >> $GITHUB_STEP_SUMMARY + echo "| _Old_ | [TimefoldAI's $BASELINE]($BASELINE_URL) | $BASELINE_RANGE_MID ± $BASELINE_DEV % |" >> $GITHUB_STEP_SUMMARY + echo "| _New_ | [$OWNER's $BRANCH]($SUT_URL) | $SUT_RANGE_MID ± $SUT_DEV % |" >> $GITHUB_STEP_SUMMARY echo "| _Diff_ | | $DIFF_MID % |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY