feat: automate release versioning and metadata injection for CI evaluations (#163)

Author: omkargaikwad23

cloudbuild.yaml

@@ -21,17 +21,52 @@ steps:
   - name: 'us-central1-docker.pkg.dev/cloud-db-nl2sql/evalbench/eval_server:latest'
     entrypoint: 'bash'
     # Decrypts the secret from Secret Manager into the DB_PASSWORD environment variable
-    secretEnv: ['DB_PASSWORD'] 
+    secretEnv: ['DB_PASSWORD', 'GITHUB_TOKEN']
     args:
       - '-c'
       - |
         set -e
+
+        # Only run on release branches
+        if [[ "$_HEAD_BRANCH" != release-please-* ]]; then
+          echo "Not a release-please branch. Exiting."
+          exit 0
+        fi
+        echo "Release branch detected. Fetching PR data from GitHub API..."
+
+        # Fetch PR data and status code
+        HTTP_STATUS=$(curl -s -o pr_data.json -w "%{http_code}" -H "Authorization: token $$GITHUB_TOKEN" \
+          "https://api.github.com/repos/$REPO_FULL_NAME/pulls/$_PR_NUMBER")
+
+        if [ "$$HTTP_STATUS" -ne 200 ]; then
+          echo "Error fetching PR data: HTTP $$HTTP_STATUS"
+          cat pr_data.json
+          exit 1
+        fi
+
+        PR_DATA=$(cat pr_data.json)
+
+        # Extract labels and title from PR data (Use $$ to escape bash variables)
+        PR_LABELS=$(echo "$$PR_DATA" | jq -r '[.labels[].name] | join(",")')
+        PR_TITLE=$(echo "$$PR_DATA" | jq -r '.title')
+
+        # Determine Release Version (Use double quotes and $$ for bash variables)
+        if [[ "$$PR_LABELS" == *"autorelease: triggered"* ]]; then
+          if [[ "$$PR_TITLE" =~ release\ ([0-9]+\.[0-9]+\.[0-9]+) ]]; then
+            export RELEASE_VERSION="$${BASH_REMATCH[1]}"
+          else
+            export RELEASE_VERSION="unknown"
+          fi
+        else
+          export RELEASE_VERSION="unknown"
+        fi
+
         # Workaround for evalbench bug: settings are only applied if path basename matches extension ID
         ln -s /workspace /workspace/cloud-sql-postgresql
         cd /evalbench
 
         export EVAL_GCP_PROJECT_ID=$PROJECT_ID
-        export EVAL_GCP_PROJECT_REGION=us-central1
+        export EVAL_GCP_PROJECT_REGION=$_CLOUD_SQL_REGION
         export GOOGLE_CLOUD_PROJECT=$PROJECT_ID
         export CLOUD_SQL_POSTGRES_PROJECT=$PROJECT_ID
         export CLOUD_SQL_POSTGRES_INSTANCE=$_CLOUD_SQL_INSTANCE
@@ -43,6 +78,9 @@ steps:
         # Maps the decrypted DB_PASSWORD to the exact variable expected by gemini_cli and extension skills
         export CLOUD_SQL_POSTGRES_PASSWORD=$$DB_PASSWORD
 
+        # Combine CI metadata with run config
+        cat /workspace/evals/ci_metadata.yaml >> /workspace/evals/run_config.yaml
+
         # Substitute environment variables in model_config.yaml
         python3 /workspace/evals/substitute_env.py
 
@@ -58,3 +96,5 @@ availableSecrets:
   secretManager:
   - versionName: projects/$PROJECT_ID/secrets/daily-ci-evals-db-password/versions/latest
     env: 'DB_PASSWORD'
+  - versionName: projects/$PROJECT_ID/secrets/GITHUB_TOKEN/versions/latest
+    env: 'GITHUB_TOKEN'

evals/ci_metadata.yaml

@@ -0,0 +1,22 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+############################################################
+### CI Metadata (Repository Specific)
+### Note: These fields are used for version tracking in BQ
+### and are not part of the core Evalbench schema.
+############################################################
+
+extension_id: cloud-sql-postgresql
+release_version: ${RELEASE_VERSION}

evals/run_config.yaml

@@ -12,14 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-extension_id: cloud-sql-postgresql
-
 dataset_config: /workspace/evals/dataset.json
 dataset_format: gemini-cli-format
 
 orchestrator: geminicli
 model_config: /workspace/evals/model_config.yaml
-# You can reference default simulated user models provided by the evalbench repo:
 simulated_user_model_config: /workspace/evals/gemini_2.5_pro_model.yaml
 
 scorers:

evals/substitute_env.py

@@ -2,16 +2,17 @@
 import re
 
 def main():
-    yaml_path = '/workspace/evals/model_config.yaml'
-    if os.path.exists(yaml_path):
-        with open(yaml_path, 'r') as f:
-            content = f.read()
-        content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content)
-        with open(yaml_path, 'w') as f:
-            f.write(content)
-        print(f"Successfully substituted environment variables in {yaml_path}")
-    else:
-        print(f"File not found: {yaml_path}")
+    yaml_paths = ['/workspace/evals/model_config.yaml', '/workspace/evals/run_config.yaml']
+    for yaml_path in yaml_paths:
+        if os.path.exists(yaml_path):
+            with open(yaml_path, 'r') as f:
+                content = f.read()
+            content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content)
+            with open(yaml_path, 'w') as f:
+                f.write(content)
+            print(f"Successfully substituted environment variables in {yaml_path}")
+        else:
+            print(f"File not found: {yaml_path}")
 
 if __name__ == '__main__':
     main()