Merge pull request #183 from getindata/release-0.12.0

Release 0.12.0

Author: em-pe

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.11.1
+current_version = 0.12.0
 
 [bumpversion:file:pyproject.toml]
 

.copier-answers.yml

@@ -10,7 +10,7 @@ description: Kedro plugin with GCP Vertex AI support
 docs_url: https://kedro-vertexai.readthedocs.io/
 full_name: Kedro Vertex AI Plugin
 github_url: https://github.com/getindata/kedro-vertexai
-initial_version: 0.11.1
+initial_version: 0.12.0
 keywords:
 - kedro-plugin
 - kedro

.github/workflows/prepare-release.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8]
+        python-version: ['3.10']
     env:
       PYTHON_PACKAGE: kedro_vertexai
     steps:

.github/workflows/test_and_publish.yml

@@ -17,8 +17,9 @@ jobs:
   unit_tests:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10']
+        python-version: ['3.9', '3.10', '3.11', '3.12']
 
     steps:
     - uses: actions/checkout@v4
@@ -31,7 +32,7 @@ jobs:
     - name: Setup poetry
       run: |
         python -V
-        python -m pip install poetry
+        python -m pip install poetry --no-cache-dir
 
     - name: Check pre-commit status
       run: |
@@ -44,7 +45,7 @@ jobs:
         tox -v
 
     - name: Store coverage reports
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: coverage-${{ matrix.python-version }}
         path: coverage.xml
@@ -59,7 +60,7 @@ jobs:
       with:
         fetch-depth: 0
 
-    - uses: actions/download-artifact@v3
+    - uses: actions/download-artifact@v4
       with:
         name: coverage-3.9
         path: .
@@ -86,15 +87,15 @@ jobs:
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v2
+        uses: github/codeql-action/init@v3
         with:
           languages: python
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v2
+        uses: github/codeql-action/autobuild@v3
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v2
+        uses: github/codeql-action/analyze@v3
 
   ### PROJECT SPECIFIC CONFIGURATION HERE
   e2e_tests:
@@ -105,7 +106,7 @@ jobs:
     strategy:
       matrix:
         e2e_case: ["standard", "grouping"]
-        python-version: ['3.8'] # todo update python
+        python-version: ['3.10']
     env: 
       PROJECT_NAME: kedro-vertexai
       IMAGE_REGISTRY: gcr.io/gid-ml-ops-sandbox
@@ -191,7 +192,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8']
+        python-version: ['3.10']
     env:
       PYTHON_PACKAGE: kedro_vertexai
     steps:

.readthedocs.yaml

@@ -9,7 +9,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.8"
+    python: "3.10"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 ## [Unreleased]
 
+## [0.12.0] - 2025-03-11
+
+-   Support for python 3.11 & 3.12 added, dropped support for python 3.8
+-   Brought back the Vertex AI Pipelines scheduling capability
+-   Migrated to kfp 2 (locked to <2.9.0  due to pipeline spec changes)
+-   Removed `image_pull_policy` parameter from configuration, as it only applies to Kubernetes backend and not Vertex AI,
+    and it's only available in `kfp-kubernetes` extension package
+-   Removed `--timeout-seconds` parameter from `run-once` command for now, as in the old version of the plugin exceeding the specified time
+    didn't alter the remote pipeline execution, and only escaped the local Python processs. The timeout funcionality will be added later on,
+    with the proper remote pipeline execution handling, and possibly per-task timeout enabled by [the new kfp feature](https://github.com/kubeflow/pipelines/pull/10481).
+-   Assign pipelines to Vertex AI experiments
+-   Migrated `pydantic` library to v2
+-   Custom dataset that creates Vertex AI artifact
+-   Added pipeline parametrization
+
 ## [0.11.1] - 2024-07-01
 
 ## [0.11.0] - 2024-03-22
@@ -83,7 +98,9 @@
 
 -   Initial version of **kedro-vertexai** plugin extracted from [kedro-kubeflow v0.6.0](https://github.com/getindata/kedro-kubeflow/tree/0.6.0)
 
-[Unreleased]: https://github.com/getindata/kedro-vertexai/compare/0.11.1...HEAD
+[Unreleased]: https://github.com/getindata/kedro-vertexai/compare/0.12.0...HEAD
+
+[0.12.0]: https://github.com/getindata/kedro-vertexai/compare/0.11.1...0.12.0
 
 [0.11.1]: https://github.com/getindata/kedro-vertexai/compare/0.11.0...0.11.1
 

docs/source/02_installation/02_configuration.md

@@ -10,16 +10,15 @@ run_config:
   # Name of the image to run as the pipeline steps
   image: eu.gcr.io/my-gcp-mlops-project/example_model:${oc.env:KEDRO_CONFIG_COMMIT_ID}
 
-  # Pull policy to be used for the steps. Use Always if you push the images
-  # on the same tag, or Never if you use only local images
-  image_pull_policy: IfNotPresent
-
   # Location of Vertex AI GCS root
   root: bucket_name/gcs_suffix
 
-  # Name of the kubeflow experiment to be created
+  # Name of the Vertex AI experiment to be created
   experiment_name: MyExperiment
 
+  # Optional description of the Vertex AI experiment to be created
+  # experiment_description: "My experiment description."
+
   # Name of the scheduled run, templated with the schedule parameters
   scheduled_run_name: MyExperimentRun
 
@@ -98,6 +97,23 @@ run_config:
   #      client_id: iam-client-id
 
   dynamic_config_providers: []
+
+  # Schedules configuration
+  schedules:
+    default_schedule:
+      cron_expression: "0 * * * *"
+      timezone: Etc/UTC
+      # Optional. Timestamp after which the first run can be scheduled. If unspecified, it defaults to the schedule creation timestamp.
+      start_time: null
+      # Optional. Timestamp after which no more runs will be scheduled. If unspecified, then runs will be scheduled indefinitely.
+      end_time: null
+      # Optional. Whether new scheduled runs can be queued when max_concurrent_runs limit is reached.
+      allow_queueing: false
+      # Optional. Maximum run count of the schedule. If specified, The schedule will be completed when either started_run_count >= max_run_count or when end_time is reached. Must be positive and <= 2^63-1.
+      max_run_count: null
+      # Optional. Maximum number of runs that can be started concurrently for this PipelineJobSchedule.
+      max_concurrent_run_count: 1
+
 ```
 
 ## Dynamic configuration support

docs/source/03_getting_started/01_quickstart.md

@@ -108,17 +108,14 @@ Adjusted `catalog.yml` should look like this (note: remove the rest of the entri
 companies:
   type: pandas.CSVDataSet
   filepath: data/01_raw/companies.csv
-  layer: raw
 
 reviews:
   type: pandas.CSVDataSet
   filepath: data/01_raw/reviews.csv
-  layer: raw
 
 shuttles:
   type: pandas.ExcelDataSet
   filepath: data/01_raw/shuttles.xlsx
-  layer: raw
 ```
 
 All intermediate and output data will be stored in the location with the following pattern:
@@ -180,3 +177,9 @@ As you can see, the pipeline was compiled and started in Vertex AI Pipelines. Wh
 ![Kedro pipeline running in Vertex AI Pipelines](vertexai_running_pipeline.gif)
 
 
+## Log datasets to Vertex AI Metadata
+
+The plugin implements custom `kedro_vertexai.vertex_ai.datasets.KedroVertexAIMetadataDataset` dataset that creates an Vertex AI Artifact.
+It allows to specify any Kedro dataset in the `base_dataset` argument, and it uses its `_save` and `_load` methods for the io.
+The base dataset arguments are passed in `base_dataset_args` argument as dictionary. The created artifact is associated with Vertex AI run id and job name as metadata, and additional metadata can be specified in the `metadata` argument.
+The `dispaly_name` and `schema` arguments are used for the artifact creation, please reference [Vertex AI docs](https://cloud.google.com/vertex-ai/docs/ml-metadata/tracking#create-artifact) to learn more about them.

kedro_vertexai/__init__.py

@@ -1 +1 @@
-__version__ = "0.11.1"
+__version__ = "0.12.0"