Merge branch 'master' into change_android_gradle_kotlin_version

Author: yunhanw-google

.github/workflows/tests.yaml

@@ -546,6 +546,22 @@ jobs:
                   scripts/run_in_python_env.sh out/venv 'src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
                   scripts/run_in_python_env.sh out/venv 'scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py --all-clusters out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app'
 
+            - name: Execute Jupyter Notebooks
+              run: |
+                  scripts/run_in_build_env.sh './scripts/build_python.sh --jupyter-lab --install_virtual_env out/venv'
+                  ./scripts/run_in_build_env.sh \
+                   "./scripts/build/build_examples.py \
+                      --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-clang-test \
+                      build \
+                      --copy-artifacts-to objdir-clone \
+                   "
+                  scripts/run_in_python_env.sh out/venv \
+                  "jupyter execute docs/development_controllers/chip-repl/Matter_REPL_Intro.ipynb \
+                  docs/development_controllers/chip-repl/Matter_Basic_Interactions.ipynb \
+                  docs/development_controllers/chip-repl/Matter_Access_Control.ipynb \
+                  docs/development_controllers/chip-repl/Matter_Multi_Fabric_Commissioning.ipynb \
+                  "
+
             - name: Uploading core files
               uses: actions/upload-artifact@v4
               if: ${{ failure() && !env.ACT }}

docs/development_controllers/chip-repl/Matter_Access_Control.ipynb

@@ -103,6 +103,8 @@ To build and run the Python CHIP controller:
     chip-repl
     ```
 
+NOTE: To get more verbose logs, pass the debug flag: `chip-repl --debug`
+
 <hr>
 
 ## Using Python CHIP Controller REPL for Matter accessory testing
@@ -133,8 +135,8 @@ An uncommissioned accessory device advertises over Bluetooth LE or via mDNS if
 already on the network. Run the following command to scan all advertised Matter
 devices:
 
-```
-devCtrl.DiscoverCommissionableNodes()
+```python
+await devCtrl.DiscoverCommissionableNodes()
 ```
 
 ### Step 4: Set network pairing credentials
@@ -173,7 +175,7 @@ network interface, such as Thread or Wi-Fi.
 2. Set the previously obtained Active Operational Dataset as a byte array using
    the following command:
 
-    ```
+    ```python
     thread_dataset = bytes.fromhex("0e080000000000010000000300001335060004001fffe002084fe76e9a8b5edaf50708fde46f999f0698e20510d47f5027a414ffeebaefa92285cc84fa030f4f70656e5468726561642d653439630102e49c0410b92f8c7fbb4f9f3e08492ee3915fbd2f0c0402a0fff8")
     devCtrl.SetThreadOperationalDataset(thread_dataset)
     ```
@@ -183,7 +185,7 @@ network interface, such as Thread or Wi-Fi.
 Assuming your Wi-Fi SSID is _TESTSSID_, and your Wi-Fi password is _P455W4RD_,
 set the credentials to the controller by executing the following command:
 
-```
+```python
 devCtrl.SetWiFiCredentials(<ssid>, <password>)
 ```
 
@@ -213,8 +215,8 @@ with the following assumptions for the Matter accessory device:
 -   The setup pin code of the device is _20202021_
 -   The temporary Node ID is _1234_
 
-```
-devCtrl.ConnectBLE(3840, 20202021, 1234)
+```python
+await devCtrl.ConnectBLE(3840, 20202021, 1234)
 ```
 
 You can skip the last parameter, the Node ID, in the command. If you skip it,
@@ -230,8 +232,8 @@ CHIP:SVR: SetupQRCode: [MT:-24J0AFN00KA0648G00]
 
 Use the following command to commission the device with the QR code:
 
-```
-devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
+```python
+await devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
 ```
 
 After connecting the device over Bluetooth LE, the controller will go through
@@ -262,14 +264,14 @@ the following stages:
 For the light bulb example, execute the following command to toggle the LED
 state:
 
-```
+```python
 await devCtrl.SendCommand(1234, 1, Clusters.OnOff.Commands.Toggle())
 ```
 
 To change the brightness of the LED, use the following command, with the level
 value somewhere between 0 and 255.
 
-```
+```python
 commandToSend = LevelControl.Commands.MoveToLevel(level=50, transitionTime=Null, optionsMask=0, optionsOverride=0)
 await devCtrl.SendCommand(1234, 1, commandToSend)
 ```
@@ -281,7 +283,7 @@ maintains collection of attributes that a controller can obtain from a device,
 such as the vendor name, the product name, or software version. Use
 `ReadAttribute()` command to read those values from the device:
 
-```
+```python
 attributes = [
     (0, Clusters.BasicInformation.Attributes.VendorName),
     (0, Clusters.BasicInformation.Attributes.ProductName),
@@ -309,7 +311,7 @@ the full list of available commands.
 Provides the controller with Thread network credentials that will be used in the
 device commissioning procedure to configure the device with a Thread interface.
 
-```
+```python
 thread_dataset = bytes.fromhex("0e080000000000010000000300001335060004001fffe002084fe76e9a8b5edaf50708fde46f999f0698e20510d47f5027a414ffeebaefa92285cc84fa030f4f70656e5468726561642d653439630102e49c0410b92f8c7fbb4f9f3e08492ee3915fbd2f0c0402a0fff8")
 devCtrl.SetThreadOperationalDataset(thread_dataset)
 ```
@@ -319,7 +321,7 @@ devCtrl.SetThreadOperationalDataset(thread_dataset)
 Provides the controller with Wi-Fi network credentials that will be used in the
 device commissioning procedure to configure the device with a Wi-Fi interface.
 
-```
+```python
 devCtrl.SetWiFiCredentials('TESTSSID', 'P455W4RD')
 ```
 
@@ -328,8 +330,8 @@ devCtrl.SetWiFiCredentials('TESTSSID', 'P455W4RD')
 Commission with the given nodeid from the setupPayload. setupPayload may be a QR
 or the manual setup code.
 
-```
-devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
+```python
+await devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
 ```
 
 ### `SendCommand(<nodeid>: int, <endpoint>: int, Clusters.<cluster>.Commands.<command>(<arguments>))`

docs/development_controllers/chip-repl/Matter_Basic_Interactions.ipynb

@@ -4,14 +4,15 @@
 # jupyter-lab
 #
 # --------------------------------------
-# import chip.native
-# import pkgutil
-# module = pkgutil.get_loader('chip.ChipReplStartup')
-# %run {module.path}
+# %reset -f
+# import importlib.util
+# spec = importlib.util.find_spec('chip.ChipReplStartup')
+# %run {spec.origin}
 # --------------------------------------
 #
 
 jupyterlab
 ipykernel
 jupyterlab-lsp
 python-lsp-server
+jupyterlab-git

docs/development_controllers/chip-repl/Matter_Multi_Fabric_Commissioning.ipynb

@@ -82,10 +82,11 @@ CHIP_ERROR GenericAppendOnlyBuffer::EnsureAppendCapacity(size_t numElements)
 
     if (mBuffer == nullptr)
     {
-        mBuffer            = static_cast<uint8_t *>(Platform::MemoryCalloc(numElements, mElementSize));
+        mBuffer = static_cast<uint8_t *>(Platform::MemoryCalloc(numElements, mElementSize));
+        VerifyOrReturnError(mBuffer != nullptr, CHIP_ERROR_NO_MEMORY);
         mCapacity          = numElements;
         mBufferIsAllocated = true;
-        return mBuffer != nullptr ? CHIP_NO_ERROR : CHIP_ERROR_NO_MEMORY;
+        return CHIP_NO_ERROR;
     }
 
     // we already have the data in buffer. we have two choices:
@@ -100,9 +101,9 @@ CHIP_ERROR GenericAppendOnlyBuffer::EnsureAppendCapacity(size_t numElements)
     else
     {
         // this is NOT an allocated buffer, but it should become one
-        auto new_buffer    = static_cast<uint8_t *>(Platform::MemoryCalloc(mElementCount + numElements, mElementSize));
-        mBufferIsAllocated = true;
+        auto new_buffer = static_cast<uint8_t *>(Platform::MemoryCalloc(mElementCount + numElements, mElementSize));
         VerifyOrReturnError(new_buffer != nullptr, CHIP_ERROR_NO_MEMORY);
+        mBufferIsAllocated = true;
         memcpy(new_buffer, mBuffer, mElementCount * mElementSize);
         mBuffer = new_buffer;
     }
@@ -119,7 +120,7 @@ CHIP_ERROR GenericAppendOnlyBuffer::AppendSingleElementRaw(const void * buffer)
     return CHIP_NO_ERROR;
 }
 
-CHIP_ERROR GenericAppendOnlyBuffer::AppendElementArrayRaw(const void * buffer, size_t numElements)
+CHIP_ERROR GenericAppendOnlyBuffer::AppendElementArrayRaw(const void * __restrict__ buffer, size_t numElements)
 {
     ReturnErrorOnFailure(EnsureAppendCapacity(numElements));
 

docs/development_controllers/chip-repl/Matter_REPL_Intro.ipynb

@@ -47,7 +47,7 @@ class GenericAppendOnlyBuffer
     /// Ensure that at least the specified number of elements
     /// can be appended to the internal buffer;
     ///
-    /// This will cause the internal buffer to become and allocated buffer
+    /// This will cause the internal buffer to become an allocated buffer
     CHIP_ERROR EnsureAppendCapacity(size_t numElements);
 
     bool IsEmpty() const { return mElementCount == 0; }
@@ -66,7 +66,10 @@ class GenericAppendOnlyBuffer
     ///
     /// This ALWAYS COPIES the elements internally.
     /// Additional capacity is AUTOMATICALLY ADDED.
-    CHIP_ERROR AppendElementArrayRaw(const void * buffer, size_t numElements);
+    ///
+    /// buffer MUST NOT point inside "own" buffer as mBuffer may be reallocated
+    /// as part of the appending.
+    CHIP_ERROR AppendElementArrayRaw(const void * __restrict__ buffer, size_t numElements);
 
     /// Appends a list of elements from a raw array.
     ///
@@ -93,7 +96,11 @@ class GenericAppendOnlyBuffer
 
 /// Represents a RAII instance owning a buffer.
 ///
-/// It auto-frees the owned buffer on destruction
+/// It auto-frees the owned buffer on destruction via `Platform::MemoryFree`.
+///
+/// This class is designed to be a storage class for `GenericAppendOnlyBuffer` and
+/// its subclasses (i.e. GenericAppendOnlyBuffer uses PlatformMemory and this class
+/// does the same. They MUST be kept in sync.)
 class ScopedBuffer
 {
 public:
@@ -165,6 +172,10 @@ class ListBuilder : public detail::GenericAppendOnlyBuffer
     ///
     /// Automatically attempts to allocate sufficient space to fulfill the element
     /// requirements.
+    ///
+    /// `span` MUST NOT point inside "own" buffer (and generally will not
+    /// as this class does not expose buffer access except by releasing ownership
+    /// via `Take`)
     CHIP_ERROR AppendElements(SpanType span) { return AppendElementArrayRaw(span.data(), span.size()); }
 
     /// Append a single element.

docs/development_controllers/chip-repl/python_chip_controller_building.md

@@ -83,7 +83,11 @@ class ProviderMetadataTree
     virtual void Temporary_ReportAttributeChanged(const AttributePathParams & path) = 0;
 
     // "convenience" functions that just return the data and ignore the error
-    // This returns the builder as-is even after the error (e.g. not found would return empty data)
+    // This returns the `ListBuilder<..>::TakeBuffer` from their equivalent fuctions as-is,
+    // even after an error (e.g. not found would return empty data).
+    //
+    // Usage of these indicates no error handling (not even logging) and code should
+    // consider handling errors instead.
     ReadOnlyBuffer<EndpointEntry> EndpointsIgnoreError();
     ReadOnlyBuffer<ServerClusterEntry> ServerClustersIgnoreError(EndpointId endpointId);
     ReadOnlyBuffer<AttributeEntry> AttributesIgnoreError(const ConcreteClusterPath & path);

scripts/jupyterlab_requirements.txt

@@ -26,9 +26,9 @@
 def main():
     c = Config()
     c.InteractiveShellApp.exec_lines = [
-        "import pkgutil",
-        "module = pkgutil.get_loader('chip.ChipReplStartup')",
-        "%run {module.path} " + " ".join(sys.argv[1:])
+        "import importlib.util",
+        "spec = importlib.util.find_spec('chip.ChipReplStartup')",
+        "%run {spec.origin} " + " ".join(sys.argv[1:])
     ]
 
     sys.argv = [sys.argv[0]]

src/app/data-model-provider/MetadataList.cpp

@@ -6,6 +6,7 @@
 import pathlib
 
 import chip.CertificateAuthority
+import chip.clusters as Clusters  # noqa: F401
 import chip.FabricAdmin
 import chip.logging
 import chip.native
@@ -163,7 +164,7 @@ def main():
     console.print(
         '''\t[red]certificateAuthorityManager[blue]:\tManages a list of CertificateAuthority instances.
     \t[red]caList[blue]:\t\t\t\tThe list of CertificateAuthority instances.
-    \t[red]caList[n][m][blue]:\t\t\tA specific FabricAdmin object at index m for the nth CertificateAuthority instance.''')
+    \t[red]caList\[n].adminList\[m][blue]:\t\tA specific FabricAdmin object at index m for the nth CertificateAuthority instance.''')
 
     console.print(
         f'\n\n[blue]Default CHIP Device Controller (NodeId: {devCtrl.nodeId}): '