mypy fixes (project-chip#34850)

* Add marker files for matter_idl and matter_yamltests

* type-annotations and other mypy fixes

* change name of BLE callbacks

* integrating comments

Author: Alami-Amine

scripts/py_matter_idl/BUILD.gn

@@ -31,6 +31,9 @@ pw_python_package("matter_idl") {
     # Dependency grammar
     "matter_idl/matter_grammar.lark",
 
+    #marker file to indicate to mypy that matter_idl is type-annotated
+    "matter_idl/py.typed",
+
     # Unit test data
     "matter_idl/tests/available_tests.yaml",
     "matter_idl/tests/inputs/cluster_struct_attribute.matter",

scripts/py_matter_idl/matter_idl/py.typed

@@ -39,3 +39,4 @@ matter_idl =
     generators/java/ClusterIDMapping.jinja
     generators/java/ClusterReadMapping.jinja
     generators/java/ClusterWriteMapping.jinja
+    py.typed

scripts/py_matter_idl/setup.cfg

@@ -43,6 +43,7 @@ pw_python_package("matter_yamltests") {
     "matter_yamltests/pseudo_clusters/clusters/system_commands.py",
     "matter_yamltests/pseudo_clusters/pseudo_cluster.py",
     "matter_yamltests/pseudo_clusters/pseudo_clusters.py",
+    "matter_yamltests/py.typed",
     "matter_yamltests/runner.py",
     "matter_yamltests/websocket_runner.py",
     "matter_yamltests/yaml_loader.py",

scripts/py_matter_yamltests/BUILD.gn

@@ -25,3 +25,7 @@ install_requires=
    diskcache
    lark
    websockets
+
+[options.package_data]
+matter_yamltests = 
+    py.typed

scripts/py_matter_yamltests/matter_yamltests/py.typed

@@ -3,3 +3,4 @@ click
 colorama
 diskcache
 websockets
+mypy==1.10.1

scripts/py_matter_yamltests/setup.cfg

@@ -21,7 +21,7 @@
 import ctypes
 import logging
 from ctypes import c_void_p
-from typing import List
+from typing import List, Optional
 
 import chip.exceptions
 from chip import ChipStack, FabricAdmin
@@ -92,7 +92,7 @@ def __init__(self, chipStack: ChipStack.ChipStack, caIndex: int, persistentStora
             raise ValueError("Encountered error initializing OpCreds adapter")
 
         self._isActive = True
-        self._activeAdmins = []
+        self._activeAdmins: List[FabricAdmin.FabricAdmin] = []
 
     def LoadFabricAdminsFromStorage(self):
         ''' If FabricAdmins had been setup previously, this re-creates them using information from persistent storage.
@@ -221,14 +221,13 @@ def __init__(self, chipStack: ChipStack.ChipStack, persistentStorage: Persistent
             persistentStorage:  If provided, over-rides the default instance in the provided chipStack
                                 when initializing CertificateAuthority instances.
         '''
-        self._activeCaIndexList = []
         self._chipStack = chipStack
 
         if (persistentStorage is None):
             persistentStorage = self._chipStack.GetStorageManager()
 
         self._persistentStorage = persistentStorage
-        self._activeCaList = []
+        self._activeCaList: List[CertificateAuthority] = []
         self._isActive = True
 
     def _AllocateNextCaIndex(self):
@@ -259,7 +258,7 @@ def LoadAuthoritiesFromStorage(self):
             ca = self.NewCertificateAuthority(int(caIndex))
             ca.LoadFabricAdminsFromStorage()
 
-    def NewCertificateAuthority(self, caIndex: int = None, maximizeCertChains: bool = False):
+    def NewCertificateAuthority(self, caIndex: Optional[int] = None, maximizeCertChains: bool = False):
         ''' Creates a new CertificateAuthority instance with the provided CA Index and the PersistentStorage
             instance previously setup in the constructor.
 

scripts/tests/requirements.txt

@@ -33,18 +33,18 @@
 import uuid
 from ctypes import CFUNCTYPE, PYFUNCTYPE, c_int, c_void_p, cast, pythonapi
 
-import dbus
-import dbus.mainloop.glib
-import dbus.service
+import dbus  # type: ignore
+import dbus.mainloop.glib  # type: ignore
+import dbus.service  # type: ignore
 
 from .ChipBleBase import ChipBleBase
 from .ChipBleUtility import BLE_ERROR_REMOTE_DEVICE_DISCONNECTED, BleDisconnectEvent, ParseServiceData
 
 try:
-    from gi.repository import GObject
+    from gi.repository import GObject  # type: ignore
 except Exception:
     logging.exception("Unable to find GObject from gi.repository")
-    from pgi.repository import GObject
+    from pgi.repository import GObject  # type: ignore
 
 chip_service = uuid.UUID("0000FFF6-0000-1000-8000-00805F9B34FB")
 chip_tx = uuid.UUID("18EE2EF5-263D-4559-959F-4F9C429F9D11")

src/controller/python/chip/CertificateAuthority.py

@@ -26,6 +26,7 @@
 from __future__ import absolute_import, print_function
 
 from ctypes import CDLL, POINTER, c_void_p, pointer
+from typing import Any
 
 from .ChipStack import ChipStack
 from .native import PyChipError
@@ -49,7 +50,7 @@ class ChipCommissionableNodeController(object):
     def __init__(self, chipStack: ChipStack):
         self.commissionableNodeCtrl = None
         self._ChipStack = chipStack
-        self._dmLib = None
+        self._dmLib: Any = None
 
         self._InitLib()
 

src/controller/python/chip/ChipBluezMgr.py

@@ -31,6 +31,7 @@
 import os
 from ctypes import CFUNCTYPE, Structure, c_bool, c_char_p, c_uint16, c_uint32, c_void_p, py_object, pythonapi
 from threading import Condition, Lock
+from typing import Any, Optional
 
 import chip.native
 from chip.native import PyChipError
@@ -90,7 +91,7 @@ def __call__(self):
             self._cv.notify_all()
         pythonapi.Py_DecRef(py_object(self))
 
-    def Wait(self, timeoutMs: int = None):
+    def Wait(self, timeoutMs: Optional[int] = None):
         timeout = None
         if timeoutMs is not None:
             timeout = float(timeoutMs) / 1000
@@ -143,7 +144,7 @@ class ChipStack(object):
     def __init__(self, persistentStoragePath: str, enableServerInteractions=True):
         builtins.enableDebugMode = False
 
-        self._ChipStackLib = None
+        self._ChipStackLib: Any = None
         self._chipDLLPath = None
         self.devMgr = None
         self._enableServerInteractions = enableServerInteractions
@@ -209,14 +210,14 @@ def Shutdown(self):
 
         delattr(builtins, "chipStack")
 
-    def Call(self, callFunct, timeoutMs: int = None):
+    def Call(self, callFunct, timeoutMs: Optional[int] = None):
         '''Run a Python function on CHIP stack, and wait for the response.
         This function is a wrapper of PostTaskOnChipThread, which includes some handling of application specific logics.
         Calling this function on CHIP on CHIP mainloop thread will cause deadlock.
         '''
         return self.PostTaskOnChipThread(callFunct).Wait(timeoutMs)
 
-    async def CallAsyncWithResult(self, callFunct, timeoutMs: int = None):
+    async def CallAsyncWithResult(self, callFunct, timeoutMs: Optional[int] = None):
         '''Run a Python function on CHIP stack, and wait for the response.
         This function will post a task on CHIP mainloop and waits for the call response in a asyncio friendly manner.
         '''
@@ -232,7 +233,7 @@ async def CallAsyncWithResult(self, callFunct, timeoutMs: int = None):
 
         return await asyncio.wait_for(callObj.future, timeoutMs / 1000 if timeoutMs else None)
 
-    async def CallAsync(self, callFunct, timeoutMs: int = None) -> None:
+    async def CallAsync(self, callFunct, timeoutMs: Optional[int] = None) -> None:
         '''Run a Python function on CHIP stack, and wait for the response.'''
         res: PyChipError = await self.CallAsyncWithResult(callFunct, timeoutMs)
         res.raise_on_error()

src/controller/python/chip/ChipCommissionableNodeCtrl.py

@@ -19,7 +19,7 @@
 from __future__ import annotations
 
 import logging
-from typing import List
+from typing import List, Optional
 
 from chip import CertificateAuthority, ChipDeviceCtrl
 from chip.crypto import p256keypair
@@ -61,9 +61,9 @@ def __init__(self, certificateAuthority: CertificateAuthority.CertificateAuthori
         LOGGER.info(f"New FabricAdmin: FabricId: 0x{self._fabricId:016X}, VendorId = 0x{self.vendorId:04X}")
 
         self._isActive = True
-        self._activeControllers = []
+        self._activeControllers: List[ChipDeviceCtrl.ChipDeviceController] = []
 
-    def NewController(self, nodeId: int = None, paaTrustStorePath: str = "",
+    def NewController(self, nodeId: Optional[int] = None, paaTrustStorePath: str = "",
                       useTestCommissioner: bool = False, catTags: List[int] = [], keypair: p256keypair.P256Keypair = None):
         ''' Create a new chip.ChipDeviceCtrl.ChipDeviceController instance on this fabric.
 

src/controller/python/chip/ChipStack.py

@@ -31,12 +31,12 @@ def ScanFoundCallback(closure, address: str, discriminator: int, vendor: int,
 
 
 @ScanDoneCallback
-def ScanDoneCallback(closure):
+def ScanIsDoneCallback(closure):
     closure.OnScanComplete()
 
 
 @ScanErrorCallback
-def ScanErrorCallback(closure, errorCode: int):
+def ScanHasErrorCallback(closure, errorCode: int):
     closure.OnScanError(errorCode)
 
 
@@ -106,7 +106,7 @@ def DiscoverSync(timeoutMs: int, adapter=None) -> Generator[DeviceInfo, None, No
             scanner = handle.pychip_ble_scanner_start(
                 ctypes.py_object(receiver),
                 handle.pychip_ble_adapter_list_get_raw_adapter(nativeList),
-                timeoutMs, ScanFoundCallback, ScanDoneCallback, ScanErrorCallback)
+                timeoutMs, ScanFoundCallback, ScanIsDoneCallback, ScanHasErrorCallback)
 
             if scanner == 0:
                 raise Exception('Failed to start BLE scan')

src/controller/python/chip/FabricAdmin.py

@@ -382,7 +382,7 @@ async def SendBatchCommands(future: Future, eventLoop, device, commands: List[In
     '''
     handle = chip.native.GetLibraryHandle()
 
-    responseTypes = []
+    responseTypes: List[Type] = []
     pyBatchCommandsData = _BuildPyInvokeRequestData(commands, timedRequestTimeoutMs, responseTypes)
 
     transaction = AsyncBatchCommandsTransaction(future, eventLoop, responseTypes)
@@ -417,7 +417,7 @@ def TestOnlySendBatchCommands(future: Future, eventLoop, device, commands: List[
 
     handle = chip.native.GetLibraryHandle()
 
-    responseTypes = []
+    responseTypes: List[Type] = []
     pyBatchCommandsData = _BuildPyInvokeRequestData(commands, timedRequestTimeoutMs,
                                                     responseTypes, suppressTimedRequestMessage=suppressTimedRequestMessage)
 

src/controller/python/chip/ble/scan_devices.py

@@ -19,7 +19,7 @@
 import dataclasses
 import enum
 import os
-from typing import Set, Tuple, Union
+from typing import Optional, Set, Tuple, Union
 
 ROOT_ENDPOINT_ID = 0
 
@@ -118,8 +118,8 @@ class GetCommissioneeCredentialsResponse:
     ipk: bytes
     case_admin_node: int
     admin_vendor_id: int
-    node_id: int = None
-    fabric_id: int = None
+    node_id: Optional[int] = None
+    fabric_id: Optional[int] = None
 
 
 class CredentialProvider:
@@ -137,4 +137,4 @@ async def get_commissionee_credentials(self, request: GetCommissioneeCredentials
 
 class ExampleCredentialProvider:
     async def get_commissionee_credentials(self, request: GetCommissioneeCredentialsRequest) -> GetCommissioneeCredentialsResponse:
-        pass
+        raise NotImplementedError("This method needs to be implemented.")

src/controller/python/chip/clusters/Command.py

@@ -14,7 +14,7 @@
 #    limitations under the License.
 #
 
-from typing import Optional
+from typing import Optional, cast
 
 # Represents the node ID that is to be used when creating device
 # controllers/commissioning devices
@@ -45,7 +45,8 @@ def GetLocalNodeId() -> int:
     if _local_node_id is None:
         SetLocalNodeId(DEFAULT_LOCAL_NODE_ID)
 
-    return _local_node_id
+    # cast is used to tell mypy typechecker that _local_node_id will always be an int from this point onwards
+    return cast(int, _local_node_id)
 
 
 def SetCommissionerCAT(cat: int):
@@ -68,4 +69,4 @@ def GetCommissionerCAT() -> int:
     if _local_cat is None:
         SetCommissionerCAT(DEFAULT_COMMISSIONER_CAT)
 
-    return _local_cat
+    return cast(int, _local_cat)

src/controller/python/chip/commissioning/__init__.py

@@ -78,24 +78,24 @@ class PendingDiscovery:
 
 @dataclass
 class CommissionableNode():
-    instanceName: str = None
-    hostName: str = None
-    port: int = None
-    longDiscriminator: int = None
-    vendorId: int = None
-    productId: int = None
-    commissioningMode: int = None
-    deviceType: int = None
-    deviceName: str = None
-    pairingInstruction: str = None
-    pairingHint: int = None
-    mrpRetryIntervalIdle: int = None
-    mrpRetryIntervalActive: int = None
-    mrpRetryActiveThreshold: int = None
-    supportsTcpClient: bool = None
-    supportsTcpServer: bool = None
-    isICDOperatingAsLIT: bool = None
-    addresses: List[str] = None
+    instanceName: Optional[str] = None
+    hostName: Optional[str] = None
+    port: Optional[int] = None
+    longDiscriminator: Optional[int] = None
+    vendorId: Optional[int] = None
+    productId: Optional[int] = None
+    commissioningMode: Optional[int] = None
+    deviceType: Optional[int] = None
+    deviceName: Optional[str] = None
+    pairingInstruction: Optional[str] = None
+    pairingHint: Optional[int] = None
+    mrpRetryIntervalIdle: Optional[int] = None
+    mrpRetryIntervalActive: Optional[int] = None
+    mrpRetryActiveThreshold: Optional[int] = None
+    supportsTcpClient: Optional[bool] = None
+    supportsTcpServer: Optional[bool] = None
+    isICDOperatingAsLIT: Optional[bool] = None
+    addresses: Optional[List[str]] = None
     rotatingId: Optional[str] = None
 
 

src/controller/python/chip/configuration/__init__.py

@@ -18,7 +18,7 @@
 # Generally thread credentials are assumed to be binary objects, however for
 # testing purposes, we expose the internal structure here.
 
-from construct import Byte, Bytes, Int16ul, Int64ul, PaddedString, Struct
+from construct import Byte, Bytes, Int16ul, Int64ul, PaddedString, Struct  # type: ignore
 
 ThreadNetworkInfo = Struct(
     "ActiveTimestamp" / Int64ul,

src/controller/python/chip/discovery/__init__.py

@@ -24,7 +24,7 @@
 import json
 import logging
 from ctypes import CFUNCTYPE, POINTER, c_bool, c_char, c_char_p, c_uint16, c_void_p, py_object
-from typing import Dict
+from typing import IO, Dict, Optional
 
 import chip.exceptions
 import chip.native
@@ -39,12 +39,12 @@
 
 
 @_SyncSetKeyValueCbFunct
-def _OnSyncSetKeyValueCb(storageObj, key: str, value, size):
+def _OnSyncSetKeyValueCb(storageObj, key: bytes, value, size):
     storageObj.SetSdkKey(key.decode("utf-8"), ctypes.string_at(value, size))
 
 
 @_SyncGetKeyValueCbFunct
-def _OnSyncGetKeyValueCb(storageObj, key: str, value, size, is_found):
+def _OnSyncGetKeyValueCb(storageObj, key: bytes, value, size, is_found):
     ''' This does not adhere to the API requirements of
     PersistentStorageDelegate::SyncGetKeyValue, but that is okay since
     the C++ storage binding layer is capable of adapting results from
@@ -94,7 +94,7 @@ class PersistentStorage:
         Object must be resident before the Matter stack starts up and last past its shutdown.
     '''
 
-    def __init__(self, path: str = None, jsonData: Dict = None):
+    def __init__(self, path: Optional[str] = None, jsonData: Optional[Dict] = None):
         ''' Initializes the object with either a path to a JSON file that contains the configuration OR
             a JSON dictionary that contains an in-memory representation of the configuration.
 
@@ -118,7 +118,7 @@ def __init__(self, path: str = None, jsonData: Dict = None):
 
         if (self._path):
             try:
-                self._file = open(path, 'r')
+                self._file: Optional[IO[str]] = open(path, 'r')
                 self._file.seek(0, 2)
                 size = self._file.tell()
                 self._file.seek(0)