diff --git a/.gitignore b/.gitignore index 98c8b38..1fa76b0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ __pycache__ .coverage __testdb__* +.idea/ +smadata_venv/ +notes.txt + diff --git a/doc/database.md b/doc/database.md new file mode 100644 index 0000000..a5fd776 --- /dev/null +++ b/doc/database.md @@ -0,0 +1,110 @@ +# Python SMAData2 Database + +This describes the database classes and table structure. + +### base.py +Uses abstract base classes (Python `abc`) to define an interface (class and set of methods) that are implemented by `mock.py` and `sqllite.py`. These deal with the physical sqllite database, or a mock database (Python ``set``) for testing purposes. + +### Table: generation +Stores readings +```sql +CREATE TABLE generation ( + inverter_serial INTEGER NOT NULL, + timestamp INTEGER NOT NULL, + sample_type INTEGER CHECK (sample_type = 0 OR + sample_type = 1 OR + sample_type = 2), + total_yield INTEGER, + PRIMARY KEY ( + inverter_serial, + timestamp, + sample_type + ) +); +``` +See base.py for further details. + +```pythonstub +# Ad hoc samples, externally controlled +SAMPLE_ADHOC = 0 +# Inverter recorded high(ish) frequency samples +SAMPLE_INV_FAST = 1 +# Inverted recorded daily samples +SAMPLE_INV_DAILY = 2 +``` + +### Table: pvoutput +Stores records of uploads to PVOutput.org +```sql +CREATE TABLE pvoutput ( + sid STRING, + last_datetime_uploaded INTEGER +); +``` + + + +### Table: EventData +Stores events as reported by the SMA device. +These include setting time, error conditions. +```sql +CREATE TABLE EventData ( + EntryID INT (4), + TimeStamp DATETIME NOT NULL, + Serial INT (4) NOT NULL, + SusyID INT (2), + EventCode INT (4), + EventType VARCHAR (32), + Category VARCHAR (32), + EventGroup VARCHAR (32), + Tag VARCHAR (200), + OldValue VARCHAR (32), + NewValue VARCHAR (32), + UserGroup VARCHAR (10), + PRIMARY KEY ( + Serial, + EntryID + ) +); +``` + +### Table: SpotData +Stores spot readings from the SMA device. +These include V, I for 3 phases , Grid Frequency, Temperature, BT signal. +Readings have any scale factor applied. +Some redundant fields +```sql +CREATE TABLE SpotData ( + TimeStamp DATETIME NOT NULL, + Serial INT (4) NOT NULL, + Pdc1 INT, + Pdc2 INT, + Idc1 FLOAT, + Idc2 FLOAT, + Udc1 FLOAT, + Udc2 FLOAT, + Pac1 INT, + Pac2 INT, + Pac3 INT, + Iac1 FLOAT, + Iac2 FLOAT, + Iac3 FLOAT, + Uac1 FLOAT, + Uac2 FLOAT, + Uac3 FLOAT, + EToday INT (8), + ETotal INT (8), + Frequency FLOAT, + OperatingTime DOUBLE, + FeedInTime DOUBLE, + BT_Signal FLOAT, + Status VARCHAR (10), + GridRelay VARCHAR (10), + Temperature FLOAT, + PRIMARY KEY ( + TimeStamp, + Serial + ) +); + +``` \ No newline at end of file diff --git a/doc/example.smadata2.json b/doc/example.smadata2.json index 9caa72a..b94c9db 100644 --- a/doc/example.smadata2.json +++ b/doc/example.smadata2.json @@ -11,11 +11,15 @@ "inverters": [{ "name": "Inverter 1", "bluetooth": "00:80:25:xx:yy:zz", - "serial": "2130012345" + "serial": "2130012345", + "start-time": "2019-01-17", + "password": "0000" }, { "name": "Inverter 2", "bluetooth": "00:80:25:pp:qq:rr", - "serial": "2130012346" + "serial": "2130012346", + "start-time": "2019-01-24", + "password": "1234" }] }] } diff --git a/doc/explore_usage.md b/doc/explore_usage.md new file mode 100644 index 0000000..4921960 --- /dev/null +++ b/doc/explore_usage.md @@ -0,0 +1,367 @@ +# How to use sma2-explore + +The `sma2-explore` tool allows the SMA protocol to be explored interactively & tested with a command-line interface that sends commands and displays the output. +The output packet is formatted to separate the outer bluetooth protocol from the inner PPP protocol. + +This shows examples of the available commands and typical output. + +## Getting Started + +Ensure the application is running on your local machine. The sma2explore command does not use the json config file. See the Deployment section in readme.md for notes on how to deploy the project on a live system. + +*Note that sma2-explore does not run under Windows as it uses the unsupported ``os.fork`` in ``SMAData2CLI`` to start a second thread that listens for incoming packets.* An alternative approach is needed. + +------------------------- + +### sma2-explore commands + +Command-line arguments are handled by class `SMAData2CLI`. +This CLI class overrides the Connection class defined in `smabluetooth`. +Implements most of the same functions, but calls a dump_ function first. The dump function prints a formatted packet format to the stdout. + +```sh +pi@raspberrypi:~ $ python3 python-smadata2/sma2-explore "00:80:25:2C:11:B2" +``` +This will connect to the supplied Bluetooth address and starts a terminal session with the SMA inverter. + +Upon initial connection, the SMA device issues a "hello" packet once per second, +until the host replies with an identical (?) hello packet. + +The terminal window shows Rx (received) and Tx (transmitted) lines. The first two lines are the raw bytes from the packet, and the following lines are interpreted by the relevant function in sma2-explore. + +```sh + +pi@raspberrypi:~ $ python3 python-smadata2/sma2-explore "00:80:25:2C:11:B2" +Connected B8:27:EB:F4:80:EB -> 00:80:25:2C:11:B2 +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 1F 00 61 B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 02 00 00 04 70 00 04 00-00 00 00 01 00 00 00 +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 02 +Rx< HELLO! + +... repeats every second + +Rx< 0000: 7E 1F 00 61 B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 02 00 00 04 70 00 04 00-00 00 00 01 00 00 00 +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 02 +Rx< HELLO! + +``` +Commands can be sent to the inverter by simply typing and entering in the window. Although the "hello" messages continue to scroll up, your keystrokes are displayed and interpreted. + +The commands are coded in class SMAData2CLI and these are supported + +| Command| Function | Description | +| ------ | ------ | --------- | +| `hello`| `cmd_hello(self)` | Level 1 hello command responds to the SMA with the same data packet sent. | +| `logon`| `cmd_logon(self, password=b'0000', timeout=900)` | logon to the inverter with the default password. | +| `quit`| `cmd_quit(self)` | close the SMA connection, return to shell. | +| `getvar`| `cmd_getvar(self, varid)` | Level 1 getvar requests the value or a variable from the SMA inverter | +| `ppp`| `cmd_ppp(self, protocol, *args)` | Sends a SMA Level 2 packet from payload, calls tx_outer to wrap in Level 1 packet | +| `send2`| `cmd_send2(self, *args)` | Sends a SMA Level 2 request (builds a PPP frame for Transmission and calls tx_ppp to wrap for transmission). | +| `gdy`| `cmd_gdy(self)` | Sends a SMA Level 2 request to get Daily data.| +| `yield`| `cmd_yield(self)` | Sends a SMA Level 2 request to get Yield.| +| `historic`| `cmd_historic(self, fromtime=None, totime=None)` | Sends a SMA Level 2 request to get historic data between dates specified.| + + +#### Packet type 0x02: "Hello" + +Level 1 hello command responds to the SMA with the same data packet received in the above broadcast. + +The inverter responds with three packets, and stops transmitting the hello, so the screen stops scrolling. +``` +TYPE 0A +TYPE 0C +TYPE 05 +``` + +```sh + +hello + +Tx> 0000: 7E 1F 00 61 00 00 00 00-00 00 B2 11 2C 25 80 00 +Tx> 0010: 02 00 00 04 70 00 04 00-00 00 00 01 00 00 00 +Tx> 00:00:00:00:00:00 -> 00:80:25:2C:11:B2 TYPE 02 +Tx> HELLO! +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 1F 00 61 B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 0A 00 B2 11 2C 25 80 00-01 EB 80 F4 EB 27 B8 +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 0A + +Rx< 0000: 7E 14 00 6A B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 0C 00 02 00 +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 0C + +Rx< 0000: 7E 22 00 5C B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 05 00 B2 11 2C 25 80 00-01 01 EB 80 F4 EB 27 B8 +Rx< 0020: 02 01 +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 05 + + +``` +#### logon +Establish an authorised connection enabling further requests. + +The password is hard-coded as '0000'. *Note: it does not use the config file entry.* + +Inverter responds with a type 01 packet, containing PPP frame. + +tag 0001 (first, last) response 0x040c subtype 0xfffd +```sh +logon + +Tx> 0000: 7E 52 00 2C EB 80 F4 EB-27 B8 FF FF FF FF FF FF +Tx> 0010: 01 00 7E FF 03 60 65 0E-A0 FF FF FF FF FF FF 00 +Tx> 0020: 01 78 00 3F 10 FB 39 00-01 00 00 00 00 01 80 0C +Tx> 0030: 04 FD FF 07 00 00 00 84-03 00 00 AA AA BB BB 00 +Tx> 0040: 00 00 00 B8 B8 B8 B8 88-88 88 88 88 88 88 88 45 +Tx> 0050: 54 7E +Tx> B8:27:EB:F4:80:EB -> ff:ff:ff:ff:ff:ff TYPE 01 +Tx> PPP frame; protocol 0x6560 [56 bytes] +Tx> 0000: 0E A0 FF FF FF FF FF FF-00 01 78 00 3F 10 FB 39 +Tx> 0010: 00 01 00 00 00 00 01 80-0C 04 FD FF 07 00 00 00 +Tx> 0020: 84 03 00 00 AA AA BB BB-00 00 00 00 B8 B8 B8 B8 +Tx> 0030: 88 88 88 88 88 88 88 88- +Tx> SMA INNER PROTOCOL PACKET +Tx> 78.00.3F.10.FB.39 => FF.FF.FF.FF.FF.FF +Tx> control A0 00 01 00 01 +Tx> tag 0001 (first, last) +Tx> command 0x040c subtype 0xfffd +Tx> 0000: AA AA BB BB 00 00 00 00-B8 B8 B8 B8 88 88 88 88 +Tx> 0010: 88 88 88 88 +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 53 00 2D B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 01 00 7E FF 03 60 65 0E-D0 78 00 3F 10 FB 39 00 +Rx< 0020: 01 8A 00 1C 78 F8 7D 5E-00 01 00 00 00 00 01 80 +Rx< 0030: 0D 04 FD FF 07 00 00 00-84 03 00 00 AA AA BB BB +Rx< 0040: 00 00 00 00 B8 B8 B8 B8-88 88 88 88 88 88 88 88 +Rx< 0050: C5 E3 7E +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 01 +Rx< Partial PPP data frame begins frame ends +Rx< PPP frame; protocol 0x6560 [56 bytes] +Rx< 0000: 0E D0 78 00 3F 10 FB 39-00 01 8A 00 1C 78 F8 7E +Rx< 0010: 00 01 00 00 00 00 01 80-0D 04 FD FF 07 00 00 00 +Rx< 0020: 84 03 00 00 AA AA BB BB-00 00 00 00 B8 B8 B8 B8 +Rx< 0030: 88 88 88 88 88 88 88 88- +Rx< SMA INNER PROTOCOL PACKET +Rx< 8A.00.1C.78.F8.7E => 78.00.3F.10.FB.39 +Rx< control D0 00 01 00 01 +Rx< tag 0001 (first, last) +Rx< response 0x040c subtype 0xfffd +Rx< 0000: AA AA BB BB 00 00 00 00-B8 B8 B8 B8 88 88 88 88 +Rx< 0010: 88 88 88 88 +``` + +#### gdy (get daily) +Sends a SMA Level 2 request to get Daily data. + +Inverter responds with a type 01 packet, containing PPP frame. + +tag 0005 (first, last) 0x0200 subtype 0x5400 +```sh +gdy + +Tx> 0000: 7E 3E 00 40 EB 80 F4 EB-27 B8 FF FF FF FF FF FF +Tx> 0010: 01 00 7E FF 03 60 65 09-A0 FF FF FF FF FF FF 00 +Tx> 0020: 00 78 00 3F 10 FB 39 00-00 00 00 00 00 05 80 00 +Tx> 0030: 02 00 54 00 22 26 00 FF-22 26 00 BB D6 7E +Tx> B8:27:EB:F4:80:EB -> ff:ff:ff:ff:ff:ff TYPE 01 +Tx> PPP frame; protocol 0x6560 [36 bytes] +Tx> 0000: 09 A0 FF FF FF FF FF FF-00 00 78 00 3F 10 FB 39 +Tx> 0010: 00 00 00 00 00 00 05 80-00 02 00 54 00 22 26 00 +Tx> 0020: FF 22 26 00 +Tx> SMA INNER PROTOCOL PACKET +Tx> 78.00.3F.10.FB.39 => FF.FF.FF.FF.FF.FF +Tx> control A0 00 00 00 00 +Tx> tag 0005 (first, last) +Tx> command 0x0200 subtype 0x5400 + +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 50 00 2E B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 01 00 7E FF 03 60 65 0D-90 78 00 3F 10 FB 39 00 +Rx< 0020: A0 8A 00 1C 78 F8 7D 5E-00 00 00 00 00 00 05 80 +Rx< 0030: 01 02 00 54 01 00 00 00-01 00 00 00 01 22 26 00 +Rx< 0040: 81 7D 5D 9C 5D 31 5D 00-00 00 00 00 00 3C 94 7E +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 01 +Rx< Partial PPP data frame begins frame ends +Rx< PPP frame; protocol 0x6560 [52 bytes] +Rx< 0000: 0D 90 78 00 3F 10 FB 39-00 A0 8A 00 1C 78 F8 7E +Rx< 0010: 00 00 00 00 00 00 05 80-01 02 00 54 01 00 00 00 +Rx< 0020: 01 00 00 00 01 22 26 00-81 7D 9C 5D 31 5D 00 00 +Rx< 0030: 00 00 00 00 +Rx< SMA INNER PROTOCOL PACKET +Rx< 8A.00.1C.78.F8.7E => 78.00.3F.10.FB.39 +Rx< control 90 00 A0 00 00 +Rx< tag 0005 (first, last) +Rx< response 0x0200 subtype 0x5400 +Rx< 0000: 01 22 26 00 81 7D 9C 5D-31 5D 00 00 00 00 00 00 + +``` + + +#### yield +Sends a SMA Level 2 request to get Yield. + +Inverter responds with a type 01 packet, containing PPP frame. + +tag 0006 (first, last) 0x0200 subtype 0x5400 + +```sh +yield + +Tx> 0000: 7E 3E 00 40 EB 80 F4 EB-27 B8 FF FF FF FF FF FF +Tx> 0010: 01 00 7E FF 03 60 65 09-A0 FF FF FF FF FF FF 00 +Tx> 0020: 00 78 00 3F 10 FB 39 00-00 00 00 00 00 06 80 00 +Tx> 0030: 02 00 54 00 01 26 00 FF-01 26 00 37 72 7E +Tx> B8:27:EB:F4:80:EB -> ff:ff:ff:ff:ff:ff TYPE 01 +Tx> PPP frame; protocol 0x6560 [36 bytes] +Tx> 0000: 09 A0 FF FF FF FF FF FF-00 00 78 00 3F 10 FB 39 +Tx> 0010: 00 00 00 00 00 00 06 80-00 02 00 54 00 01 26 00 +Tx> 0020: FF 01 26 00 +Tx> SMA INNER PROTOCOL PACKET +Tx> 78.00.3F.10.FB.39 => FF.FF.FF.FF.FF.FF +Tx> control A0 00 00 00 00 +Tx> tag 0006 (first, last) +Tx> command 0x0200 subtype 0x5400 + +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 4F 00 31 B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 01 00 7E FF 03 60 65 0D-90 78 00 3F 10 FB 39 00 +Rx< 0020: A0 8A 00 1C 78 F8 7D 5E-00 00 00 00 00 00 06 80 +Rx< 0030: 01 02 00 54 00 00 00 00-00 00 00 00 01 01 26 00 +Rx< 0040: F3 50 9C 5D 20 6D 64 02-00 00 00 00 D3 57 7E +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 01 +Rx< Partial PPP data frame begins frame ends +Rx< PPP frame; protocol 0x6560 [52 bytes] +Rx< 0000: 0D 90 78 00 3F 10 FB 39-00 A0 8A 00 1C 78 F8 7E +Rx< 0010: 00 00 00 00 00 00 06 80-01 02 00 54 00 00 00 00 +Rx< 0020: 00 00 00 00 01 01 26 00-F3 50 9C 5D 20 6D 64 02 +Rx< 0030: 00 00 00 00 +Rx< SMA INNER PROTOCOL PACKET +Rx< 8A.00.1C.78.F8.7E => 78.00.3F.10.FB.39 +Rx< control 90 00 A0 00 00 +Rx< tag 0006 (first, last) +Rx< response 0x0200 subtype 0x5400 +Rx< 0000: 01 01 26 00 F3 50 9C 5D-20 6D 64 02 00 00 00 00 +``` +#### getvar [var] +Sends a SMA Level 1 request to get variable value, as 2 digit hex number. + +Variables 01, 02, 03, 04, 05, 06: valid +Inverter responds with a type 04 packet giving a variable value. + +e.g. 05 is Signal 0x00C4 is 196/255 = 76.9% + +e.g. 09 is a version string: "CG2000 V1.212 Jul 2 2010 14:52:52" + +Variables 0x00, 0x0B, 0x10, 0x11: Invalid + Causes a type 07 error packet to be issued +```sh +getvar 5 + +Tx> 0000: 7E 14 00 6A 00 00 00 00-00 00 B2 11 2C 25 80 00 +Tx> 0010: 03 00 05 00 +Tx> 00:00:00:00:00:00 -> 00:80:25:2C:11:B2 TYPE 03 +Tx> GETVAR 0x05 +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 18 00 66 B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 04 00 05 00 00 00 C4 00- +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 04 +Rx< VARVAL 0x05 +Rx< Signal level 76.9% + +getvar 9 + +Tx> 0000: 7E 14 00 6A 00 00 00 00-00 00 B2 11 2C 25 80 00 +Tx> 0010: 03 00 09 00 +Tx> 00:00:00:00:00:00 -> 00:80:25:2C:11:B2 TYPE 03 +Tx> GETVAR 0x09 +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 38 00 46 B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 04 00 09 00 00 00 43 47-32 30 30 30 20 56 31 2E +Rx< 0020: 32 31 32 20 4A 75 6C 20-20 32 20 32 30 31 30 20 +Rx< 0030: 31 34 3A 35 32 3A 35 32- +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 04 +Rx< VARVAL 0x09 + +``` +#### spotacvoltage +Sends a SMA Level 1 request to get spotacvoltage. + +More complex example as the response is spread across 3 frames, with 204 bytes of PPP data. This includes voltages for 3 phase power. +```sh +spotacvoltage + +Tx> 0000: 7E 3E 00 40 EB 80 F4 EB-27 B8 FF FF FF FF FF FF +Tx> 0010: 01 00 7E FF 03 60 65 09-A0 FF FF FF FF FF FF 00 +Tx> 0020: 00 78 00 3F 10 FB 39 00-00 00 00 00 00 02 80 00 +Tx> 0030: 02 00 51 00 48 46 00 FF-55 46 00 CF 77 7E +Tx> B8:27:EB:F4:80:EB -> ff:ff:ff:ff:ff:ff TYPE 01 +Tx> PPP frame; protocol 0x6560 [36 bytes] +Tx> 0000: 09 A0 FF FF FF FF FF FF-00 00 78 00 3F 10 FB 39 +Tx> 0010: 00 00 00 00 00 00 02 80-00 02 00 51 00 48 46 00 +Tx> 0020: FF 55 46 00 +Tx> SMA INNER PROTOCOL PACKET +Tx> 78.00.3F.10.FB.39 => FF.FF.FF.FF.FF.FF +Tx> control A0 00 00 00 00 +Tx> tag 0002 (first, last) +Tx> command 0x0200 subtype 0x5100 + +SMA2 00:80:25:2C:11:B2 >> +Rx< 0000: 7E 6D 00 13 B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 08 00 7E FF 03 60 65 33-90 78 00 3F 10 FB 39 00 +Rx< 0020: A0 8A 00 1C 78 F8 7D 5E-00 00 00 00 00 00 02 80 +Rx< 0030: 01 02 00 51 0A 00 00 00-0F 00 00 00 01 48 46 00 +Rx< 0040: DB 9C 9D 5D 9B 5E 00 00-9B 5E 00 00 9B 5E 00 00 +Rx< 0050: 9B 5E 00 00 01 00 00 00-01 49 46 00 DB 9C 9D 5D +Rx< 0060: FF FF FF FF FF FF FF FF-FF FF FF FF FF +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 08 +Rx< Partial PPP data frame begins + +Rx< 0000: 7E 6D 00 13 B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 08 00 FF FF FF 01 00 00-00 01 4A 46 00 DB 9C 9D +Rx< 0020: 5D FF FF FF FF FF FF FF-FF FF FF FF FF FF FF FF +Rx< 0030: FF 01 00 00 00 01 50 46-00 DB 9C 9D 5D BC 00 00 +Rx< 0040: 00 BC 00 00 00 BC 00 00-00 BC 00 00 00 01 00 00 +Rx< 0050: 00 01 51 46 00 DB 9C 9D-5D FF FF FF FF FF FF FF +Rx< 0060: FF FF FF FF FF FF FF FF-FF 01 00 00 00 +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 08 +Rx< Partial PPP data + +Rx< 0000: 7E 31 00 4F B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 01 00 01 52 46 00 DB 9C-9D 5D FF FF FF FF FF FF +Rx< 0020: FF FF FF FF FF FF FF FF-FF FF 01 00 00 00 95 2B +Rx< 0030: 7E +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 01 +Rx< Partial PPP data frame ends +Rx< PPP frame; protocol 0x6560 [204 bytes] +Rx< 0000: 33 90 78 00 3F 10 FB 39-00 A0 8A 00 1C 78 F8 7E +Rx< 0010: 00 00 00 00 00 00 02 80-01 02 00 51 0A 00 00 00 +Rx< 0020: 0F 00 00 00 01 48 46 00-DB 9C 9D 5D 9B 5E 00 00 +Rx< 0030: 9B 5E 00 00 9B 5E 00 00-9B 5E 00 00 01 00 00 00 +Rx< 0040: 01 49 46 00 DB 9C 9D 5D-FF FF FF FF FF FF FF FF +Rx< 0050: FF FF FF FF FF FF FF FF-01 00 00 00 01 4A 46 00 +Rx< 0060: DB 9C 9D 5D FF FF FF FF-FF FF FF FF FF FF FF FF +Rx< 0070: FF FF FF FF 01 00 00 00-01 50 46 00 DB 9C 9D 5D +Rx< 0080: BC 00 00 00 BC 00 00 00-BC 00 00 00 BC 00 00 00 +Rx< 0090: 01 00 00 00 01 51 46 00-DB 9C 9D 5D FF FF FF FF +Rx< 00a0: FF FF FF FF FF FF FF FF-FF FF FF FF 01 00 00 00 +Rx< 00b0: 01 52 46 00 DB 9C 9D 5D-FF FF FF FF FF FF FF FF +Rx< 00c0: FF FF FF FF FF FF FF FF-01 00 00 00 +Rx< SMA INNER PROTOCOL PACKET +Rx< 8A.00.1C.78.F8.7E => 78.00.3F.10.FB.39 +Rx< control 90 00 A0 00 00 +Rx< tag 0002 (first, last) +Rx< response 0x0200 subtype 0x5100 +Rx< 0000: 01 48 46 00 DB 9C 9D 5D-9B 5E 00 00 9B 5E 00 00 +Rx< 0010: 9B 5E 00 00 9B 5E 00 00-01 00 00 00 01 49 46 00 +Rx< 0020: DB 9C 9D 5D FF FF FF FF-FF FF FF FF FF FF FF FF +Rx< 0030: FF FF FF FF 01 00 00 00-01 4A 46 00 DB 9C 9D 5D +Rx< 0040: FF FF FF FF FF FF FF FF-FF FF FF FF FF FF FF FF +Rx< 0050: 01 00 00 00 01 50 46 00-DB 9C 9D 5D BC 00 00 00 +Rx< 0060: BC 00 00 00 BC 00 00 00-BC 00 00 00 01 00 00 00 +Rx< 0070: 01 51 46 00 DB 9C 9D 5D-FF FF FF FF FF FF FF FF +Rx< 0080: FF FF FF FF FF FF FF FF-01 00 00 00 01 52 46 00 +Rx< 0090: DB 9C 9D 5D FF FF FF FF-FF FF FF FF FF FF FF FF +Rx< 00a0: FF FF FF FF 01 00 00 00- +``` \ No newline at end of file diff --git a/doc/logging.md b/doc/logging.md new file mode 100644 index 0000000..12e6a6d --- /dev/null +++ b/doc/logging.md @@ -0,0 +1,55 @@ +# How to use logging + +The application uses Python standard logging to log information and (debugging) data from the inverter sessions. + +This is highly configurable through changes to the `smadata2.logging_config.py` file. This contains a Python dictionary definition that is used to initialise the logging, based on the `dictconfig()` model. + +References +[https://docs.python.org/3/howto/logging.html]() +[https://docs.python.org/3/howto/logging-cookbook.html]() + +A common scenario is to modify the way that logs are stored or sent +[https://docs.python.org/3/howto/logging-cookbook.html#customizing-handlers-with-dictconfig]() + +This shows examples of the available commands and typical output. + +## How it works + +Within the application, information is written to the log as in the example below. +The logging level is `info` in the example, and more detailed messages may use `debug`. + +```pythonstub +log.info("\tDaily generation at %s:\t%d Wh" % (smadata2.datetimeutil.format_time(dtime), daily)) +``` +The logging_config file has a section `handlers`. +This shows that all the logged information is written to either the console (`default, 'stream': 'ext://sys.stdout'`) and to a file (`'filename': 'sma2.log'`). The level for each is set in the handler. + +```pythonstub + 'handlers': { + 'default': { + 'level': 'DEBUG', + 'formatter': 'simple', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout', # Default is stderr + }, + 'file': { + 'level': 'DEBUG', + 'formatter': 'standard', + 'class': 'logging.FileHandler', + 'filename': 'sma2.log', + }, + }, +``` + +## Reference + +Logging levels are as below + +|Level | When it’s used | +|------|----------------| +|DEBUG|Detailed information, typically of interest only when diagnosing problems. Details of the messages to/from the inverter, the database are shown at this level.| +|INFO|Confirmation that things are working as expected. Results like Power, Voltage from the inverter are at this level.| +|WARNING|An indication that something unexpected happened, or indicative of some problem in the near future . The software is still working as expected.| +|ERROR|Due to a more serious problem, the software has not been able to perform some function.| +|CRITICAL|A serious error, indicating that the program itself may be unable to continue running.| + diff --git a/doc/protocol.md b/doc/protocol.md new file mode 100644 index 0000000..057115a --- /dev/null +++ b/doc/protocol.md @@ -0,0 +1,391 @@ +# Notes on the protocol for Bluetooth enabled SMA inverters + +There seem to be two "interesting" protocol layers in the SMA +bluetooth protocol. The "outer" protocol is a packet protocol over +Bluetooth RFCOMM. It seems mainly to deal with Bluetooth specific +things - signal levels etc. + +Speculation: + - Used for communication between the bluetooth adapters, rather than + the equipment itself? + - e.g. SMA bluetooth repeaters would talk this protocol, then forward + the inner frames on to the actual equipment? + +Some of the outer protocol frame types encapsulate PPP frames. All +PPP frames observed are PPP protocol number 0x6560, which appears to +be an SMA allocated ID for their control protocol. + +Speculation: + - Is PPP and the inner protocol used directly over serial when using + RS485 instead of Bluetooth connections? + - Allows for shared use of RS485 lines, maybe? + +### Acknowledgements +This has been derived from various sources, and analysis of the data" +- David Gibson [https://github.com/dgibson]() +- James Ball [http://blog.jamesball.co.uk]() +- SBFspot project [https://github.com/SBFspot/SBFspot]() +- Point-to-Point Protocol [https://en.wikipedia.org/wiki/Point-to-Point_Protocol]() + +## Outer protocol + +Packet based protocol over RFCOMM channel 1 over Bluetooth. The same +packet format appears to be used in both directions. + + +### Packet header +```text +Offset Value +--------------------- +0 0x7e +1 length of packet (including header), max 0x70 +2 0x00 +3 check byte, XOR of bytes 0..2 inclusive +4..9 "From" bluetooth address +10..15 "To" bluetooth address +16..17 Command type (LE16) +18.. Payload (format depends on packet type) + +0x0100 - L2 packet/L2 packet end +0x0200 - 'hello' message (from inverter, or computer) +0x0300 - Request for information +0x0400 - Response to request +0x0500 - Response 3 from inverter to 'hello', followed by inverter address. +0x0700 - Error +0x0800 - L2 part packet +0x0a00 - Response 1 from inverter to 'hello', followed by inverter address. +0x0c00 - Response 2 from inverter to 'hello', followed by 0x00 +``` + +The Bluetooth addresses are encoded in the reverse order to how they're usually written. So `00:80:25:2C:11:B2` would be sent in the +packet header as: `B2 11 2C 25 80 00` and that can be seen in the example below. + +For broadcast (sending the 'hello' packets) the destination address is `00:00:00:00:00:00`. + +For packets which don't relate to the inner protocol, `00:00:00:00:00:00` seems to be used instead of the initiating host's +MAC address. + +In this example packet `50` is the length, ln, `2E` the checksum, etc. +The payload (Level 2 packet) starts in the second row with the header `7E FF 03 60 65` and the PPP frame starts with `0D 90`. See inner protocol below. +The packet is 5 rows of 16 bytes, i.e. length 0x50. + +The payload is 52 or 0x34 bytes, printed again for clarity below, and broken down into the known elements. +```sh + ln chk <-- from --> <-- to --> +Rx< 0000: 7E 50 00 2E B2 11 2C 25-80 00 EB 80 F4 EB 27 B8 +Rx< 0010: 01 00 7E FF 03 60 65 0D-90 78 00 3F 10 FB 39 00 +Rx< 0020: A0 8A 00 1C 78 F8 7D 5E-00 00 00 00 00 00 05 80 +Rx< 0030: 01 02 00 54 01 00 00 00-01 00 00 00 01 22 26 00 +Rx< 0040: 81 7D 5D 9C 5D 31 5D 00-00 00 00 00 00 3C 94 7E +Rx< 00:80:25:2C:11:B2 -> B8:27:EB:F4:80:EB TYPE 01 + +Rx< Partial PPP data frame begins frame ends +Rx< PPP frame; protocol 0x6560 [52 bytes] +Rx< 0000: 0D 90 78 00 3F 10 FB 39-00 A0 8A 00 1C 78 F8 7E +Rx< 0010: 00 00 00 00 00 00 05 80-01 02 00 54 01 00 00 00 +Rx< 0020: 01 00 00 00 01 22 26 00-81 7D 9C 5D 31 5D 00 00 +Rx< 0030: 00 00 00 00 + +Rx< SMA INNER PROTOCOL PACKET +Rx< 8A.00.1C.78.F8.7E => 78.00.3F.10.FB.39 +Rx< control 90 00 A0 00 00 +Rx< tag 0005 (first, last) +Rx< response 0x0200 subtype 0x5400 +Rx< 0000: 01 22 26 00 81 7D 9C 5D-31 5D 00 00 00 00 00 00 +``` + +### Packet type 0x01: PPP frame (last piece) + +```text +Offset Value +16 0x01 +17 0x00 +18.. PPP data +``` +The PPP data is raw as it would be transmitted over serial. i.e. it +includes flag bytes (0x7e at start and end of each PPP packet), PPP +escaping, and the PPP CRC16 checksum at end of each frame. + +### Packet type 0x02: "Hello" + +Upon connection, SMA device issues one of these ~once per second, +until host replies with an identical (?) hello packet. 16-byte header and 15-byte payload. + +```shell script +Rx< 0000: 7E 1F 00 61 B2 11 2C 25-80 00 00 00 00 00 00 00 +Rx< 0010: 02 00 00 04 70 00 04 00-00 00 00 01 00 00 00 +Rx< 00:80:25:2C:11:B2 -> 00:00:00:00:00:00 TYPE 02 +Rx< HELLO! + +hello +Tx> 0000: 7E 1F 00 61 00 00 00 00-00 00 B2 11 2C 25 80 00 +Tx> 0010: 02 00 00 04 70 00 04 00-00 00 00 01 00 00 00 +Tx> 00:00:00:00:00:00 -> 00:80:25:2C:11:B2 TYPE 02 +Tx> HELLO! +``` + +```text +Offset Value +--------------------- +16 0x02 +17 0x00 +18 0x00 4 byte long 0x00700400 +19 0x04 +20 0x70 +21 0x00 +22 0x01 1 byte NetID typically (0x01, 0x04) +23 0x00 4 byte long 0x00000000 +24 0x00 +25 0x00 +26 0x00 +27 0x01 4 byte long 0x00000001 +28 0x00 +29 0x00 +30 0x00 +``` +### Packet type 0x03: GETVAR + +Causes device to issue a type 04 packet giving a variable value (?) +```text + +Offset Value +--------------------- +16 0x03 +17 0x00 +18..19 variable ID (LE16) +``` + +### Packet type 0x04: VARIABLE + +``` +Offset Value +--------------------- +16 0x04 +17 0x00 +18..19 variable ID (LE16) +20.. variable contents +``` + +Variables: + Variable 0x00, 0x10, 0x11: Invalid + Causes a type 07 error packet to be issued + + Variable 0x05: Signal Level +``` +Offset Value +---------------------- +`18 0x05 +19 0x00 +20 0x00 +21 0x00 +22 signal level, out of 255 +23 0x00` + +ID Meaning Length +-------------------------------------- +0x05 signal level 4 bytes +``` + +### Packet type 0x05: Unknown + +### Packet type 0x07: Error + +### Packet type 0x08: PPP frame (not last piece) +As type 0x01 + +### Packet type 0x0a: Unknown + +### Packet type 0x0c: Unknown + + +## Inner protocol (PPP protocol 0x6560) +Example: +```sh +Rx< Partial PPP data frame begins frame ends +Rx< PPP frame; protocol 0x6560 [52 bytes] +Rx< 0000: 0D 90 78 00 3F 10 FB 39-00 A0 8A 00 1C 78 F8 7E +Rx< 0010: 00 00 00 00 00 00 05 80-01 02 00 54 01 00 00 00 +Rx< 0020: 01 00 00 00 01 22 26 00-81 7D 9C 5D 31 5D 00 00 +Rx< 0030: 00 00 00 00 + +Rx< SMA INNER PROTOCOL PACKET +Rx< 8A.00.1C.78.F8.7E => 78.00.3F.10.FB.39 +Rx< control 90 00 A0 00 00 +Rx< tag 0005 (first, last) +Rx< response 0x0200 subtype 0x5400 +Rx< 0000: 01 22 26 00 81 7D 9C 5D-31 5D 00 00 00 00 00 00 +``` + +``` +PPP header +0 Flag byte, 0x7E, the beginning of a PPP frame +1 Address 0xFF, standard broadcast address +2 Control 0x03, unnumbered data +3..4 Protocol PPP ID of embedded data SMA Net2+ protocol 0x6560 + +5.. Start of datagram information below: + +Offset Value +---------------------- +0 Length of packet, in 32-bit words, including (inner) header, but not ppp header?? +1 Destination header (0xA0 broadcast; 0xE0 destination is inverter; 0x80, 0x90, 0xC0, desination is computer) +2..7 Destination address; 6 bytes 78.00.3F.10.FB.39 +8 ? B1, padding 0x00 +9 Source header ? B2 (0xA0 broadcast; 0xE0 destination is inverter; 0x80, 0x90, 0xC0, desination is computer) +10..15 Source address; 6-bytes 8A.00.1C.78.F8.7E +16..17 ??? C1,C2 usually 0x00, 0x00 +18..19 error code?, usually 0x00, 0x15; 0x15 seems to be ack from inverter +20 Telegram number packet count for multi packet response; 0x00 for single packet; 0x06 means 7 packets in response +21 Telegram ? always 0x00 +22..23 LE16, low 15 bits are tag value + MSB is "first packet" flag for multi packet response?? +24..25 Packet type + LSB is command/response flag +26..27 Packet subtype +28..31 Arg 1 (LE) +32..35 Arg 2 (LE) + + Padding + 2-byte Frame Check Sequence +last Footer 0x7E, termination +``` + + +### Command: Total Yield + +``` +COMMAND: + A2: A0 + B1,B2: 00 00 + C1,C2: 00 00 + Type: 0200 + Subtype: 5400 + Arg1: 0x00260100 + Arg2: 0x002601ff + +RESPONSE: + PAYLOAD: + 0..3 timestamp (LE) + 4..7 total yield in Wh (LE) +``` + +### Command: Daily Yield + +``` +COMMAND: + A2: A0 + B1,B2: 00 00 + C1,C2: 00 00 + Type: 0200 + Subtype: 5400 + Arg1: 0x00262200 + Arg2: 0x002622ff + +RESPONSE: + PAYLOAD: + 0..3 timestamp (LE) + 4..7 day's yield in Wh (LE) +``` + +### Command: Historic data (5 minute intervals) + +``` +COMMAND: + A2: E0 + B1,B2: 00 00 + C1,C2: 00 00 + Type: 0200 + Subtype: 7000 + Arg1: start time + Arg2: end time + +RESPONSE: + PAYLOAD: + 0..3 timestamp (LE) + 4..7 yield in Wh (LE) + 8..11 unknown + PATTERN REPEATS +``` + +### Command: Historic data (daily intervals) +``` +COMMAND: + A2: E0 + B1,B2: 00 00 + C1,C2: 00 00 + Type: 0200 + Subtype: 7020 + Arg1: start time (unix date, LE) + Arg2: end time (unix date, LE) + +RESPONSE: + PAYLOAD: + 0..3 timestamp (unix date, LE) + 4..7 total yield at that time in Wh (LE) + 8..11 ??? + ... Pattern repeated +``` + +### Command: Set time +``` +COMMAND: + A2: A0 + B1,B2: 00 00 + C1,C2: 00 00 + Type: 020A + Subtype: F000 + Arg1: 0x00236d00 + Arg2: 0x00236d00 + PAYLOAD: + 0..3 00 6D 23 00 + 4..7 timestamp + 8..11 timestamp (again) + 12..15 timestamp (again) + 16..17 localtime offset from UTC in seconds + 18..19 00 00 + 20..23 30 FE 7E 00 + 24..27 01 00 00 00 + +RESPONSE: + PAYLOAD: +``` +``` +(smadata_venv) C:\workspace\python-smadata2>python sma2mon info TypeLabel +C:\workspace\.smadata2.json +System 20 Medway: + 20 Medway 1: +TypeLabel +func:'wait_6560_multi' args:[(> 24; + time_t datetime = (time_t)get_long(pcktBuf + ii + 4); + +RESPONSE: + PAYLOAD: + 0 index? 01 + 1..2 data type, 0x821E, corresponds to middle 2 bytes of arg1 LriDef in SMASpot + 3 datatype 0x10 =text, 0x08 = status, 0x00, 0x40 = Dword 64 bit data + 4..7 timestamp (unix date, LE) + 4..7 total yield at that time in Wh (LE) + 8..22 text string, terminated in 00 00 10 + 23.31 padding 00 + ... Pattern repeated on 40 byte cycle, 4 times +``` +``` \ No newline at end of file diff --git a/doc/testing.md b/doc/testing.md new file mode 100644 index 0000000..43a2e33 --- /dev/null +++ b/doc/testing.md @@ -0,0 +1,41 @@ + +(smadata_venv) C:\workspace\python-smadata2\smadata2>nosetests test_config.py +..............F.... +====================================================================== +FAIL: smadata2.test_config.TestConfigUTCSystem.test_timezone +---------------------------------------------------------------------- +Traceback (most recent call last): + File "c:\users\frigaarda\envs\smadata_venv\lib\site-packages\nose\case.py", line 198, in runTest + self.test(*self.arg) + File "C:\workspace\python-smadata2\smadata2\test_config.py", line 166, in test_timezone + assert_equals(dt.tzname(), "UTC") +AssertionError: 'Coordinated Universal Time' != 'UTC' +- Coordinated Universal Time ++ UTC + + +---------------------------------------------------------------------- +Ran 19 tests in 0.072s + +FAILED (failures=1) + +(smadata_venv) C:\workspace\python-smadata2\smadata2>nosetests sma2mon.py + +---------------------------------------------------------------------- +Ran 0 tests in 0.000s + +OK + +(smadata_venv) C:\workspace\python-smadata2\smadata2>nosetests test_sma2mon.py +. +---------------------------------------------------------------------- +Ran 1 test in 0.008s + +OK + +(smadata_venv) C:\workspace\python-smadata2\smadata2>nosetests test_datetimeutil.py +...... +---------------------------------------------------------------------- +Ran 6 tests in 0.743s + +OK \ No newline at end of file diff --git a/doc/usage.md b/doc/usage.md new file mode 100644 index 0000000..4c4e441 --- /dev/null +++ b/doc/usage.md @@ -0,0 +1,107 @@ +# Python SMAData2 Usage + +This shows examples of the available commands and typical output + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Prerequisites + + + +OS: Should run on Linux with Bluetooth support. Tested with a Raspberry Pi Zero W running Jessie/Debian. The application will also run under Windows, but requires PyBluez for Bluetooth support. + + +### sma2mon commands + +The application works by issuing a command in the form of ``python3 sma2mon [argument] ``. Command-line arguments are handled by argparse [https://docs.python.org/3/library/argparse.html.]() + +#### --help + +```sh +pi@raspberrypi:~ $ python-smadata2/sma2mon --help +usage: sma2mon [-h] [--config CONFIG] + {status,yieldat,download,setupdb,settime,upload,historic_daily,spotacvoltage} + ... + +Work with Bluetooth enabled SMA photovoltaic inverters + +positional arguments: + {status,yieldat,download,setupdb,settime,upload,historic_daily,spotacvoltage} + status Read inverter status + yieldat Get production at a given date + download Download power history and record in database + setupdb Create database or update schema + settime Update inverters' clocks + upload Upload power history to pvoutput.org + historic_daily Get historic production for a date range + spotacvoltage Get spot AC voltage now. + +optional arguments: + -h, --help show this help message and exit + --config CONFIG +``` +#### status +Establish a connection and Read inverter status +```sh +pi@raspberrypi:~ $ python3 python-smadata2/sma2mon status + +System 20 Medway: + 20 Medway 1: + Daily generation at Mon, 07 Oct 2019 18:15:35 ACDT: 17276 Wh + Total generation at Mon, 07 Oct 2019 18:15:39 ACDT: 40111820 Wh +``` +#### yieldat +Get production at a given date +```sh +pi@raspberrypi:~ $ python-smadata2/sma2mon yieldat "2019-02-14" +System 20 Medway: + Total generation at 2019-02-14 00:00:00+10:30: 73189184 Wh +pi@raspberrypi:~ $ python-smadata2/sma2mon yieldat "2019-03-14" +System 20 Medway: + Total generation at 2019-03-14 00:00:00+10:30: 37315778 Wh + +``` +#### download +Download power history and record in database +```sh +pi@raspberrypi:~ $ python-smadata2/sma2mon download +20 Medway 1 (SN: 2130212892) +starttime: 1546263000 +1546263000 +1546263000 +Downloaded 268 observations from Sun, 06 Oct 2019 20:20:00 ACDT to Mon, 07 Oct 2019 18:35:00 ACDT +Downloaded 1 daily observations from Mon, 07 Oct 2019 00:00:00 ACDT to Mon, 07 Oct 2019 00:00:00 ACDT +``` + +#### settime +Update inverters' clocks +ToDo - does this actually work? Example below not updating? + +```sh +pi@raspberrypi:~ $ python3 python-smadata2/sma2mon settime +/home/pi/.smadata2.json +20 Medway 1 (SN: 2130212892) + Previous time: Mon, 07 Oct 2019 19:29:05 ACDT + New time: Mon, 07 Oct 2019 19:36:57 ACDT (TZ 34201) + Updated time: Mon, 07 Oct 2019 19:29:05 ACDT +``` + +#### upload +Download power history and record in database +```sh +pi@raspberrypi:~ $ python-smadata2/sma2mon download +``` + +#### historic_daily [date_from] [date_to] +Get historic production for a date range +```sh +pi@raspberrypi:~ $ python-smadata2/sma2mon download +``` + +#### spotacvoltage +Get spot AC grid voltage now. +```sh +pi@raspberrypi:~ $ python-smadata2/sma2mon download +``` diff --git a/readme.md b/readme.md new file mode 100644 index 0000000..5664483 --- /dev/null +++ b/readme.md @@ -0,0 +1,230 @@ +# Python SMAData 2 + +Python code for communicating with SMA photovoltaic inverters within built in Bluetooth. + +The code originates from dgibson (written for his own use only) and I came across his project while looking for a fully Python-based SMA inverter tool, that would be easier to maintain & enhance than the various C-language projects, like SBFSpot. +I liked the code and spent some time to understand how it works and to set it up. It has some nice features for discovering the SMA protocol at the command line. + +The purpose of this fork initially is to make this code-base accessible to a wider audience with some good documentation. Then, depending on time, to extend with some other features. + +- Support for a wider range of inverter data, including real-time "spot" values. +- Sending inverter data via MQTT, for use in home automation, or remote monitoring. +- Provide a ready to use Docker image (for x86 and ARM) +- Make it possible to run the application on a ESP32 or ESP8266 +- Consolidate information on the protocol and commands for SMA Inverters - see ``/doc/protocol.md`` + + +## Getting Started + +These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system. + +### Prerequisites + + + +OS: Should run on Linux with Bluetooth support. Tested with a Raspberry Pi Zero W running Jessie/Debian. The application will also run under Windows, but requires PyBluez for Bluetooth support. + +Software: This requires Python 3.x, and was earlier converted from 2.7 by dgibson, the original author. I am running on 3.6, and am not aware of any version dependencies. + +Packages: +- It uses the "dateutil" external package for date/time formatting. +- PyBluez is used to provide Bluetooth functions on both Linux and Windows. +- readline was used for command line support, but is not required (legacy from 2.7?). + +Debugging: For remote debugging code on the Pi Zero I found web_pdb to be useful. [https://pypi.org/project/web-pdb/]() +This displays a remote debug session over http on port 5555, e.g. [http://192.168.1.25:5555/]() + +Testing: + +Hardware: This runs on a Linux PC with Bluetooth (e.g. Raspberry Pi Zero W). +Inverter: Any type of SunnyBoy SMA inverter that supports their Bluetooth interface. This seems to be most models from the last 10 years. However this has not been tested widely, only on a SMA5000TL + + +### Installing + + + +Install or clone the project to an appropriate location, for example. + +```sh +Linux: +/home/pi/python-smadata2 +Windows: +C:\workspace\python-smadata2 +``` +Install and activate a virtual environment, as needed. +Install any required packages + +Windows: To install Pybluez under Windows, pip may not work. It is more reliable to download the whl file from here +https://www.lfd.uci.edu/~gohlke/pythonlibs/#pybluez +Examples tutorial: +https://people.csail.mit.edu/albert/bluez-intro/x232.html + +```shell script +pip install -r requirements.txt +``` +Next install a configuration file and database. Copy the example file from the doc folder to your preferred location and edit for your local settings. + +```shell script +Example file: +/python-smadata2/doc/example.smadata2.json +Copy to: +~/.smadata2.json +``` +These lines in the json file determine the location of the database: +```json + "database": { + "filename": "~/.smadata2.sqlite" +``` + +The json file with configuration details (for development environment) should be stored separately, in a file stored in home, say: ```/home/pi/smadata2.json```. +This file should not be in Git, as it will contain the users confidential data. +There is an example provided in the source ```/doc/example.samdata2.json``` file and below. +The source file config.py references that file, so ensure that is correct for your environment: +```pythonstub +# for Linux +# DEFAULT_CONFIG_FILE = os.path.expanduser("~/.smadata2.json") +# DEFAULT_CONFIG_FILE = os.environ.get('USERPROFILE') + "\.smadata2.json" +# Windows +DEFAULT_CONFIG_FILE = "C:\workspace\.smadata2.json" +``` +Then run this command to create the database: +```shell script +pi@raspberrypi:~/python-smadata2 $ python3 sma2mon setupdb +Creating database '/home/pi/.smadata2.sqlite'... +``` + +TODO - where a new user can discover these values. + +## Settings file +The json file with configuration details (for development environment) should be stored separately, in the user's home, say: ```/home/pi/smadata2.json```. + +This file should not be in Git, as it will contain the users confidential data. + +There is an example provided in the source ```/doc/example.samdata2.json``` file and below. + +The source file ``config.py`` references that file, so ensure the path is correct for your environment: + +```json +{ + "database": { + "filename": "~/.smadata2.sqlite" + }, + "pvoutput.org": { + "apikey": "2a0ahhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh" + }, + "systems": [{ + "name": "My Photovoltaic System", + "pvoutput-sid": NNNNN, + "inverters": [{ + "name": "Inverter 1", + "bluetooth": "00:80:25:xx:yy:zz", + "serial": "2130012345", + "start-time": "2019-10-17", + "password": "1234" + }, { + "name": "Inverter 2", + "bluetooth": "00:80:25:pp:qq:rr", + "serial": "2130012346" + }] + }] +} + +``` +These are optional parameters: + +Todo - full explanation +```json + "start-time": "2019-10-17", + "password": "0000" +``` + +If all is setup correctly, then run an example command likst ``sma2mon status`` which will login to the SMA device and report on the +daily generation: + +```sh +pi@raspberrypi:~/python-smadata2 $ python3 sma2mon status +System 2My Photovoltaic System: + Inverter 1: + Daily generation at Sun, 20 Oct 2019 21:27:40 ACDT: 30495 Wh + Total generation at Sun, 20 Oct 2019 19:43:37 ACDT: 40451519 Wh +pi@raspberrypi:~/python-smadata2 $ + +``` +For further commands see the other documents in the ``/doc`` folder + +## Running the tests + +Explain how to run the automated tests for this system + +### Break down into end to end tests + +Explain what these tests test and why + +``` +Give an example +``` + +### And coding style tests + +Explain what these tests test and why + +``` +Give an example +``` + +## Deployment + +See also the ``/doc/usage.md`` file for explanation and examples of the command line options. + +### on Raspberry Pi +I have been running this on a dedicated Raspberry Pi Zero W (built-in Wifi and Bluetooth). This is convenient as it can be located close to the inverter (Bluetooth range ~5m) and within Wifi range of the home router. It runs headless (no display) and any changes are made via SSH, VNC. + +The package is copied to an appropriate location, say: ```/home/pi/python-smadata2``` and another directory for the database, say: ```/home/pi/python-smadata2```. +The json file with configuration details (local configuration for that environment) should be stored separately, in a file stored in the user's home, say: ```/home/pi/smadata2.json``` + +This file is referenced in the config object loaded from ``smadata2/config.py`` on startup +```pythonstub +# Linux +DEFAULT_CONFIG_FILE = os.path.expanduser("~/.smadata2.json") +``` +### on Windows +This does run on Windows device with built-in Bluetooth. + +The json file with configuration details (local configuration for that environment) should be stored separately, in a file stored in the user's profile, say: ```C:\Users\\smadata2.json``` + +This file is referenced in the config object loaded from ``smadata2/config.py`` on startup +```pythonstub +# Windows +DEFAULT_CONFIG_FILE = "C:\Users\\.smadata2.json" +``` + +## Built With + + +## Contributing + +Feedback is very welcome, and suggestions for other features. Do log an issue. + +Testing with other SMA devices is also needed. The protocol should be similar for all devices. + + + +## License + +This project is licensed under the GNU General Public License - see [https://www.gnu.org/licenses/]() . + +## Acknowledgments + +* dgibson [https://github.com/dgibson/python-smadata2]() +* SBFspot [https://github.com/SBFspot/SBFspot]() +* Stuart Pittaway [https://github.com/stuartpittaway/nanodesmapvmonitor]() diff --git a/requirements.txt b/requirements.txt index e276fe2..8dbdff9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ python-dateutil coverage nose +PyBluez diff --git a/sma2-correct-date b/sma2-correct-date index 12873e3..99f3cf2 100755 --- a/sma2-correct-date +++ b/sma2-correct-date @@ -89,7 +89,7 @@ def main(argv=sys.argv): db = config.database() for system in config.systems(): for inv in system.inverters(): - do_inv(system, inv, db) + do_inv(system, inv, db) if __name__ == '__main__': main() diff --git a/sma2-explore b/sma2-explore index 15bbdb1..a108a2d 100755 --- a/sma2-explore +++ b/sma2-explore @@ -17,12 +17,15 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from __future__ import print_function -from __future__ import division +#AF checked and file is python 3, so not needed +#from __future__ import print_function +#from __future__ import division import sys import os import signal -import readline +#import gnureadline #readline is deprecated +#import pyreadline +import threading import time from smadata2.inverter.smabluetooth import Connection @@ -81,7 +84,7 @@ def dump_ppp(prefix, protocol, payload): def a65602str(addr): return "%02X.%02X.%02X.%02X.%02X.%02X" % tuple(addr) - +#called from def dump_6560(prefix, from2, to2, a2, b1, b2, c1, c2, tag, type_, subtype, arg1, arg2, extra, response, error, pktcount, first): @@ -109,6 +112,11 @@ def dump_6560(prefix, from2, to2, a2, b1, b2, c1, c2, tag, class SMAData2CLI(Connection): + """CLI class overrides the Connection class defined in smabluetooth + + Implements most of the same functions, but calls a dump_ function first + the dump function prints a formatted packet format to the stdout + """ def __init__(self, addr): super(SMAData2CLI, self).__init__(addr) print("Connected %s -> %s" @@ -119,19 +127,36 @@ class SMAData2CLI(Connection): if self.rxpid: os.kill(self.rxpid, signal.SIGTERM) + # Function call order + # wait() > rx() > rx-raw() > rx_outer() > rx_ppp_raw() > rx_ppp() >rx_6560 > rxfilter_6560 + def rxloop(self): while True: + #print ("Thread: running") self.rx() + # todo can we use https://docs.python.org/3.6/library/multiprocessing.html or asyncio so this works in windows? + # This attempt to work with threading did not work on windows - does not return control to the main thread + # def start_rxthread(self): + # print('start_rxthread') + # print(__name__) + # print(self.rxpid) + # if __name__ == '__main__': + # p = threading.Thread(target = self.rxloop(), args=(), daemon=True, name="rx_listen") + # p.start() + # print('start_main_thread') + + def start_rxthread(self): - self.rxpid = os.fork() - if self.rxpid == 0: + self.rxpid = os.fork() # creates another process which will resume at exactly the same place as this one + if self.rxpid == 0: # is zero for child process while True: try: self.rxloop() except Exception as e: print(e) + def rx_raw(self, pkt): print("\n" + hexdump(pkt, "Rx< ")) super(SMAData2CLI, self).rx_raw(pkt) @@ -153,6 +178,7 @@ class SMAData2CLI(Connection): dump_ppp("Rx< ", protocol, payload) super(SMAData2CLI, self).rx_ppp(from_, protocol, payload) + # in rx_ppp this is set: error = bytes2int(payload[18:20]) def rx_6560(self, from2, to2, a2, b1, b2, c1, c2, tag, type_, subtype, arg1, arg2, extra, response, error, pktcount, first): @@ -170,6 +196,14 @@ class SMAData2CLI(Connection): print("\n" + hexdump(pkt, "Tx> ")) def tx_outer(self, from_, to_, type_, payload): + """Builds a SMA Level 1 packet from supplied Level 2, calls tx_raw to transmit + + :param from_: str source Bluetooth address in string representation + :param to_: str destination Bluetooth address in string representation + :param type_: int the command to send, e.g. OTYPE_PPP = 0x01 L2 Packet start + :param payload: bytearray Data or payload in Level 1 packet + :return: + """ super(SMAData2CLI, self).tx_outer(from_, to_, type_, payload) dump_outer("Tx> ", from_, to_, type_, payload) @@ -238,11 +272,21 @@ class SMAData2CLI(Connection): self.tx_outer(from_, to_, type_, payload) def cmd_hello(self): + """Level 1 hello command responds to the SMA with the same data packet sent, + + the smabluetooth.hello function checks hello packet NetID from inverter and returns the same one + This is hardcoded, as the inbound packet has not been read. + Byte 5 below was x01, needed x04 - inspect the packet from your inverter. + """ self.tx_outer("00:00:00:00:00:00", self.remote_addr, OTYPE_HELLO, - bytearray('\x00\x04\x70\x00\x01\x00\x00\x00' + - '\x00\x01\x00\x00\x00')) +# bytearray(b'\x00\x04\x70\x00\x01\x00\x00\x00\x00\x01\x00\x00\x00')) + bytearray(b'\x00\x04\x70\x00\x04\x00\x00\x00\x00\x01\x00\x00\x00')) #from TL5000 def cmd_getvar(self, varid): + """Level 1 getvar requests the value or a variable from the SMA inverter + + values include: + """ varid = int(varid, 16) self.tx_outer("00:00:00:00:00:00", self.remote_addr, OTYPE_GETVAR, int2bytes16(varid)) @@ -253,6 +297,20 @@ class SMAData2CLI(Connection): self.tx_ppp("ff:ff:ff:ff:ff:ff", protocol, payload) def cmd_send2(self, *args): + """Sends a Level 2 packet request to the inverter + + COMMAND: + A2: A0 + B1,B2: 00 00 + C1,C2: 00 00 + Type: 0200 + Subtype: 5400 + Arg1: 0x00260100 + Arg2: 0x002601ff + + :param args: + :return: + """ bb = [int(x, 16) for x in args] a2 = bb[0] b1, b2 = bb[1], bb[2] @@ -264,8 +322,8 @@ class SMAData2CLI(Connection): self.tx_6560(self.local_addr2, self.BROADCAST2, a2, b1, b2, c1, c2, self.gettag(), type_, subtype, arg1, arg2, extra) - - def cmd_logon(self, password='0000', timeout=900): + #AF added b' + def cmd_logon(self, password=b'0000', timeout=900): timeout = int(timeout) self.tx_logon(password, timeout) @@ -275,6 +333,9 @@ class SMAData2CLI(Connection): def cmd_yield(self): self.tx_yield() + def cmd_spotacvoltage(self): + self.tx_spotacvoltage() + def cmd_historic(self, fromtime=None, totime=None): if fromtime is None and totime is None: fromtime = 1356958800 # 1 Jan 2013 @@ -291,6 +352,9 @@ if __name__ == '__main__': sys.exit(1) cli = SMAData2CLI(sys.argv[1]) - + # attempt to thread under Windows + # p = threading.Thread(target=cli.rxloop(), args=(), daemon=True, name="rx_listen") + # p = threading.Thread(target=cli.cli(), args=(), daemon=True, name="rx_listen") + # p.start() cli.start_rxthread() cli.cli() diff --git a/sma2mon b/sma2mon index 60186e4..cc73772 100755 --- a/sma2mon +++ b/sma2mon @@ -2,5 +2,6 @@ import smadata2.sma2mon +# main entry point when running from command line if __name__ == '__main__': smadata2.sma2mon.main() diff --git a/smadata2/__init__.py b/smadata2/__init__.py index 9364d26..ec887ae 100644 --- a/smadata2/__init__.py +++ b/smadata2/__init__.py @@ -16,3 +16,9 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import logging.config +from smadata2.logging_config import SMA2_LOGGING_CONFIG #only this, in case file has other unwanted content + +logging.config.dictConfig(SMA2_LOGGING_CONFIG) +#log = logging.getLogger(__name__) # once in each module \ No newline at end of file diff --git a/smadata2/config.py b/smadata2/config.py index bda17cf..706d467 100644 --- a/smadata2/config.py +++ b/smadata2/config.py @@ -23,16 +23,38 @@ import dateutil.tz import json +import logging.config + from .inverter import smabluetooth from . import pvoutputorg from . import datetimeutil from . import db -DEFAULT_CONFIG_FILE = os.path.expanduser("~/.smadata2.json") +# for var in ('HOME', 'USERPROFILE', 'HOMEPATH', 'HOMEDRIVE'): +var = os.environ.get('USERPROFILE') +# for Linux +# DEFAULT_CONFIG_FILE = os.path.expanduser("~/.smadata2.json") +# DEFAULT_CONFIG_FILE = os.environ.get('USERPROFILE') + "\.smadata2.json" +# Windows +DEFAULT_CONFIG_FILE = "C:\workspace\.smadata2.json" +# print(DEFAULT_CONFIG_FILE) class SMAData2InverterConfig(object): + """Represents a PV Inverter defined in the config file with properties: inverters, timezone + + Args: + invjson (str): json string describing the inverter. + code (:obj:`int`, optional): Error code. + + Attributes: + bdaddr (str): Bluetooth address in hex, like '00:80:25:2C:11:B2' + serial (str): Inverter serial, like + name (str): Inverter name, like "West-facing" + """ + def __init__(self, invjson, defname): + self.log = logging.getLogger(__name__) self.bdaddr = invjson["bluetooth"] self.serial = invjson["serial"] self.name = invjson.get("name", defname) @@ -40,14 +62,21 @@ def __init__(self, invjson, defname): self.starttime = datetimeutil.parse_time(invjson["start-time"]) else: self.starttime = None - + if "password" in invjson: + self.password = bytearray(invjson["password"],encoding="utf-8", errors="ignore") + else: + self.password = b"0000" def connect(self): return smabluetooth.Connection(self.bdaddr) def connect_and_logon(self): + """ Make Bluetooth connection the device + + :return: conn, connection from the smabluetooth Connection + """ conn = self.connect() conn.hello() - conn.logon() + conn.logon(password=self.password) return conn def __str__(self): @@ -57,7 +86,21 @@ def __str__(self): class SMAData2SystemConfig(object): + """Represents the PV System defined in the config file with properties: inverters, timezone + + Args: + invjson (str): json string describing the inverter. + code (:obj:`int`, optional): Error code. + + Attributes: + name (str): System name, like "Medway farm" + pvoutput_sid (str): SID, system identifier, from manufacturer + tz (str): Timezone, like + + """ + def __init__(self, index, sysjson=None, invjson=None): + self.log = logging.getLogger(__name__) if sysjson: assert invjson is None @@ -97,7 +140,15 @@ def __str__(self): class SMAData2Config(object): + """Reads the json config file, generates systems list, database and pvoutput + + Attributes: + configfile (file): json file, defaults is DEFAULT_CONFIG_FILE above + + """ + def __init__(self, configfile=None): + self.log = logging.getLogger(__name__) if configfile is None: configfile = DEFAULT_CONFIG_FILE diff --git a/smadata2/datetimeutil.py b/smadata2/datetimeutil.py index ea4d9c4..b1f7384 100644 --- a/smadata2/datetimeutil.py +++ b/smadata2/datetimeutil.py @@ -52,6 +52,10 @@ def format_time(timestamp): st = time.localtime(timestamp) return time.strftime("%a, %d %b %Y %H:%M:%S %Z", st) +def format_time2(timestamp): + st = time.localtime(timestamp) + return time.strftime("%d/%m/%y %X", st) + def format_date(timestamp): st = time.localtime(timestamp) diff --git a/smadata2/download.py b/smadata2/download.py index 2ff8ffb..19502ca 100644 --- a/smadata2/download.py +++ b/smadata2/download.py @@ -23,13 +23,28 @@ def download_type(ic, db, sample_type, data_fn): + """Gets a data set from the inverter, fast or daily samples, using the provided data_fn, + + Checks the database for last sample time, and then passes to the data_fn + the data_fn does all the work to query the inverter and parse the packets + data_fn is from smabluetooth, is one of (sma.historic, sma.historic_daily..) + Timestamps are int like 1548523800 (17/02/2019) + + :param ic: inverter + :param db: sqlite database object + :param sample_type: fast or daily samples, SAMPLE_INV_FAST, SAMPLE_INV_DAILY defined in db class + :param data_fn: calling function, like data_fn = eventData; +# long calPdcTot; +# long calPacTot; +# float calEfficiency; +# unsigned long BatChaStt; // Current battery charge status +# unsigned long BatDiagCapacThrpCnt; // Number of battery charge throughputs +# unsigned long BatDiagTotAhIn; // Amp hours counter for battery charge +# unsigned long BatDiagTotAhOut; // Amp hours counter for battery discharge +# unsigned long BatTmpVal; // Battery temperature +# unsigned long BatVol; // Battery voltage +# long BatAmp; // Battery current +# long Temperature; // Inverter Temperature +# int32_t MeteringGridMsTotWOut; // Power grid feed-in (Out) +# int32_t MeteringGridMsTotWIn; // Power grid reference (In) +# bool hasBattery; // Smart Energy device +# } InverterData; + +# From SBFSpot.h October 2019 +# Lists all the SMA device types +# Todo confirm this is ENUM or actual SMA device, incorporate in data request unpacking +# part of TypeLabel response INV_CLASS +# typedef enum +# { +# AllDevices = 8000, // DevClss0 +# SolarInverter = 8001, // DevClss1 +# WindTurbineInverter = 8002, // DevClss2 +# BatteryInverter = 8007, // DevClss7 +# Consumer = 8033, // DevClss33 +# SensorSystem = 8064, // DevClss64 +# ElectricityMeter = 8065, // DevClss65 +# CommunicationProduct = 8128 // DevClss128 +# } DEVICECLASS; + +# todo - use this? +# from Archdata.cpp +# ArchiveDayData +# writeLong(pcktBuf, 0x70000200); +# writeLong(pcktBuf, startTime - 300); +# writeLong(pcktBuf, startTime + 86100); +# +# ArchiveMonthData +# writeLong(pcktBuf, 0x70200200); +# writeLong(pcktBuf, startTime - 86400 - 86400); +# writeLong(pcktBuf, startTime + 86400 * (sizeof(inverters[inv]->monthData) / sizeof(MonthData) + 1)); +# +# ArchiveEventData +# writeLong(pcktBuf, UserGroup == UG_USER ? 0x70100200 : 0x70120200); +# writeLong(pcktBuf, startTime); +# writeLong(pcktBuf, endTime); +# + + +# source of this is the LriDef in SBFspot.h +# todo, add datatype to this list +# Dictionary, used to lookup SMA data element parameters +# :param type: SMA request type mostly 0x0200 +# :param subtype:SMA request subtype often 0x5100 + +# :param arg1: pointer to range: from +# :param element_name: short name for element +# data type code, SMA, 4 values 0x10 =text, 0x08 = status, 0x00, 0x40 = Dword 64 bit data +# :param extra: normally 0 +# todo - add these items and merge the two lists, or consider multi-language? + +sma_data_element ={ +0x2148: ('OperationHealth', 0x08, 'Condition (aka INV_STATUS)', '','','',1), +0x2377: ('CoolsysTmpNom', 0x40, 'Operating condition temperatures', '','','',1), +0x251E: ('DcMsWatt', 0x40, 'DC power input (aka SPOT_PDC1 / SPOT_PDC2)', 'DC spot Power String', 'W', 'Watts', 1), +0x2601: ('MeteringTotWhOut', 0x00, 'Total yield (aka SPOT_ETOTAL)', 'Total generated', 'kWh', 'kiloWatt hours', 1000), +0x2622: ('MeteringDyWhOut', 0x00, 'Day yield (aka SPOT_ETODAY)','Total generated today', 'kWh', 'kiloWatt hours', 1000), +0x263F: ('GridMsTotW', 0x40, 'Power (aka SPOT_PACTOT)', 'Power now', 'W', 'Watts', 1), #//This function gives us the time when the inverter was switched off +0x295A: ('BatChaStt', 0x00, 'Current battery charge status', '','','',1), +0x411E: ('OperationHealthSttOk', 0x00, 'Nominal power in Ok Mode (aka INV_PACMAX1)', 'Nominal power OK Mode', 'W', 'Watts', 1), +0x411F: ('OperationHealthSttWrn', 0x00, 'Nominal power in Warning Mode (aka INV_PACMAX2)', 'Nominal power Warning Mode', 'W', 'Watts', 1), +0x4120: ('OperationHealthSttAlm', 0x00, 'Nominal power in Fault Mode (aka INV_PACMAX3)', 'Nominal power Fault Mode', 'W', 'Watts', 1), +0x4164: ('OperationGriSwStt', 0x08, 'Grid relay/contactor (aka INV_GRIDRELAY)', '','','',1), +0x4166: ('OperationRmgTms', 0x00, 'Waiting time until feed-in', '','','',1), +0x451F: ('DcMsVol', 0x40, 'DC voltage input (aka SPOT_UDC1 / SPOT_UDC2)', 'DC voltage String', 'V', 'Volts', 100), +0x4521: ('DcMsAmp', 0x40, 'DC current input (aka SPOT_IDC1 / SPOT_IDC2)', 'DC current String', 'mA', 'milli Amps', 1), +0x4623: ('MeteringPvMsTotWhOut', 0x00, 'PV generation counter reading'), +0x4624: ('MeteringGridMsTotWhOut', 0x00, 'Grid feed-in counter reading', 'Grid Counter? ', 'Wh', 'Watt Hours', 1), +0x4625: ('MeteringGridMsTotWhIn', 0x00, 'Grid reference counter reading','Grid Counter? ', 'Wh', 'Watt Hours', 1), +0x4626: ('MeteringCsmpTotWhIn', 0x00, 'Meter reading consumption meter', 'Grid Counter? ', 'Wh', 'Watt Hours', 1), +0x4627: ('MeteringGridMsDyWhOut', 0x00, '?'), +0x4628: ('MeteringGridMsDyWhIn', 0x00, '?'), +0x462E: ('MeteringTotOpTms', 0x00, 'Operating time (aka SPOT_OPERTM)', 'Inverter operating time', 's', 'Seconds', 1), +0x462F: ('MeteringTotFeedTms', 0x00, 'Feed-in time (aka SPOT_FEEDTM)', 'Inverter feed-in time', 's', 'Seconds', 1), +0x4631: ('MeteringGriFailTms', 0x00, 'Power outage', '','','',1), +0x463A: ('MeteringWhIn', 0x00, 'Absorbed energy', '','','',1), +0x463B: ('MeteringWhOut', 0x00, 'Released energy', '','','',1), +0x4635: ('MeteringPvMsTotWOut', 0x40, 'PV power generated', '','','',1), +0x4636: ('MeteringGridMsTotWOut', 0x40, 'Power grid feed-in', '','','',1), +0x4637: ('MeteringGridMsTotWIn', 0x40, 'Power grid reference', '','','',1), +0x4639: ('MeteringCsmpTotWIn', 0x40, 'Consumer power', '','','',1), +0x4640: ('GridMsWphsA', 0x40, 'Power L1 (aka SPOT_PAC1)', '','','',1), +0x4641: ('GridMsWphsB', 0x40, 'Power L2 (aka SPOT_PAC2)', '','','',1), +0x4642: ('GridMsWphsC', 0x40, 'Power L3 (aka SPOT_PAC3)', '','','',1), +0x4648: ('GridMsPhVphsA', 0x00, 'Grid voltage phase L1 (aka SPOT_UAC1)', 'AC spot line voltage phase 1', 'V', 'Volts', 100), +0x4649: ('GridMsPhVphsB', 0x00, 'Grid voltage phase L2 (aka SPOT_UAC2)', 'AC spot line voltage phase 2', 'V', 'Volts', 100), +0x464A: ('GridMsPhVphsC', 0x00, 'Grid voltage phase L3 (aka SPOT_UAC3)', 'AC spot line voltage phase 3', 'V', 'Volts', 100), +0x4650: ('GridMsAphsA_1', 0x00, 'Grid current phase L1 (aka SPOT_IAC1)', 'AC spot current phase 1', 'mA', 'milli Amps', 1), +0x4651: ('GridMsAphsB_1', 0x00, 'Grid current phase L2 (aka SPOT_IAC2)', 'AC spot current phase 2', 'mA', 'milli Amps', 1), +0x4652: ('GridMsAphsC_1', 0x00, 'Grid current phase L3 (aka SPOT_IAC3)', 'AC spot current phase 3', 'mA', 'milli Amps', 1), +0x4653: ('GridMsAphsA', 0x00, 'Grid current phase L1 (aka SPOT_IAC1_2)'), +0x4654: ('GridMsAphsB', 0x00, 'Grid current phase L2 (aka SPOT_IAC2_2)'), +0x4655: ('GridMsAphsC', 0x00, 'Grid current phase L3 (aka SPOT_IAC3_2)'), +0x4657: ('GridMsHz', 0x00, 'Grid frequency (aka SPOT_FREQ)', 'Spot Grid frequency', 'Hz', 'Hertz', 100), +0x46AA: ('MeteringSelfCsmpSelfCsmpWh', 0x00, 'Energy consumed internally', '','','',1), +0x46AB: ('MeteringSelfCsmpActlSelfCsmp', 0x00, 'Current self-consumption', '','','',1), +0x46AC: ('MeteringSelfCsmpSelfCsmpInc', 0x00, 'Current rise in self-consumption', '','','',1), +0x46AD: ('MeteringSelfCsmpAbsSelfCsmpInc', 0x00, 'Rise in self-consumption', '','','',1), +0x46AE: ('MeteringSelfCsmpDySelfCsmpInc', 0x00, 'Rise in self-consumption today', '','','',1), +0x491E: ('BatDiagCapacThrpCnt', 0x40, 'Number of battery charge throughputs', '','','',1), +0x4926: ('BatDiagTotAhIn', 0x00, 'Amp hours counter for battery charge', '','','',1), +0x4927: ('BatDiagTotAhOut', 0x00, 'Amp hours counter for battery discharge', '','','',1), +0x495B: ('BatTmpVal', 0x40, 'Battery temperature', '','','',1), +0x495C: ('BatVol', 0x40, 'Battery voltage', '','','',1), +0x495D: ('BatAmp', 0x40, 'Battery current', '','','',1), +0x821E: ('NameplateLocation', 0x10, 'Device name (aka INV_NAME)', '','','',1), +0x821F: ('NameplateMainModel', 0x08, 'Device class (aka INV_CLASS)', '','','',1), +0x8220: ('NameplateModel', 0x08, 'Device type (aka INV_TYPE)', '','','',1), +0x8221: ('NameplateAvalGrpUsr', 0x00, 'Unknown', '','','',1), +0x8234: ('NameplatePkgRev', 0x08, 'Software package (aka INV_SWVER)', '','','',1), +0x832A: ('InverterWLim', 0x00, 'Maximum active power device (aka INV_PACMAX1_2) (Some inverters like SB3300/SB1200)', '','','',1), +0x464B: ('GridMsPhVphsA2B6100', 0x00, 'Grid voltage new-undefined', '','','',1), +0x464C: ('GridMsPhVphsB2C6100', 0x00, 'Grid voltage new-undefined', '','','',1), +0x464D: ('GridMsPhVphsC2A6100', 0x00, 'Grid voltage new-undefined', '','','',1), +} + +# From SBFSpot.h October 2019 +# Lists all the SMA requests with the arg1 parameter (start of range in SMA data register) +# for example, from dict sma_request_type, this entry: +# // SPOT_UAC1, SPOT_UAC2, SPOT_UAC3, SPOT_IAC1, SPOT_IAC2, SPOT_IAC3 +# 'SpotACVoltage': (0x0200, 0x5100, 0x00464800, 0x004655FF, 0), +# corresponds to the 0x00464800 line below: +# GridMsPhVphsA = 0x00464800, // *00* Grid voltage phase L1 (aka SPOT_UAC1) +# typedef enum +# { +# OperationHealth = 0x00214800, // *08* Condition (aka INV_STATUS) +# CoolsysTmpNom = 0x00237700, // *40* Operating condition temperatures +# DcMsWatt = 0x00251E00, // *40* DC power input (aka SPOT_PDC1 / SPOT_PDC2) +# MeteringTotWhOut = 0x00260100, // *00* Total yield (aka SPOT_ETOTAL) +# MeteringDyWhOut = 0x00262200, // *00* Day yield (aka SPOT_ETODAY) +# GridMsTotW = 0x00263F00, // *40* Power (aka SPOT_PACTOT) +# BatChaStt = 0x00295A00, // *00* Current battery charge status +# OperationHealthSttOk = 0x00411E00, // *00* Nominal power in Ok Mode (aka INV_PACMAX1) +# OperationHealthSttWrn = 0x00411F00, // *00* Nominal power in Warning Mode (aka INV_PACMAX2) +# OperationHealthSttAlm = 0x00412000, // *00* Nominal power in Fault Mode (aka INV_PACMAX3) +# OperationGriSwStt = 0x00416400, // *08* Grid relay/contactor (aka INV_GRIDRELAY) +# OperationRmgTms = 0x00416600, // *00* Waiting time until feed-in +# DcMsVol = 0x00451F00, // *40* DC voltage input (aka SPOT_UDC1 / SPOT_UDC2) +# DcMsAmp = 0x00452100, // *40* DC current input (aka SPOT_IDC1 / SPOT_IDC2) +# MeteringPvMsTotWhOut = 0x00462300, // *00* PV generation counter reading +# MeteringGridMsTotWhOut = 0x00462400, // *00* Grid feed-in counter reading +# MeteringGridMsTotWhIn = 0x00462500, // *00* Grid reference counter reading +# MeteringCsmpTotWhIn = 0x00462600, // *00* Meter reading consumption meter +# MeteringGridMsDyWhOut = 0x00462700, // *00* ? +# MeteringGridMsDyWhIn = 0x00462800, // *00* ? +# MeteringTotOpTms = 0x00462E00, // *00* Operating time (aka SPOT_OPERTM) +# MeteringTotFeedTms = 0x00462F00, // *00* Feed-in time (aka SPOT_FEEDTM) +# MeteringGriFailTms = 0x00463100, // *00* Power outage +# MeteringWhIn = 0x00463A00, // *00* Absorbed energy +# MeteringWhOut = 0x00463B00, // *00* Released energy +# MeteringPvMsTotWOut = 0x00463500, // *40* PV power generated +# MeteringGridMsTotWOut = 0x00463600, // *40* Power grid feed-in +# MeteringGridMsTotWIn = 0x00463700, // *40* Power grid reference +# MeteringCsmpTotWIn = 0x00463900, // *40* Consumer power +# GridMsWphsA = 0x00464000, // *40* Power L1 (aka SPOT_PAC1) +# GridMsWphsB = 0x00464100, // *40* Power L2 (aka SPOT_PAC2) +# GridMsWphsC = 0x00464200, // *40* Power L3 (aka SPOT_PAC3) +# GridMsPhVphsA = 0x00464800, // *00* Grid voltage phase L1 (aka SPOT_UAC1) +# GridMsPhVphsB = 0x00464900, // *00* Grid voltage phase L2 (aka SPOT_UAC2) +# GridMsPhVphsC = 0x00464A00, // *00* Grid voltage phase L3 (aka SPOT_UAC3) +# GridMsAphsA_1 = 0x00465000, // *00* Grid current phase L1 (aka SPOT_IAC1) +# GridMsAphsB_1 = 0x00465100, // *00* Grid current phase L2 (aka SPOT_IAC2) +# GridMsAphsC_1 = 0x00465200, // *00* Grid current phase L3 (aka SPOT_IAC3) +# GridMsAphsA = 0x00465300, // *00* Grid current phase L1 (aka SPOT_IAC1_2) +# GridMsAphsB = 0x00465400, // *00* Grid current phase L2 (aka SPOT_IAC2_2) +# GridMsAphsC = 0x00465500, // *00* Grid current phase L3 (aka SPOT_IAC3_2) +# GridMsHz = 0x00465700, // *00* Grid frequency (aka SPOT_FREQ) +# MeteringSelfCsmpSelfCsmpWh = 0x0046AA00, // *00* Energy consumed internally +# MeteringSelfCsmpActlSelfCsmp = 0x0046AB00, // *00* Current self-consumption +# MeteringSelfCsmpSelfCsmpInc = 0x0046AC00, // *00* Current rise in self-consumption +# MeteringSelfCsmpAbsSelfCsmpInc = 0x0046AD00, // *00* Rise in self-consumption +# MeteringSelfCsmpDySelfCsmpInc = 0x0046AE00, // *00* Rise in self-consumption today +# BatDiagCapacThrpCnt = 0x00491E00, // *40* Number of battery charge throughputs +# BatDiagTotAhIn = 0x00492600, // *00* Amp hours counter for battery charge +# BatDiagTotAhOut = 0x00492700, // *00* Amp hours counter for battery discharge +# BatTmpVal = 0x00495B00, // *40* Battery temperature +# BatVol = 0x00495C00, // *40* Battery voltage +# BatAmp = 0x00495D00, // *40* Battery current +# NameplateLocation = 0x00821E00, // *10* Device name (aka INV_NAME) +# NameplateMainModel = 0x00821F00, // *08* Device class (aka INV_CLASS) +# NameplateModel = 0x00822000, // *08* Device type (aka INV_TYPE) +# NameplateAvalGrpUsr = 0x00822100, // * * Unknown +# NameplatePkgRev = 0x00823400, // *08* Software package (aka INV_SWVER) +# InverterWLim = 0x00832A00, // *00* Maximum active power device (aka INV_PACMAX1_2) (Some inverters like SB3300/SB1200) +# GridMsPhVphsA2B6100 = 0x00464B00, +# GridMsPhVphsB2C6100 = 0x00464C00, +# GridMsPhVphsC2A6100 = 0x00464D00 +# } LriDef; diff --git a/smadata2/inverter/smabluetooth.py b/smadata2/inverter/smabluetooth.py index 2bfb917..376557e 100644 --- a/smadata2/inverter/smabluetooth.py +++ b/smadata2/inverter/smabluetooth.py @@ -17,14 +17,25 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + import sys import getopt import time import socket +import logging.config + +from smadata2.inverter.sma_devices import * +# AF for Windows +from bluetooth import * + +# development tools only, debugging, timing +import web_pdb # debugger use: web_pdb.set_trace() #set a breakpoint +from functools import wraps from . import base from .base import Error from smadata2.datetimeutil import format_time +from smadata2.datetimeutil import format_time2 __all__ = ['Connection', 'OTYPE_PPP', 'OTYPE_PPP2', 'OTYPE_HELLO', 'OTYPE_GETVAR', @@ -34,6 +45,7 @@ OUTER_HLEN = 18 +# commands in SMA Level 1 packet format - see document OTYPE_PPP = 0x01 OTYPE_HELLO = 0x02 OTYPE_GETVAR = 0x03 @@ -48,19 +60,53 @@ SMA_PROTOCOL_ID = 0x6560 +def timing(f): + """Decorator function to write duration of f to the console + + :param f: function to be timed + :return: duration in s + """ + + @wraps(f) + def wrap(*args, **kw): + ts = time.time() + result = f(*args, **kw) + te = time.time() + print('func:{!r} args:[{!r}, {!r}] took: {:2.4f} sec'.format(f.__name__, args, kw, te - ts)) + return result + + return wrap + + def waiter(fn): + """ Decorator function on the Rx functions, checks wait conditions on self, used with connection.wait() to wait for packets + + The trick is that the Rx functions have been decorated with @waiter, which augments the bare Rx function + with code to check if the special wait variables are set, and if so check the results of the Rx to + see if it's something we're currently waiting for, and if so put it somewhere that wait will be able to find it. + If the wait condition matches then save the args on the waitvar attribute. + attributes are created in the connection.wait() function below """ + def waitfn(self, *args): - fn(self, *args) + fn(self, *args) # call the provided function, with any arguments from the decorated function if hasattr(self, '__waitcond_' + fn.__name__): wc = getattr(self, '__waitcond_' + fn.__name__) if wc is None: self.waitvar = args else: self.waitvar = wc(*args) - return waitfn + + return waitfn # return the return value of the decorated function, like rx_raw, tx_raw def _check_header(hdr): + """ Checks for known errors in the Level 1 Outer packet header (18 bytes), raises errors. + + Packet length between 18 and 91 bytes + :param hdr: bytearray part of the pkt + :return: byte: packet length + """ + if len(hdr) < OUTER_HLEN: raise ValueError() @@ -74,6 +120,14 @@ def _check_header(hdr): def ba2bytes(addr): + """Transform a bluetooth address in bytearray of length 6 to a string representation, like '00:80:25:2C:11:B2' + + This revereses the order of the bytes and formats as a string with the : delimiter + + :param addr: part of the pkt bytearray + :return: string like like '00:80:25:2C:11:B2' + """ + if len(addr) != 6: raise ValueError("Bad length for bluetooth address") assert len(addr) == 6 @@ -81,6 +135,13 @@ def ba2bytes(addr): def bytes2ba(s): + """Transform a Bluetooth address in string representation to a bytearray of length 6 + + This reverses the order of the string and convert to bytearray + + :param s string like like '00:80:25:2C:11:B2' + :return: bytearray length 6, addr + """ addr = [int(x, 16) for x in s.split(':')] addr.reverse() if len(addr) != 6: @@ -96,14 +157,19 @@ def int2bytes32(v): return bytearray([v & 0xff, (v >> 8) & 0xff, (v >> 16) & 0xff, v >> 24]) -def bytes2int(b): +def bytes2int(b) -> int: + """Convert arbitrary length bytes or bytearray in little-endian to integer + + :param b: bytes, memoryview, or bytearray; converted to bytearray which is mutable + :return: integer + """ v = 0 - while b: + ba = bytearray(b) + while ba: v = v << 8 - v += b.pop() + v += ba.pop() return v - crc16_table = [0x0000, 0x1189, 0x2312, 0x329b, 0x4624, 0x57ad, 0x6536, 0x74bf, 0x8c48, 0x9dc1, 0xaf5a, 0xbed3, 0xca6c, 0xdbe5, 0xe97e, 0xf8f7, 0x1081, 0x0108, 0x3393, 0x221a, 0x56a5, 0x472c, 0x75b7, 0x643e, @@ -136,7 +202,7 @@ def bytes2int(b): 0x6b46, 0x7acf, 0x4854, 0x59dd, 0x2d62, 0x3ceb, 0x0e70, 0x1ff9, 0xf78f, 0xe606, 0xd49d, 0xc514, 0xb1ab, 0xa022, 0x92b9, 0x8330, 0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78] -assert(len(crc16_table) == 256) +assert (len(crc16_table) == 256) def crc16(iv, data): @@ -147,43 +213,76 @@ def crc16(iv, data): class Connection(base.InverterConnection): + """Connection via IP socket connection to inverter, with all functions needed to receive data + + Args: + addr (str): Bluetooth address in hex, like '00:80:25:2C:11:B2' + + Attributes: + """ MAXBUFFER = 512 BROADCAST = "ff:ff:ff:ff:ff:ff" BROADCAST2 = bytearray(b'\xff\xff\xff\xff\xff\xff') def __init__(self, addr): - self.sock = socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, - socket.BTPROTO_RFCOMM) + """ initialise the python IP socket as a Bluetooth socket""" + # Original Linux connection + # self.sock = socket.socket(socket.AF_BLUETOOTH, socket.SOCK_STREAM, + # socket.BTPROTO_RFCOMM) + # self.sock.connect((addr, 1)) + + # Windows connection + self.log = logging.getLogger(__name__) + self.sock = BluetoothSocket(RFCOMM) self.sock.connect((addr, 1)) self.remote_addr = addr - self.local_addr = self.sock.getsockname()[0] - - self.local_addr2 = bytearray(b'\x78\x00\x3f\x10\xfb\x39') - + self.local_addr = self.sock.getsockname()[0] # from pi, 'B8:27:EB:F4:80:EB', PC CC:AF:78:E9:07:62 + + # todo what is this hardcoded for? not from the local BT or MAC address + # James Ball: 6-byte address. It seems to be one byte value, one byte 0 then serial number (not MAC address) of device. + # In the example the serial number is 2001787857 which translates to 0xd1db5077. + # Seems to work with any value - this default from dgibson: 3F10FB39 1058077497 + # self.local_addr2 = bytearray(b'\x78\x00\x3f\x10\xfb\x39') + # self.local_addr2 = bytearray(b'\xB8\x27\xEB\xF4\x80\xEB') # B8:27:EB:F4:80:EB Pi local address + self.local_addr2 = bytearray(b'\xCC\xAF\x78\xE9\x07\x62') # CC:AF:78:E9:07:62 T520 local address + # print('self.local_addr2', binascii.hexlify(self.local_addr2)) # as bytearray(b'x\x00?\x10\xfb9') + self.log.debug('self.local_addr2: %s', (binascii.hexlify(self.local_addr2)).decode()) self.rxbuf = bytearray() self.pppbuf = dict() self.tagcounter = 0 def gettag(self): + """Generates an incrementing tag used in PPP packets to keep them unique for this session""" self.tagcounter += 1 return self.tagcounter # # RX side + # Function call order + # wait() > rx() > rx-raw() > rx_outer() > rx_ppp_raw() > rx_ppp() >rx_6560 > rxfilter_6560 # def rx(self): + """Receive raw data from socket, pass up the chain to rx_raw, etc + + Called by the wait() function + Receive raw data from socket, to the limit of available space in rxbuf + :return: + """ space = self.MAXBUFFER - len(self.rxbuf) self.rxbuf += self.sock.recv(space) while len(self.rxbuf) >= OUTER_HLEN: + # get the pktlen, while checking this is the expected packet pktlen = _check_header(self.rxbuf[:OUTER_HLEN]) + # the receive buffer should be at least as long as packet if len(self.rxbuf) < pktlen: return + # get the packet, and clear the buffer, pkt is bytearray, e.g. 31 bytes for hello pkt = self.rxbuf[:pktlen] del self.rxbuf[:pktlen] @@ -191,14 +290,20 @@ def rx(self): @waiter def rx_raw(self, pkt): - from_ = ba2bytes(pkt[4:10]) - to_ = ba2bytes(pkt[10:16]) + # SMA Level 1 Packet header 0 to 18 bytes + from_ = ba2bytes(pkt[4:10]) # Level 1 "From" bluetooth address + to_ = ba2bytes(pkt[10:16]) # Level 1 "To" bluetooth address type_ = bytes2int(pkt[16:18]) payload = pkt[OUTER_HLEN:] self.rx_outer(from_, to_, type_, payload) def rxfilter_outer(self, to_): + """Validate that we are intended recipient of packet (return True) based on Level 1 To address + + :param to_: bytearray of To BT address + :return: True + """ return ((to_ == self.local_addr) or (to_ == self.BROADCAST) or (to_ == "00:00:00:00:00:00")) @@ -206,12 +311,18 @@ def rxfilter_outer(self, to_): @waiter def rx_outer(self, from_, to_, type_, payload): if not self.rxfilter_outer(to_): - return + return # discard packet if (type_ == OTYPE_PPP) or (type_ == OTYPE_PPP2): self.rx_ppp_raw(from_, payload) def rx_ppp_raw(self, from_, payload): + """Validate the PPP or PPP2 packet, raise errors, strip protocol from header + + :param from_: Level 1 "From" bluetooth address + :param payload: raw PPP or PPP2 packet + :return: payload as frame[4:-2] + """ if from_ not in self.pppbuf: self.pppbuf[from_] = bytearray() pppbuf = self.pppbuf[from_] @@ -221,8 +332,8 @@ def rx_ppp_raw(self, from_, payload): if term < 0: return - raw = pppbuf[:term+1] - del pppbuf[:term+1] + raw = pppbuf[:term + 1] + del pppbuf[:term + 1] assert raw[-1] == 0x7e if raw[0] != 0x7e: @@ -251,6 +362,12 @@ def rx_ppp_raw(self, from_, payload): @waiter def rx_ppp(self, from_, protocol, payload): + """Using SMA Level 2 packet, slice into meaningful SMA data elements + todo make this memoroyview now + :param from_: Level 1 "From" bluetooth address (not used beyond here) + :param protocol: expects 0x6560 + :param payload: validated PPP packet in SMA protocol + """ if protocol == SMA_PROTOCOL_ID: innerlen = payload[0] if len(payload) != (innerlen * 4): @@ -281,6 +398,8 @@ def rx_ppp(self, from_, protocol, payload): response, error, pktcount, first) def rxfilter_6560(self, to2): + """Validate that we are intended recipient of packet (return True) based on Level 2 To address + """ return ((to2 == self.local_addr2) or (to2 == self.BROADCAST2)) @@ -295,15 +414,29 @@ def rx_6560(self, from2, to2, a2, b1, b2, c1, c2, tag, # # Tx side + # functions call in this order: tx_historic > tx_6560 > tx_ppp > tx_outer > tx_raw # def tx_raw(self, pkt): + """Transmits a raw packet via Bluetooth socket interface + + :param pkt: bytearray PPP packet + :return: + """ if _check_header(pkt) != len(pkt): raise ValueError("Bad packet") self.sock.send(bytes(pkt)) def tx_outer(self, from_, to_, type_, payload): - pktlen = len(payload) + OUTER_HLEN - pkt = bytearray([0x7e, pktlen, 0x00, pktlen ^ 0x7e]) + """Builds a SMA Level 1 packet from supplied Level 2, calls tx_raw to transmit + + :param from_: str source Bluetooth address in string representation + :param to_: str destination Bluetooth address in string representation + :param type_: int the command to send, e.g. OTYPE_PPP = 0x01 L2 Packet start + :param payload: bytearray Data or payload in Level 1 packet + :return: + """ + pktlen = len(payload) + OUTER_HLEN # SMA Level 2 + SMA Level 1 + pkt = bytearray([0x7e, pktlen, 0x00, pktlen ^ 0x7e]) # start, length, 0x00, check byte pkt += bytes2ba(from_) pkt += bytes2ba(to_) pkt += int2bytes16(type_) @@ -312,14 +445,31 @@ def tx_outer(self, from_, to_, type_, payload): self.tx_raw(pkt) + # PPP frame is built + # (Point-to-point protocol in data link layer 2) + # called from tx_6560 and builds the fr def tx_ppp(self, to_, protocol, payload): + """Builds a SMA Level 2 packet from payload, calls tx_outer to wrap in Level 1 packet + + Builds a SMA Level 2 packet from payload + Adds CRC check 2 bytes + Adds header and footer + Escapes any reserved characters that may be in the payload + Calls tx_outer to wrap in Level 1 packet + + :param to_: str Bluetooth address in string representation + :param protocol: SMA_PROTOCOL_ID = 0x6560 + :param payload: + :return: + """ + # Build the Level 2 frame: Header 4 bytes; payload; frame = bytearray(b'\xff\x03') frame += int2bytes16(protocol) frame += payload frame += int2bytes16(crc16(0xffff, frame)) rawpayload = bytearray() - rawpayload.append(0x7e) + rawpayload.append(0x7e) # Head byte for b in frame: # Escape \x7e (FLAG), 0x7d (ESCAPE), 0x11 (XON) and 0x13 (XOFF) if b in [0x7e, 0x7d, 0x11, 0x13]: @@ -327,13 +477,44 @@ def tx_ppp(self, to_, protocol, payload): rawpayload.append(b ^ 0x20) else: rawpayload.append(b) - rawpayload.append(0x7e) + rawpayload.append(0x7e) # Foot byte self.tx_outer(self.local_addr, to_, OTYPE_PPP, rawpayload) def tx_6560(self, from2, to2, a2, b1, b2, c1, c2, tag, type_, subtype, arg1, arg2, extra=bytearray(), response=False, error=0, pktcount=0, first=True): + """Builds a PPP frame for Transmission and calls tx_ppp to wrap for transmission + + All PPP frames observed are PPP protocol number 0x6560, which appears to + be an SMA allocated ID for their control protocol. + + Parameters - too many to list + :param from2: + :param to2: + :param a2: + :param b1: + :param b2: + :param c1: + :param c2: + :param tag: + :param type_: byte: Command group 3; always 0x02 + :param subtype: 2 byte: Commmand 0x0070 Request 5 min data. + :param arg1: int fromtime + :param arg2: int totime + :param extra: + :param response: + :param error: + :param pktcount: + :param first: + :return: + + :return: tag: integer unique to each PPP packet. + """ + #print('tx_6560: from2 =', binascii.hexlify(from2)) + # Build the Level 2 frame: + # From byte 6 Packet length, to + # to byte if len(extra) % 4 != 0: raise Error("Inner protocol payloads must" + " have multiple of 4 bytes length") @@ -349,6 +530,7 @@ def tx_6560(self, from2, to2, a2, b1, b2, c1, c2, tag, payload.append(c2) payload.extend(int2bytes16(error)) payload.extend(int2bytes16(pktcount)) + # first packet 0x80, subsequent are 0x00 if first: xtag = tag | 0x8000 else: @@ -369,6 +551,7 @@ def tx_6560(self, from2, to2, a2, b1, b2, c1, c2, tag, self.tx_ppp("ff:ff:ff:ff:ff:ff", SMA_PROTOCOL_ID, payload) return tag + # AF 0000 is hardcoded default user password for SMA inverter, as bytes def tx_logon(self, password=b'0000', timeout=900): if len(password) > 12: raise ValueError @@ -382,6 +565,11 @@ def tx_logon(self, password=b'0000', timeout=900): 0x040c, 0xfffd, 7, timeout, extra) def tx_gdy(self): + """ EnergyProduction: + like SBFSpot arg2 same, arg 1 different? +# // SPOT_ETODAY, SPOT_ETOTAL + :return: + """ return self.tx_6560(self.local_addr2, self.BROADCAST2, 0xa0, 0x00, 0x00, 0x00, 0x00, self.gettag(), 0x200, 0x5400, 0x00262200, 0x002622ff) @@ -391,6 +579,24 @@ def tx_yield(self): 0xa0, 0x00, 0x00, 0x00, 0x00, self.gettag(), 0x200, 0x5400, 0x00260100, 0x002601ff) + # data_type = sma_data_unit.get(uom, ['Unknown', '?', '?', 1])[0] + + def tx_level2_request(self, type, subtype, arg1, arg2, extra): + return self.tx_6560(self.local_addr2, self.BROADCAST2, + 0xa0, 0x00, 0x00, 0x00, 0x00, self.gettag(), + type, subtype, arg1, arg2) + + def tx_spotacvoltage(self): + return self.tx_6560(self.local_addr2, self.BROADCAST2, + 0xa0, 0x00, 0x00, 0x00, 0x00, self.gettag(), + 0x200, 0x5100, 0x00464800, + 0x004655FF) # SpotACVoltage SPOT_UAC1, SPOT_UAC2, SPOT_UAC3, SPOT_IAC1, SPOT_IAC2, SPOT_IAC3 + # 0x200, 0x5100, 0x00464800, 0x004657FF) #SpotACVoltage above and SpotGridFrequency SPOT_FREQ, can link closely related ones + + # 0x200, 0x5100, 0x00464700, 0x004657FF) #SpotACVoltage above and SpotGridFrequency SPOT_FREQ, can link closely related ones + # 0x200, 0x5380, 0x00251E00, 0x00251EFF) #SpotDCPower SPOT_PDC1, SPOT_PDC2 2x28 bytes + # 0x200, 0x5380, 0x00451F00, 0x004521FF) #SpotDCVoltage SPOT_UDC1, SPOT_UDC2, SPOT_IDC1, SPOT_IDC2 4x28 bytes + def tx_set_time(self, ts, tzoffset): payload = bytearray() payload.extend(int2bytes32(0x00236d00)) @@ -406,16 +612,51 @@ def tx_set_time(self, ts, tzoffset): 0x20a, 0xf000, 0x00236d00, 0x00236d00, payload) def tx_historic(self, fromtime, totime): + """Builds a SMA request command 0x7000 for 5 min data and calls tx_6560 to wrap for transmission + + called by historic function to get historic fast sample data + Uses + Command Group 3 0x02 Request + Commmand 0x7000 Request 5 min data. + + :param fromtime: + :param totime: + :return: tag: int unique packet sequence id + """ return self.tx_6560(self.local_addr2, self.BROADCAST2, 0xe0, 0x00, 0x00, 0x00, 0x00, self.gettag(), 0x200, 0x7000, fromtime, totime) def tx_historic_daily(self, fromtime, totime): + """Builds a SMA request command 0x7000 for daily data and calls tx_6560 to wrap for transmission + + called by historic function to get historic daily data + Uses + Command Group 3 0x02 Request + Commmand 0x7020 Request Daily data. + :param fromtime: + :param totime: + :return: + """ return self.tx_6560(self.local_addr2, self.BROADCAST2, 0xe0, 0x00, 0x00, 0x00, 0x00, self.gettag(), 0x200, 0x7020, fromtime, totime) + # The tx_*() function sends some request to the inverter, then we wait for a response. + # The wait_*() functions are wrappers around wait(), which is the magic bit. wait() takes parameters saying what + # type of packet we're looking for at what protocol layer. It pokes those into some special variables + # then just calls rx() until another special variable is set. + + #@timing def wait(self, class_, cond=None): + """ wait() calls rx() repeatedly looking for a packet that matches the waitcond + Sets attribute on smadata2.inverter.smabluetooth.Connection like __waitcond_rx_outer + and then deletes the attribute once something is received. + + :param class_: + :param cond: + :return: + """ self.waitvar = None setattr(self, '__waitcond_rx_' + class_, cond) while self.waitvar is None: @@ -424,12 +665,27 @@ def wait(self, class_, cond=None): return self.waitvar def wait_outer(self, wtype, wpl=bytearray()): + """Calls the above wait, with class="outer", cond = the wfn function Connection.wait_outer..wfn + + :param wtype: Outer message types, defined above, like OTYPE_HELLO + :param wpl: + :return: the wait function defined above, + """ + def wfn(from_, to_, type_, payload): if ((type_ == wtype) and payload.startswith(wpl)): - return payload + return payload # payload a PPP packet + return self.wait('outer', wfn) def wait_6560(self, wtag): + """Called from all Level 2 requests to get SMA protocol data + +AF: changed to memoryview(extra)) from extra. Appears to reduce time from 0.1010 sec to 0.0740s + :param wtag: tag function + :return: list of bytearray types (from2, type_, subtype, arg1, arg2, extra) + """ + def tagfn(from2, to2, a2, b1, b2, c1, c2, tag, type_, subtype, arg1, arg2, extra, response, error, pktcount, first): @@ -438,10 +694,20 @@ def tagfn(from2, to2, a2, b1, b2, c1, c2, tag, raise Error("Unexpected multipacket reply") if error: raise Error("SMA device returned error 0x%x\n", error) - return (from2, type_, subtype, arg1, arg2, extra) + return (from2, type_, subtype, arg1, arg2, memoryview(extra)) + return self.wait('6560', tagfn) + #@timing def wait_6560_multi(self, wtag): + """Calls the above wait, with class="6560", cond = the multiwait_6560 function + + Assembles multiple packets into + Called from sma_request to get any data element + from sma.historic to get multiple 5 min samples + :param wtag: + :return: list of bytearray types (from2, type_, subtype, arg1, arg2, extra) + """ tmplist = [] def multiwait_6560(from2, to2, a2, b1, b2, c1, c2, tag, @@ -467,16 +733,24 @@ def multiwait_6560(from2, to2, a2, b1, b2, c1, c2, tag, return True self.wait('6560', multiwait_6560) - assert(len(tmplist) == (tmplist[0] + 1)) + assert (len(tmplist) == (tmplist[0] + 1)) return tmplist[1:] # Operations def hello(self): + """Sends hello packet response to the SMA device. + + The packet is based on the "hello" received, and this varies with the NetID + NetID is 5th byte, + # if hellopkt != bytearray(b'\x00\x04\x70\x00\x01\x00\x00\x00' + + # b'\x00\x01\x00\x00\x00'): + """ hellopkt = self.wait_outer(OTYPE_HELLO) - if hellopkt != bytearray(b'\x00\x04\x70\x00\x01\x00\x00\x00' + - b'\x00\x01\x00\x00\x00'): - raise Error("Unexpected HELLO %r" % hellopkt) + netID = hellopkt[4] #depends on inverter + + if hellopkt[0:4] != bytearray(b'\x00\x04\x70\x00'): + raise Error("smabluetooth: Unexpected HELLO %r" % hellopkt) self.tx_outer("00:00:00:00:00:00", self.remote_addr, OTYPE_HELLO, hellopkt) self.wait_outer(0x05) @@ -515,10 +789,233 @@ def daily_yield(self): daily = bytes2int(extra[8:12]) return timestamp, daily + def tx_level2_request(self, type, subtype, arg1, arg2, extra): + """Request data set from inverter + + Sends a data request in the form of a type, subtype and from & to ranges + Seems to represent a range of registers in the SMA device memory. + + :param type: SMA request type mostly 0x0200 + :param subtype:SMA request subtype often 0x5100 + :param arg1: pointer to range: from + :param arg2: pointer to range: to + :param extra: normally 0 + :return: + """ + return self.tx_6560(self.local_addr2, self.BROADCAST2, + 0xa0, 0x00, 0x00, 0x00, 0x00, self.gettag(), + type, subtype, arg1, arg2) + + #@timing + def sma_request(self, request_name): + """Generic request from device and pass to process_sma_record to parse output and write response. + + todo identify null values and exclude them + todo 3-phase or 1-phase in settings, then query/report accordingly. + :param request_name: string from sma_request_type in sma_devices.py") + :return: list of points + """ + # web_pdb.set_trace() #set a breakpoint + + sma_rq = sma_request_type.get(request_name) # test for not found + if not sma_rq: + raise Error("Connection.sma_request: Requested SMA data not recognised: ", request_name, " Check sma_request_type in sma_devices.py") + response_data_type = sma_rq[5] + # example: tag = self.tx_level2_request(0x200, 0x5100, 0x00464800, 0x004655FF, 0) + tag = self.tx_level2_request(sma_rq[0], sma_rq[1], sma_rq[2], sma_rq[3], sma_rq[4]) + + data = self.wait_6560_multi(tag) + # print("response_data_type is: ", response_data_type) + return self.process_sma_record(data, response_data_type) + + def process_sma_record(self, data, record_length): + """Parse output from device, look-up data elements (units etc), return response as set of raw data records. + + :param data: + :param record_length: say 16, 28, 40 bytes, used to slice data into records + :return: points, list of records as (element_name, timestamp, val1, unknown) + """ + points = [] + for from2, type_, subtype, arg1, arg2, extra in data: + self.log.debug("%sPPP frame; protocol 0x%04x [%d bytes] [%d record length]" + % (1, 0x6560, len(extra), record_length)) + self.log.debug(self.hexdump(extra, 'RX<', record_length/2)) + # todo decode these number groups. + # todo interpret the status codes + # todo deal with default nightime values, when inverter is inactive. send back as nulls? + + while extra: + index = bytes2int(extra[0:1]) # index of the item (phase, object, string) part of data type + element = bytes2int(extra[1:3]) # 2 byte units of measure, data type 0x821E, same as the FROM arg1 + record_type = bytes2int(extra[3:4]) # 1 byte SMA data type, same as element_type from the dict lookup + #uom seems to increase 1E 82, 1F 82, 20 82, etc 40by cycle + element_name, element_type, element_desc, data_type, units, _, divisor = sma_data_element.get(element) + if not element_name: + raise Error("Connection.sma_request: Requested SMA element not recognised: ", element, + " Check sma_data_element in sma_devices.py") + timestamp = bytes2int(extra[4:8]) + unknown = bytes2int(extra[24:28]) # padding, unused + # element_type 0x10 =text, 0x08 = status, 0x00, 0x40 = Dword 64 bit data + if ((element_type == 0x00) or (element_type == 0x40)): + #todo - this is just phase 1A, need to get 12:16, 16:20 also? + # for SpotACVoltage 0xFFFFFFFF is null; for SpotDCVoltage 0x80000000 is null + #todo SpotDCVoltage has 2 strings, each with values, but have same element type!! + if (bytes2int(extra[8:12]) == 0xFFFFFFFF) or (bytes2int(extra[8:12]) == 0x80000000): + val1 = None #really is None + logstr = ('{:x} {:25} {} {} {} {}'.format(element, element_name, format_time2(timestamp), + val1, units, element_desc)) + else: + val1 = bytes2int(extra[8:12]) + logstr = ('{:x} {:25} {} {:.1f} {} {}'.format(element, element_name, format_time2(timestamp), val1 / divisor, units, element_desc)) + # print("{0}: {1:.1f} {2}".format(format_time2(timestamp), val1 / divisor, units)) + elif element_type == 0x08: # status + # todo - this is just 1 status attribute, need to get 12:16, 16:20 also? using loop + # there are three bytes of attribute, 1 byte/char of attribute value. + val1 = bytes2int(extra[8:12]) + # unsigned long attribute = ((unsigned long)get_long(pcktBuf + ii + idx)) & 0x00FFFFFF; + # unsigned char attValue = pcktBuf[ii + idx + 3]; + # if (attribute == 0xFFFFFE) break; // End of attributes + # if (attValue == 1) + # devList[inv]->DeviceStatus = attribute; + logstr =('{:25} {} {:x} {:x} {}'.format(element_name, format_time2(timestamp), val1, unknown, element_desc)) + elif element_type == 0x10: # string in 8:14, other bytes unused + val1 = extra[8:14].decode(encoding="utf-8", errors="ignore") + logstr =('{:25} {} {} {:x} {}'.format(element_name, format_time2(timestamp), val1, unknown, element_desc)) + else: + val1 = 0 # error to raise - element not found + raise Error("Connection.sma_request: Requested SMA element_type not recognised: ", element_type, + " Check sma_data_element in sma_devices.py") + self.log.info(logstr) + # note val = 0x028F5C28, 4294967295 after hours, 11pm means NULL or? + extra = extra[record_length:] + #todo - 2 bytes, not 4? check for element not found? + if element != 0xffffffff: + #points.append((index, units, timestamp, val1, val2, val3, val4, unknown, data_type, divisor)) + #todo, apply divisor, send units (not for db)? + #print({element_name}, {format_time(timestamp)}, {val1:x}, {unknown:x}) + points.append((element_name, timestamp, val1, unknown)) + return points + + def hexdump(self, data, prefix, width): + '''Formatted hex display of the payload + + Format such that one record displays across 2 rows + Width was 16, change to data type width, e.g. 20, 28 + :param data: bytearray to be displayed + :param prefix: + :param width: record length, determines layout + :return: formatted string, to be printed + ''' + try: + s = '' + for i, b in enumerate(data): + if (i % width) == 0: + s += '%s%04x: ' % (prefix, i) + s += '%02X' % b + if (i % width) == (width-1): + s += '\n' + elif (i % width) == (width/2 -1): + s += '-' + else: + s += ' ' + if s and (s[-1] == '\n'): + s = s[:-1] + return s + except Exception as e: + self.log.error("Connection.hexdump: ERROR! %s" % e,) + raise e + + + def spotacvoltage(self): + # web_pdb.set_trace() #set a breakpoint + + # tag = self.tx_level2_request(0x200, 0x5100, 0x00464800, 0x004655FF, 0) + # sma_rq = sma_request_type.get('SpotACTotalPower') # test for not found + sma_rq = sma_request_type.get('SpotACVoltage') # test for not found + tag = self.tx_level2_request(sma_rq[0], sma_rq[1], sma_rq[2], sma_rq[3], sma_rq[4]) + # like sma_rq = (512, 21504, 2490624, 2499327, 0) + + points = [] + # for sma_rq in sma_request_type: + # print(sma_rq[0], sma_rq[1], sma_rq[2], sma_rq[3]) + # print(bytearray(sma_rq[0]), bytearray(sma_rq[1]), bytearray(sma_rq[2]), bytearray(sma_rq[3])) + # print(bytearray.fromhex(sma_rq[0]), bytearray.fromhex(sma_rq[2]), bytearray.fromhex(sma_rq[2]), bytearray.fromhex(sma_rq[3]) ) + # print(bytearray.fromhex(sma_rq[0]), bytearray.fromhex(sma_rq[2]), bytearray.fromhex[2], bytearray.fromhex[3] ) + # tag = self.tx_6560(self.local_addr2, self.BROADCAST2, + # 0xa0, 0x00, 0x00, 0x00, 0x00, self.gettag(), + # sma_rq[0], sma_rq[1], sma_rq[2], sma_rq[3]) + data = self.wait_6560_multi(tag) + + # data = [(bytearray(b'\x8a\x00\x1cx\xf8~'), 512, 20736, 10, 15, bytearray( + # b'\x01HF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01IF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01JF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01PF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01QF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01RF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00'))] + # points = [] + # self = < smadata2.inverter.smabluetooth.Connection + # object + # at + # 0xb61f6c50 > + # sma_rq = (512, 21504, 2490624, 2499327, 0) + # tag = 2 + + # web_pdb.set_trace() #set a breakpoint + for from2, type_, subtype, arg1, arg2, extra in data: + while extra: + # can use 0:4, but 4th byte sometimes set to 0 or 4, but same units. maybe DC/AC? + index = bytes2int(extra[0:1]) # index of the item (phase, object, string) part of data type + uom = bytes2int(extra[1:3]) # 2 byte units of measure, data type + # print("units of measure, data type: {0:x}".format(uom)) + data_type, units, _, divisor = sma_data_unit.get(uom) + timestamp = bytes2int(extra[4:8]) + # note val = 0x028F5C28, 4294967295 after hours, 11pm means NULL or? + val1 = bytes2int(extra[8:12]) + val2 = bytes2int(extra[12:16]) + val3 = bytes2int(extra[16:20]) + val4 = bytes2int(extra[20:24]) + unknown = bytes2int(extra[24:28]) # padding, unused + extra = extra[28:] + if uom != 0xffffffff: + points.append((index, units, timestamp, val1, val2, val3, val4, unknown, data_type, divisor)) + # points = [(1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot line voltage phase 1', 100)] + # points = [(1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot line voltage phase 1', 100), (1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot line voltage phase 2', 100)] + # extra = bytearray(b'\x01PF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01QF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01RF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00') + # extra = bytearray(b'\x01QF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00\x01RF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00') + # points = [(1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, + # 'AC spot line voltage phase 1', 100), ( + # 1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, + # 'AC spot line voltage phase 2', 100), ( + # 1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, + # 'AC spot line voltage phase 3', 100), ( + # 1, 'mA', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, + # 'AC spot current phase 1', 1)] + # extra = bytearray(b'\x01RF\x00\xbc\xa2\x9d]\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x00\x00') + # points = [(1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot line voltage phase 1', 100), (1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot line voltage phase 2', 100), (1, 'V', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot line voltage phase 3', 100), (1, 'mA', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot current phase 1', 1), (1, 'mA', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot current phase 2', 1), (1, 'mA', 1570611900, 4294967295, 4294967295, 4294967295, 4294967295, 1, 'AC spot current phase 3', 1)] + # + return points + def historic(self, fromtime, totime): - tag = self.tx_historic(fromtime, totime) + """ Obtain Historic data (5 minute intervals), called from download_inverter which specifies "historic" as the data_fn + + Typical values after a couple of iterations through "data" + extra = bytearray(b'D\x9aL\\\x81w&\x02\x00\x00\x00\x00p\x9bL\\\x81w&\x02\x00\x00\x00\x00\x9c\x9cL\\\x81w&\x02\x00\x00\x00\x00\xc8\x9dL\\\x81w&\x02\x00\x00\x00\x00\xf4\x9eL\\\x81w&\x02\x00\x00\x00\x00 \xa0L\\\x81w&\x02\x00\x00\x00\x00L\xa1L\\\x81w&\x02\x00\x00\x00\x00x\xa2L\\\x81w&\x02\x00\x00\x00\x00\xa4\xa3L\\\x81w&\x02\x00\x00\x00\x00\xd0\xa4L\\\x81w&\x02\x00\x00\x00\x00\xfc\xa5L\\\x81w&\x02\x00\x00\x00\x00(\xa7L\\\x81w&\x02\x00\x00\x00\x00T\xa8L\\\x81w&\x02\x00\x00\x00\x00\x80\xa9L\\\x81w&\x02\x00\x00\x00\x00\xac\xaaL\\\x81w&\x02\x00\x00\x00\x00\xd8\xabL\\\x81w&\x02\x00\x00\x00\x00\x04\xadL\\\x81w&\x02\x00\x00\x00\x000\xaeL\\\x81w&\x02\x00\x00\x00\x00\\\xafL\\\x81w&\x02\x00\x00\x00\x00\x88\xb0L\\\x81w&\x02\x00\x00\x00\x00\xb4\xb1L\\\x81w&\x02\x00\x00\x00\x00\xe0\xb2L\\\x81w&\x02\x00\x00\x00\x00\x0c\xb4L\\\x81w&\x02\x00\x00\x00\x008\xb5L\\\x81w&\x02\x00\x00\x00\x00d\xb6L\\\x81w&\x02\x00\x00\x00\x00\x90\xb7L\\\x81w&\x02\x00\x00\x00\x00\xbc\xb8L\\\x81w&\x02\x00\x00\x00\x00\xe8\xb9L\\\x81w&\x02\x00\x00\x00\x00\x14\xbbL\\\x81w&\x02\x00\x00\x00\x00@\xbcL\\\x81w&\x02\x00\x00\x00\x00l\xbdL\\\x81w&\x02\x00\x00\x00\x00\x98\xbeL\\\x84w&\x02\x00\x00\x00\x00\xc4\xbfL\\\x89w&\x02\x00\x00\x00\x00\xf0\xc0L\\\x91w&\x02\x00\x00\x00\x00\x1c\xc2L\\\x9cw&\x02\x00\x00\x00\x00H\xc3L\\\xa8w&\x02\x00\x00\x00\x00t\xc4L\\\xb4w&\x02\x00\x00\x00\x00\xa0\xc5L\\\xc6w&\x02\x00\x00\x00\x00') + from2 = bytearray(b'\x8a\x00\x1cx\xf8~') + fromtime = 1 + points = [(1548523800, 36075393), (1548524100, 36075393)] + self = + subtype = 28672 + tag = 2 + timestamp = 1548523800 + totime = 1550372370 + type_ = 512 + val = 36075393 + + :param fromtime: + :param totime: + :return: + """ + #todo - does this need memoryview for data? + tag = self.tx_historic(fromtime, totime) # defines the PPP frame data = self.wait_6560_multi(tag) points = [] + # extra in 12-byte cycle (4-byte timestamp, 4-byte value in Wh, 4-byte padding) for from2, type_, subtype, arg1, arg2, extra in data: while extra: timestamp = bytes2int(extra[0:4]) @@ -528,7 +1025,16 @@ def historic(self, fromtime, totime): points.append((timestamp, val)) return points + # def historic_daily(self, fromtime, totime): + """Get Historic data (daily intervals) + + Called from download_inverter in download.py (on schedule), or sma2mon command line. + + :param fromtime: + :param totime: + :return: point list of (timestamp, value) pairs + """ tag = self.tx_historic_daily(fromtime, totime) data = self.wait_6560_multi(tag) points = [] @@ -543,9 +1049,16 @@ def historic_daily(self, fromtime, totime): def set_time(self, newtime, tzoffset): self.tx_set_time(newtime, tzoffset) + # end of the Connection class def ptime(str): + """Convert a string date, like "2013-01-01" into a timestamp + + :param str: date like "2013-01-01" + :return: int: timestamp + """ + return int(time.mktime(time.strptime(str, "%Y-%m-%d"))) @@ -570,6 +1083,15 @@ def cmd_daily(sma, args): def cmd_historic(sma, args): + """ # Command: Historic data (5 minute intervals) + + called from download_inverter which specifies "historic" as the data_fn + + + :param sma: Connection class + :param args: command line args, including [start-date [end-date]] fromtime, totime + :return: + """ fromtime = ptime("2013-01-01") totime = int(time.time()) # Now if len(args) > 1: @@ -586,6 +1108,7 @@ def cmd_historic(sma, args): % (timestamp, format_time(timestamp), val)) +# appears unused. where is this called from? def cmd_historic_daily(sma, args): fromtime = ptime("2013-01-01") totime = int(time.time()) # Now @@ -603,12 +1126,28 @@ def cmd_historic_daily(sma, args): % (timestamp, format_time(timestamp), val)) +def get_devices(): + nearby_devices = bluetooth.discover_devices( + duration=8, lookup_names=True, flush_cache=True, lookup_class=False) + + print("found %d devices" % len(nearby_devices)) + + for addr, name in nearby_devices: + try: + print(" %s - %s" % (addr, name)) + except UnicodeEncodeError: + print(" %s - %s" % (addr, name.encode('utf-8', 'replace'))) + + +# code to allow running this file from command line? if __name__ == '__main__': + bdaddr = None optlist, args = getopt.getopt(sys.argv[1:], 'b:') if not args: + get_devices() print("Usage: %s -b command args.." % sys.argv[0]) sys.exit(1) diff --git a/smadata2/logging_config.py b/smadata2/logging_config.py new file mode 100644 index 0000000..ac8a828 --- /dev/null +++ b/smadata2/logging_config.py @@ -0,0 +1,54 @@ +#! /usr/bin/python3 +# +# smadata2.logging_config - source for application logging dictconfig() +# Copyright (C) 2019 Andy Frigaard +# + +SMA2_LOGGING_CONFIG = { + 'version': 1, + 'disable_existing_loggers': True, + 'formatters': { + 'standard': { + 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s' + }, + 'simple': { + 'format': '%(message)s' + }, + }, + 'handlers': { + 'default': { + 'level': 'DEBUG', + 'formatter': 'simple', + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout', # Default is stderr + }, + 'file': { + 'level': 'DEBUG', + 'formatter': 'standard', + 'class': 'logging.FileHandler', + 'filename': 'sma2.log', + }, + }, + 'loggers': { + '': { # root logger + 'handlers': ['default', 'file'], + 'level': 'INFO', + 'propagate': False + }, + 'inverter': { + 'handlers': ['default', 'file'], + 'level': 'WARNING', + 'propagate': False + }, + 'smadata2.sma2mon': { # monitoring + 'handlers': ['default', 'file'], + 'level': 'DEBUG', + 'propagate': False + }, + '__main__': { # if __name__ == '__main__' + 'handlers': ['default', 'file'], + 'level': 'DEBUG', + 'propagate': False + }, + } +} \ No newline at end of file diff --git a/smadata2/pvoutputorg.py b/smadata2/pvoutputorg.py index 50ee15c..5629ba5 100644 --- a/smadata2/pvoutputorg.py +++ b/smadata2/pvoutputorg.py @@ -78,7 +78,7 @@ def format_datetime(dt): class API(object): """Represents the pvoutput.org web API, for a particular system - API documentation can be found at http://pvoutput.org/help.html#api-spec""" + API documentation can be found at https://pvoutput.org/help/api_specification.html""" def __init__(self, baseurl, apikey, sid): if not baseurl: @@ -334,7 +334,7 @@ def days_ago_accepted_by_api(self): def main(): if len(sys.argv) == 3: - baseurl = "http://pvoutput.org" + baseurl = "https://pvoutput.org" apikey = sys.argv[1] sid = sys.argv[2] elif len(sys.argv) == 4: diff --git a/smadata2/sma2mon.py b/smadata2/sma2mon.py index ccd24ed..4f8aa11 100644 --- a/smadata2/sma2mon.py +++ b/smadata2/sma2mon.py @@ -18,11 +18,14 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import sys -import argparse +import argparse #see https://docs.python.org/3/howto/argparse.html import os.path import datetime import dateutil.parser import time + +import logging.config +log = logging.getLogger(__name__) # once in each module import csv import smadata2.config @@ -30,30 +33,38 @@ import smadata2.datetimeutil import smadata2.download import smadata2.upload +from smadata2.datetimeutil import format_time +import web_pdb def status(config, args): for system in config.systems(): - print("%s:" % system.name) + config.log.info("%s:" % system.name) for inv in system.inverters(): - print("\t%s:" % inv.name) + config.log.info("\t%s:" % inv.name) + #web_pdb.set_trace() try: sma = inv.connect_and_logon() - dtime, daily = sma.daily_yield() - print("\t\tDaily generation at %s:\t%d Wh" + config.log.info("\tDaily generation at %s:\t%d Wh" % (smadata2.datetimeutil.format_time(dtime), daily)) ttime, total = sma.total_yield() - print("\t\tTotal generation at %s:\t%d Wh" - % (smadata2.datetimeutil.format_time(ttime), total)) + config.log.info("\tTotal generation at %s:\t%d Wh" + % (smadata2.datetimeutil.format_time(ttime), total)) except Exception as e: - print("ERROR contacting inverter: %s" % e, file=sys.stderr) + config.log.error("sma2mon ERROR contacting inverter: %s" % e, file=sys.stderr) def yieldat(config, args): + """Get production at a given date + + :param config: Config from json file + :param args: command line arguments, including datetime + :return: prints val, the aggregate for the provided date + """ db = config.database() if args.datetime is None: @@ -88,34 +99,158 @@ def yieldat(config, args): print("\tTotal generation at %s: %d Wh" % (sdt, val)) +def historic_daily(config, args): + db = config.database() -def download(config, args): + if args.fromdate is None: + print("No date specified", file=sys.stderr) + sys.exit(1) + + fromdate = dateutil.parser.parse(args.fromdate) + todate = dateutil.parser.parse(args.todate) + fromtime = int(fromdate.timestamp()) + totime = int(todate.timestamp()) + + for system in config.systems(): + print("%s:" % system.name) + + for inv in system.inverters(): + print("\t%s:" % inv.name) + # web_pdb.set_trace() + + try: + sma = inv.connect_and_logon() + + # dtime, daily = sma.historic_daily() + # print("\t\tDaily generation at %s:\t%d Wh" + # % (smadata2.datetimeutil.format_time(dtime), daily)) + hlist = sma.historic_daily(fromtime, totime) + for timestamp, val in hlist: + print("[%d] %s: Total generation %d Wh" + % (timestamp, format_time(timestamp), val)) + # ttime, total = sma.total_yield() + # print("\t\tTotal generation at %s:\t%d Wh" + # % (smadata2.datetimeutil.format_time(ttime), total)) + except Exception as e: + print("ERROR contacting inverter: %s" % e, file=sys.stderr) + +def sma_request(config, args): + """Get spot data from the inverters + + :param config: configuration file + :param args: command line args, identify the type of data requested, like 'SpotACVoltage' + """ db = config.database() for system in config.systems(): + print("%s:" % system.name) + for inv in system.inverters(): - print("%s (SN: %s)" % (inv.name, inv.serial)) + print("\t%s:" % inv.name) + # web_pdb.set_trace() + + # try: + sma = inv.connect_and_logon() + hlist = sma.sma_request(args.request_name) + for index, uom, timestamp, val1, val2, val3, val4, unknown, data_type, divisor in hlist: + # print("%s: %f %f %f %s %s" % (format_time(timestamp), val1 / 100, val2 / 100, val3 / 100, unknown, data_type)) + print("{0} {1}: {2:10.3f} {3:10.3f} {4:10.3f} {6}".format(data_type, index, val1 / divisor, val2 / divisor, val3 / divisor, unknown, uom)) + # except Exception as e: + # print("ERROR contacting inverter: %s" % e, file=sys.stderr) + +def sma_info_request(config, args): + """Get other information from the inverters, like model, type, dates, status + + todo - does this write to a database, so structure into key-value pairs or similar + :param config: configuration file + :param args: command line args, identify the type of data requested, like 'model' + """ + db = config.database() + + for system in config.systems(): + print("%s:" % system.name) + + for inv in system.inverters(): + print("\t%s:" % inv.name) + # web_pdb.set_trace() + + # try: + sma = inv.connect_and_logon() + hlist = sma.sma_request(args.request_name) + print(hlist) + #for index, uom, timestamp, val1, val2, val3, val4, unknown, data_type, divisor in hlist: + # print("%s: %f %f %f %s %s" % (format_time(timestamp), val1 / 100, val2 / 100, val3 / 100, unknown, data_type)) + # print("{0} {1}: {2:10.3f} {3:10.3f} {4:10.3f} {6}".format(data_type, index, val1 / divisor, val2 / divisor, val3 / divisor, unknown, uom)) + # print("{0}: {1:.3f} {2:.3f} {3:.3f} {4:.3f} {5}".format(format_time(timestamp), val1 / 100, val2 / 100, val3 / 100, unknown, uom)) + # except Exception as e: + # print("ERROR contacting inverter: %s" % e, file=sys.stderr) + + +def spotacvoltage(config, args): + db = config.database() + + for system in config.systems(): + print("%s:" % system.name) + + for inv in system.inverters(): + print("\t%s:" % inv.name) + # web_pdb.set_trace() try: - data, daily = smadata2.download.download_inverter(inv, db) - if len(data): - print("Downloaded %d observations from %s to %s" - % (len(data), - smadata2.datetimeutil.format_time(data[0][0]), - smadata2.datetimeutil.format_time(data[-1][0]))) - else: - print("No new fast sampled data") - if len(daily): - print("Downloaded %d daily observations from %s to %s" - % (len(daily), - smadata2.datetimeutil.format_time(daily[0][0]), - smadata2.datetimeutil.format_time(daily[-1][0]))) - else: - print("No new daily data") + sma = inv.connect_and_logon() + + # dtime, daily = sma.historic_daily() + # print("\t\tDaily generation at %s:\t%d Wh" + # % (smadata2.datetimeutil.format_time(dtime), daily)) + hlist = sma.spotacvoltage() + # for val in hlist: + # print("[%d] : Raw value %d" % (val)) + for index, uom, timestamp, val1, val2, val3, val4, unknown, data_type, divisor in hlist: + # print("%s: %f %f %f %s %s" % (format_time(timestamp), val1 / 100, val2 / 100, val3 / 100, unknown, data_type)) + print("{0} {1}: {2:10.3f} {3:10.3f} {4:10.3f} {6}".format(data_type, index, val1 / divisor, val2 / divisor, val3 / divisor, unknown, uom)) + # print("{0}: {1:.3f} {2:.3f} {3:.3f} {4:.3f} {5}".format(format_time(timestamp), val1 / 100, val2 / 100, val3 / 100, unknown, uom)) + # for timestamp, val in hlist: + # print("[%d] %s: Spot AC voltage %d Wh" % (timestamp, format_time(timestamp), val)) + # ttime, total = sma.total_yield() + # print("\t\tTotal generation at %s:\t%d Wh" + # % (smadata2.datetimeutil.format_time(ttime), total)) except Exception as e: - print("ERROR downloading inverter: %s" % e, file=sys.stderr) + print("ERROR contacting inverter: %s" % e, file=sys.stderr) +def download(config, args): + """Download power history and record in database + + :param config: Config from json file + :param args: command line arguments, not used + :return: prints observations qty, from, to or error + """ + db = config.database() + for system in config.systems(): + for inv in system.inverters(): + print("%s (SN: %s)" % (inv.name, inv.serial)) + print("starttime: %s" % (inv.starttime)) + + #try: + data, daily = smadata2.download.download_inverter(inv, db) + if len(data): + print("Downloaded %d observations from %s to %s" + % (len(data), + smadata2.datetimeutil.format_time(data[0][0]), + smadata2.datetimeutil.format_time(data[-1][0]))) + else: + print("No new fast sampled data") + if len(daily): + print("Downloaded %d daily observations from %s to %s" + % (len(daily), + smadata2.datetimeutil.format_time(daily[0][0]), + smadata2.datetimeutil.format_time(daily[-1][0]))) + else: + print("No new daily data") + #except Exception as e: + # print("ERROR downloading inverter: %s" % e, file=sys.stderr) + +# AF updated by DGibson Sept 2019 def settime(config, args): for system in config.systems(): for inv in system.inverters(): @@ -223,8 +358,29 @@ def yieldlog(config, args): print(smadata2.datetimeutil.format_date(row[0]) + "\t" + "\t".join(str(y) for y in row[1:])) +#return smabluetooth.Connection(self.bdaddr) + +def scan(config, args): + try: + smadata2.inverter.smabluetooth.get_devices() + except: + print("Scan failed") + def argparser(): + """Creates argparse object for the application, imported lib + + - ArgumentParser -- The main entry point for command-line parsing. As the + example above shows, the add_argument() method is used to populate + the parser with actions for optional and positional arguments. Then + the parse_args() method is invoked to convert the args at the + command-line into an object with attributes. + + Extend this for new arguments with an entry below, a corresponding display/database function above, + and a corresponding function in smabluetooth that gets data from the inverter + + :return: parser: ArgumentParser object, used by main + """ parser = argparse.ArgumentParser(description="Work with Bluetooth" " enabled SMA photovoltaic inverters") @@ -257,6 +413,30 @@ def argparser(): parse_upload_date.set_defaults(func=upload) parse_upload_date.add_argument("--date", type=str, dest="upload_date") + help = "Get historic production for a date range" + parse_historic_daily = subparsers.add_parser("historic_daily", help=help) + parse_historic_daily.set_defaults(func=historic_daily) + parse_historic_daily.add_argument(type=str, dest="fromdate") + parse_historic_daily.add_argument(type=str, dest="todate") + + help = "Get spot AC voltage now." + parse_spotac = subparsers.add_parser("spotacvoltage", help=help) + parse_spotac.set_defaults(func=spotacvoltage) + + help = "Get spot reading by name (SpotACVoltage, ..) from sma_request_type ." + parse_sma_request = subparsers.add_parser("spot", help=help) + parse_sma_request.set_defaults(func=sma_request) + parse_sma_request.add_argument(type=str, dest="request_name") + + help = "Get device Info by name (TypeLabel, ..) from sma_request_type ." + parse_sma_info_request = subparsers.add_parser("info", help=help) + parse_sma_info_request.set_defaults(func=sma_info_request) + parse_sma_info_request.add_argument(type=str, dest="request_name") + + help = "Scan for bluetooth devices." + parse_scan = subparsers.add_parser("scan", help=help) + parse_scan.set_defaults(func=scan) + help = "Get daily production totals" parse_yieldlog = subparsers.add_parser("yieldlog", help=help) parse_yieldlog.set_defaults(func=yieldlog) @@ -267,15 +447,21 @@ def argparser(): return parser +def ptime(str): + return int(time.mktime(time.strptime(str, "%Y-%m-%d"))) def main(argv=sys.argv): + parser = argparser() - args = parser.parse_args(argv[1:]) + args = parser.parse_args(argv[1:]) #args is a Namespace for command line args + #log.debug("Startup with args: ", args) + # creates config object, using an optional file supplied on the command line config = smadata2.config.SMAData2Config(args.config) - + # calls args.func(config, args) if __name__ == '__main__': + #log = logging.getLogger(__name__) # once in each module main()