From 1578f4638e088b8338c8b087e453ccdec6e39492 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Mon, 9 Oct 2023 23:18:16 +0200 Subject: [PATCH 01/35] deactivate ansible-lint for now --- .github/workflows/publish_herd.yml | 4 ++-- .github/workflows/sphinx_to_pages.yml | 6 ++---- .pre-commit-config.yaml | 11 ++++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish_herd.yml b/.github/workflows/publish_herd.yml index 7d1ff8b4..e3a45fb3 100644 --- a/.github/workflows/publish_herd.yml +++ b/.github/workflows/publish_herd.yml @@ -22,8 +22,8 @@ jobs: deploy: runs-on: ubuntu-latest -# needs: -# - run-quality-assurance + needs: + - run-quality-assurance env: herd_path: "./software/shepherd-herd" diff --git a/.github/workflows/sphinx_to_pages.yml b/.github/workflows/sphinx_to_pages.yml index 67a8fd94..749cba1f 100644 --- a/.github/workflows/sphinx_to_pages.yml +++ b/.github/workflows/sphinx_to_pages.yml @@ -4,8 +4,6 @@ name: Generate Docs on: push: branches: [ "main" ] - pull_request: - branches: [ "main" ] workflow_call: jobs: @@ -13,8 +11,8 @@ jobs: uses: ./.github/workflows/qa_tests.yml build-pages: runs-on: ubuntu-latest -# needs: -# - run-quality-assurance + needs: + - run-quality-assurance environment: name: github-pages url: ${{ steps.deployment.outputs.page_url }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 927021e5..2c94ba4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -95,11 +95,12 @@ repos: ### Test-Improvements - flake8-assertive - - repo: https://github.com/ansible-community/ansible-lint.git - rev: v6.20.3 - hooks: - - id: ansible-lint - #files: \.(yaml|yml)$ +# TODO: disable, until bug is resolved, maybe connected to +# https://github.com/ansible/ansible-lint/issues/3636 +# - repo: https://github.com/ansible-community/ansible-lint.git +# rev: v6.20.3 +# hooks: +# - id: ansible-lint # - repo: https://github.com/rstcheck/rstcheck # rev: v6.1.2 From dd1f9ec179ba24e3b983904a64dcfa0e7f58b0de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ingmar=20In=CA=92o=20Splitt?= Date: Wed, 11 Oct 2023 15:18:30 +0200 Subject: [PATCH 02/35] doc --- deploy/roles/ptp_host/files/phc2sys@.service | 3 +++ deploy/roles/ptp_host/files/ptp4l@.service | 3 +++ 2 files changed, 6 insertions(+) diff --git a/deploy/roles/ptp_host/files/phc2sys@.service b/deploy/roles/ptp_host/files/phc2sys@.service index d64ebb69..05c63513 100644 --- a/deploy/roles/ptp_host/files/phc2sys@.service +++ b/deploy/roles/ptp_host/files/phc2sys@.service @@ -23,3 +23,6 @@ StartLimitBurst=10 [Install] WantedBy=multi-user.target + +# check with +# sudo systemctl status phc2sys@eth0.service \ No newline at end of file diff --git a/deploy/roles/ptp_host/files/ptp4l@.service b/deploy/roles/ptp_host/files/ptp4l@.service index 6a7ba7b2..5617a16e 100644 --- a/deploy/roles/ptp_host/files/ptp4l@.service +++ b/deploy/roles/ptp_host/files/ptp4l@.service @@ -16,3 +16,6 @@ StartLimitBurst=5 [Install] WantedBy=multi-user.target + +# check with +# sudo systemctl status ptp4l@eth0.service From fbc87d6ea89055ca80ba866cc4feba5c3e55a427 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 01:01:08 +0200 Subject: [PATCH 03/35] allow py312 --- software/python-package/setup.cfg | 1 + software/shepherd-calibration/setup.cfg | 1 + software/shepherd-herd/setup.cfg | 1 + 3 files changed, 3 insertions(+) diff --git a/software/python-package/setup.cfg b/software/python-package/setup.cfg index c0539b67..04b5edfd 100644 --- a/software/python-package/setup.cfg +++ b/software/python-package/setup.cfg @@ -25,6 +25,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: MIT License Operating System :: OS Independent Natural Language :: English diff --git a/software/shepherd-calibration/setup.cfg b/software/shepherd-calibration/setup.cfg index 0c63f2c3..9bb5a218 100644 --- a/software/shepherd-calibration/setup.cfg +++ b/software/shepherd-calibration/setup.cfg @@ -24,6 +24,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: MIT License Operating System :: OS Independent Natural Language :: English diff --git a/software/shepherd-herd/setup.cfg b/software/shepherd-herd/setup.cfg index b147a3fd..5f2a4f77 100644 --- a/software/shepherd-herd/setup.cfg +++ b/software/shepherd-herd/setup.cfg @@ -24,6 +24,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 License :: OSI Approved :: MIT License Operating System :: OS Independent Natural Language :: English From bb88b7f1e89303476382ee9dcc39d1caa96bf4e2 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 01:11:16 +0200 Subject: [PATCH 04/35] update deps --- .pre-commit-config.yaml | 10 +- Pipfile | 2 + Pipfile.lock | 569 +++++++++++++++++++++------------------- 3 files changed, 308 insertions(+), 273 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c94ba4b..0c330f3a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,7 +32,7 @@ repos: args: ["--disable", "MD013"] # ignore line length - repo: https://github.com/abravalheri/validate-pyproject - rev: v0.14 + rev: v0.15 hooks: - id: validate-pyproject # files: "./software/python-package/pyproject.toml" @@ -43,7 +43,7 @@ repos: - id: isort - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 23.10.0 hooks: - id: black @@ -95,6 +95,9 @@ repos: ### Test-Improvements - flake8-assertive + ### Load Config from ... + - flake8-pyproject + # TODO: disable, until bug is resolved, maybe connected to # https://github.com/ansible/ansible-lint/issues/3636 # - repo: https://github.com/ansible-community/ansible-lint.git @@ -113,6 +116,7 @@ repos: rev: v2.2.6 hooks: - id: codespell + additional_dependencies: ["tomli"] # for py<3.11 exclude: \.(sch|brd|lbr)$ # - repo: https://github.com/amperser/proselint @@ -122,7 +126,7 @@ repos: # types_or: ["markdown", "rst"] - repo: https://github.com/pre-commit/mirrors-clang-format - rev: v16.0.6 + rev: v17.0.3 hooks: - id: clang-format types_or: [c++, c] diff --git a/Pipfile b/Pipfile index 4ffea52d..1b4755c7 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,8 @@ build = "*" coverage = "*" pyright = "*" pandas-stubs = "*" # for pyright +ruff = "*" +types-PyYAML = "*" [packages] # NOTE: pip fails installing these packages as editable (-e), but pipenv needs it diff --git a/Pipfile.lock b/Pipfile.lock index eedd5e6d..1d09ba8b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "4c91c1ad208ec602c1855f825d153a55cc09c9c813be40c1b6e6e88e83008d5a" + "sha256": "e95087f49d11308feb30411094f00c1453085bd3c1ac469e91bce01282950244" }, "pipfile-spec": 6, "requires": {}, @@ -32,27 +32,27 @@ }, "ansible": { "hashes": [ - "sha256:d601d89a4306934e7c0aae05195fd72c0719287fde165982d0ebac282b4280f1", - "sha256:f33c492690592fad12684e9897f6de2da15c9f6e1ecb79137703a06470af2ce6" + "sha256:2749032e26b0dbc9a694528b85fd89e7f950b8c7b53606f17dd997f23ac7cc88", + "sha256:327c509bdaf5cdb2489d85c09d2c107e9432f9874c8bb5c0702a731160915f2d" ], "index": "pypi", - "version": "==8.4.0" + "version": "==8.5.0" }, "ansible-core": { "hashes": [ - "sha256:5c57089405406f3004e948127b518b65509e280d524f61f91cc6360303fc388b", - "sha256:c1a8aaede985f79e5932ba2163639379f7d8025bfd9b28378db1649a4ef541ed" + "sha256:3efa234de5fce79ec98853f3369535b27cacd7ce498495b996030cd15c373735", + "sha256:8cc539cb8d4349af3ffd901c70722f7a7a203ae6427ddac95ffdf546a6e41602" ], "markers": "python_version >= '3.9'", - "version": "==2.15.4" + "version": "==2.15.5" }, "astroid": { "hashes": [ - "sha256:1defdbca052635dd29657ea674edfc45e4b5be9cd53630c5b084fcfed94344a8", - "sha256:f2510e7fdcd6cfda4ec50014726d4857abf79acfc010084ce8c26091913f1b25" + "sha256:7d5895c9825e18079c5aeac0572bc2e4c83205c95d416e0b4fee8bc361d2d9ca", + "sha256:86b0bb7d7da0be1a7c4aedb7974e391b32d4ed89e33de6ed6902b4b15c97577e" ], "markers": "python_full_version >= '3.8.0'", - "version": "==3.0.0" + "version": "==3.0.1" }, "babel": { "hashes": [ @@ -91,31 +91,27 @@ }, "black": { "hashes": [ - "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f", - "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7", - "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100", - "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573", - "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d", - "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f", - "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9", - "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300", - "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948", - "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325", - "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9", - "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71", - "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186", - "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f", - "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe", - "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855", - "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80", - "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393", - "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c", - "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204", - "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377", - "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301" + "sha256:0e232f24a337fed7a82c1185ae46c56c4a6167fb0fe37411b43e876892c76699", + "sha256:30b78ac9b54cf87bcb9910ee3d499d2bc893afd52495066c49d9ee6b21eee06e", + "sha256:31946ec6f9c54ed7ba431c38bc81d758970dd734b96b8e8c2b17a367d7908171", + "sha256:31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd", + "sha256:47c4510f70ec2e8f9135ba490811c071419c115e46f143e4dce2ac45afdcf4c9", + "sha256:481167c60cd3e6b1cb8ef2aac0f76165843a374346aeeaa9d86765fe0dd0318b", + "sha256:6901631b937acbee93c75537e74f69463adaf34379a04eef32425b88aca88a23", + "sha256:76baba9281e5e5b230c9b7f83a96daf67a95e919c2dfc240d9e6295eab7b9204", + "sha256:7fb5fc36bb65160df21498d5a3dd330af8b6401be3f25af60c6ebfe23753f747", + "sha256:960c21555be135c4b37b7018d63d6248bdae8514e5c55b71e994ad37407f45b8", + "sha256:a3c2ddb35f71976a4cfeca558848c2f2f89abc86b06e8dd89b5a65c1e6c0f22a", + "sha256:c870bee76ad5f7a5ea7bd01dc646028d05568d33b0b09b7ecfc8ec0da3f3f39c", + "sha256:d3d9129ce05b0829730323bdcb00f928a448a124af5acf90aa94d9aba6969604", + "sha256:db451a3363b1e765c172c3fd86213a4ce63fb8524c938ebd82919bf2a6e28c6a", + "sha256:e223b731a0e025f8ef427dd79d8cd69c167da807f5710add30cdf131f13dd62e", + "sha256:f20ff03f3fdd2fd4460b4f631663813e57dc277e37fb216463f3b907aa5a9bdd", + "sha256:f74892b4b836e5162aa0452393112a574dac85e13902c57dfbaaf388e4eda37c", + "sha256:f8dc7d50d94063cdfd13c82368afd8588bac4ce360e4224ac399e769d6704e98" ], "markers": "python_version >= '3.8'", - "version": "==23.9.1" + "version": "==23.10.0" }, "certifi": { "hashes": [ @@ -472,27 +468,27 @@ }, "dearpygui": { "hashes": [ - "sha256:1a4ac7ef9a585dcfd2214629c0997a80efb45be598dbdea445129f6d623b2bc8", - "sha256:52fb640ac00f385f31a86666301228f7608f76b07eaa37f1bbf86809d6efae0b", - "sha256:59342047f413ed53c545b179aa191767ddd76ad03ba6c3828d041ef5d9cfe26f", - "sha256:59c5fae21fa916d7100cc0d1bcc62837b37815aea962fff3d2c9c5b725107c38", - "sha256:616907fc1a50d17587ba689e14490aa38b012df58decd2d747fc59acb3a1f4fe", - "sha256:7183ca9e3aa168d920b3cbb97afcf31b62eadefe5778a5365af0348e37e7f683", - "sha256:8525dff74c75ec3ae9969e2dc36e61984b3ea89b2f1245e5c95fd3b816f22741", - "sha256:89b0c973073f73e08275480fd6e5162bf30623910af6ffd405bdac6eb4943ec7", - "sha256:8d1028b663bd69efa3cf89e210f7b3eca02ac36a7e184b5534f5e5c776feb16d", - "sha256:8d274f59ec0652991a75dc4e5e383d168c8c3960cf57797ee6bca6576027ef94", - "sha256:8f676b5f81bb56352602ac3b88a72ad12c412aa90db3ade1243caf0efa317147", - "sha256:970f42ac7a54ad9cf90489a8a767770e4a7631d1a5844c493d24d02e43ca2249", - "sha256:ab6ee39a29f51a3c836a4c5562db5198ea6d161c9484136035324b97b0cad754", - "sha256:b697fc0d5c17b0612ed82bd65eda791d4871dda9d4d7c48a43d5093a7f04f3b8", - "sha256:e3c9184bca4ce1192d0ea78fa44581a5ebfae0ea2738570f83a0c1b93d7fe067", - "sha256:f580a25cbb067f548a377875a82171cceb5f803591775db7b56687945ecc5841", - "sha256:fcda36f7b86ce68d98eafc743d4ff43f3fb70d1b02ee146bf37fc66f753e0d7f", - "sha256:fd466db6aa6c9754cfb6bdffef74ba0e5aaa2b1c330dc9d0e4dc2197223fcdd6" + "sha256:0e30634f1f8c13994ec59c58048e97103e672f792145a57aab4a7c39c6445e5e", + "sha256:1711a5cc991eaa6bc8a7a3cadb3845236c16b6753dc46b2fd1f4285db57d998e", + "sha256:2a8a1305ad6bfbc6f46968067b7f02d8404f2df21a13627e8c93cdabbb30f37b", + "sha256:2d94b23bed91e121ad4556b811fe164598c5b4dc94a19e3195bb70a4ff632759", + "sha256:358a221d5f53b584e7bbc76e2ad7ca012991debdd74ea3a6cfc8e2ed0ea51c92", + "sha256:5350d2a03da789684fcd384d7ee9e0ea029ecf134bef30d134c439d35c9d60b4", + "sha256:5ccb75b5377e5d4fcd6358ac2164418301b2747a78fee55422e3995d167d364a", + "sha256:7dc32b229d89b10ec4f647414738783afc2d6aa09c28c9f7384b07023b6ee261", + "sha256:8047f4ac06fb774c1335a98d3dbb47337b07e17629a85789740298a61e6ae999", + "sha256:a2e722e5b1fb8ab8d25e3ed82a790778849c1c66a522a327a73e25e218f98873", + "sha256:ac4e1cb01cd53490df20365645fbd0296bac093b8a415c805c18cad0aa85cac6", + "sha256:d1b877a147971e062d3c006027e880fd2b88dad9d198620f12b8407dddb1f795", + "sha256:d2cb73d22ec6c186c039c60c22efa5fc94f0562793bf20996d9eeae5ee601d55", + "sha256:d36b16afd0dd22674a59195e2ba37b787ea18bfb4ca96c460d6b7795f2c56980", + "sha256:e506507ee5344bb8d059d333c4dd77cd61bb4850f66fc4b5508ffa4f456f5a98", + "sha256:e87016bdd388d0e33080cd2033047f6c58df7808e1c3e14a1b0d736b2184e1ce", + "sha256:ed0d58e057636c4b79aee14d09b78042b122f42a6bef08d9901239de3361449d", + "sha256:ef1f1a8dbbd2dac9ea4fbf9d37e7812741e9a33de03ce385e398258750966a85" ], "index": "pypi", - "version": "==1.10.0" + "version": "==1.10.1" }, "decorator": { "hashes": [ @@ -751,30 +747,34 @@ }, "h5py": { "hashes": [ - "sha256:12aa556d540f11a2cae53ea7cfb94017353bd271fb3962e1296b342f6550d1b8", - "sha256:23e74b878bbe1653ab34ca49b83cac85529cd0b36b9d625516c5830cc5ca2eac", - "sha256:36408f8c62f50007d14e000f9f3acf77e103b9e932c114cbe52a3089e50ebf94", - "sha256:3f457089c5d524b7998e3649bc63240679b8fb0a3859ea53bbb06841f3d755f1", - "sha256:54f01202cdea754ab4227dd27014bdbd561a4bbe4b631424fd812f7c2ce9c6ac", - "sha256:551e358db05a874a0f827b22e95b30092f2303edc4b91bb62ad2f10e0236e1a0", - "sha256:64acceaf6aff92af091a4b83f6dee3cf8d3061f924a6bb3a33eb6c4658a8348b", - "sha256:6822a814b9d8b8363ff102f76ea8d026f0ca25850bb579d85376029ee3e73b93", - "sha256:78e44686334cbbf2dd21d9df15823bc38663f27a3061f6a032c68a3e30c47bf7", - "sha256:79bbca34696c6f9eeeb36a91776070c49a060b2879828e2c8fa6c58b8ed10dd1", - "sha256:804c7fb42a34c8ab3a3001901c977a5c24d2e9c586a0f3e7c0a389130b4276fc", - "sha256:8d9492391ff5c3c80ec30ae2fe82a3f0efd1e750833739c25b0d090e3be1b095", - "sha256:95f7a745efd0d56076999b52e8da5fad5d30823bac98b59c68ae75588d09991a", - "sha256:9da9e7e63376c32704e37ad4cea2dceae6964cee0d8515185b3ab9cbd6b947bc", - "sha256:a4e20897c88759cbcbd38fb45b507adc91af3e0f67722aa302d71f02dd44d286", - "sha256:a6284061f3214335e1eec883a6ee497dbe7a79f19e6a57fed2dd1f03acd5a8cb", - "sha256:d97409e17915798029e297a84124705c8080da901307ea58f29234e09b073ddc", - "sha256:dbf5225543ca35ce9f61c950b73899a82be7ba60d58340e76d0bd42bf659235a", - "sha256:e604db6521c1e367c6bd7fad239c847f53cc46646f2d2651372d05ae5e95f817", - "sha256:eb7bdd5e601dd1739698af383be03f3dad0465fe67184ebd5afca770f50df9d6", - "sha256:f68b41efd110ce9af1cbe6fa8af9f4dcbadace6db972d30828b911949e28fadd" + "sha256:012ab448590e3c4f5a8dd0f3533255bc57f80629bf7c5054cf4c87b30085063c", + "sha256:212bb997a91e6a895ce5e2f365ba764debeaef5d2dca5c6fb7098d66607adf99", + "sha256:2381e98af081b6df7f6db300cd88f88e740649d77736e4b53db522d8874bf2dc", + "sha256:2c8e4fda19eb769e9a678592e67eaec3a2f069f7570c82d2da909c077aa94339", + "sha256:3074ec45d3dc6e178c6f96834cf8108bf4a60ccb5ab044e16909580352010a97", + "sha256:3c97d03f87f215e7759a354460fb4b0d0f27001450b18b23e556e7856a0b21c3", + "sha256:43a61b2c2ad65b1fabc28802d133eed34debcc2c8b420cb213d3d4ef4d3e2229", + "sha256:492305a074327e8d2513011fa9fffeb54ecb28a04ca4c4227d7e1e9616d35641", + "sha256:5dfc65ac21fa2f630323c92453cadbe8d4f504726ec42f6a56cf80c2f90d6c52", + "sha256:667fe23ab33d5a8a6b77970b229e14ae3bb84e4ea3382cc08567a02e1499eedd", + "sha256:6c013d2e79c00f28ffd0cc24e68665ea03ae9069e167087b2adb5727d2736a52", + "sha256:781a24263c1270a62cd67be59f293e62b76acfcc207afa6384961762bb88ea03", + "sha256:86df4c2de68257b8539a18646ceccdcf2c1ce6b1768ada16c8dcfb489eafae20", + "sha256:90286b79abd085e4e65e07c1bd7ee65a0f15818ea107f44b175d2dfe1a4674b7", + "sha256:92273ce69ae4983dadb898fd4d3bea5eb90820df953b401282ee69ad648df684", + "sha256:93dd840bd675787fc0b016f7a05fc6efe37312a08849d9dd4053fd0377b1357f", + "sha256:9450464b458cca2c86252b624279115dcaa7260a40d3cb1594bf2b410a2bd1a3", + "sha256:ae2f0201c950059676455daf92700eeb57dcf5caaf71b9e1328e6e6593601770", + "sha256:aece0e2e1ed2aab076c41802e50a0c3e5ef8816d60ece39107d68717d4559824", + "sha256:b963fb772964fc1d1563c57e4e2e874022ce11f75ddc6df1a626f42bd49ab99f", + "sha256:ba9ab36be991119a3ff32d0c7cbe5faf9b8d2375b5278b2aea64effbeba66039", + "sha256:d4682b94fd36ab217352be438abd44c8f357c5449b8995e63886b431d260f3d3", + "sha256:d93adc48ceeb33347eb24a634fb787efc7ae4644e6ea4ba733d099605045c049", + "sha256:f42e6c30698b520f0295d70157c4e202a9e402406f50dc08f5a7bc416b24e52d", + "sha256:fd6f6d1384a9f491732cee233b99cd4bfd6e838a8815cc86722f9d2ee64032af" ], "markers": "python_version >= '3.8'", - "version": "==3.9.0" + "version": "==3.10.0" }, "identify": { "hashes": [ @@ -1251,41 +1251,41 @@ }, "numpy": { "hashes": [ - "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2", - "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292", - "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369", - "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91", - "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388", - "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299", - "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd", - "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3", - "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2", - "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69", - "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68", - "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148", - "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b", - "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a", - "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be", - "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8", - "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505", - "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c", - "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208", - "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8", - "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49", - "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95", - "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229", - "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896", - "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f", - "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c", - "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb", - "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99", - "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112", - "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581", - "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd", - "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf" + "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668", + "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9", + "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f", + "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5", + "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53", + "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2", + "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974", + "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f", + "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42", + "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2", + "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af", + "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67", + "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e", + "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c", + "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7", + "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e", + "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908", + "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66", + "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24", + "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b", + "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e", + "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe", + "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a", + "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575", + "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297", + "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104", + "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab", + "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3", + "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244", + "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124", + "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617", + "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c" ], "index": "pypi", - "version": "==1.26.0" + "version": "==1.26.1" }, "packaging": { "hashes": [ @@ -1354,63 +1354,63 @@ }, "pillow": { "hashes": [ - "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff", - "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f", - "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21", - "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635", - "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a", - "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f", - "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1", - "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d", - "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db", - "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849", - "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7", - "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876", - "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3", - "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317", - "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91", - "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d", - "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b", - "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd", - "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed", - "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500", - "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7", - "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a", - "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a", - "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0", - "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf", - "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f", - "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1", - "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088", - "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971", - "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a", - "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205", - "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54", - "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08", - "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21", - "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d", - "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08", - "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e", - "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf", - "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b", - "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145", - "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2", - "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d", - "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d", - "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf", - "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad", - "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d", - "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1", - "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4", - "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2", - "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19", - "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37", - "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4", - "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68", - "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1" + "sha256:00f438bb841382b15d7deb9a05cc946ee0f2c352653c7aa659e75e592f6fa17d", + "sha256:0248f86b3ea061e67817c47ecbe82c23f9dd5d5226200eb9090b3873d3ca32de", + "sha256:04f6f6149f266a100374ca3cc368b67fb27c4af9f1cc8cb6306d849dcdf12616", + "sha256:062a1610e3bc258bff2328ec43f34244fcec972ee0717200cb1425214fe5b839", + "sha256:0a026c188be3b443916179f5d04548092e253beb0c3e2ee0a4e2cdad72f66099", + "sha256:0f7c276c05a9767e877a0b4c5050c8bee6a6d960d7f0c11ebda6b99746068c2a", + "sha256:1a8413794b4ad9719346cd9306118450b7b00d9a15846451549314a58ac42219", + "sha256:1ab05f3db77e98f93964697c8efc49c7954b08dd61cff526b7f2531a22410106", + "sha256:1c3ac5423c8c1da5928aa12c6e258921956757d976405e9467c5f39d1d577a4b", + "sha256:1c41d960babf951e01a49c9746f92c5a7e0d939d1652d7ba30f6b3090f27e412", + "sha256:1fafabe50a6977ac70dfe829b2d5735fd54e190ab55259ec8aea4aaea412fa0b", + "sha256:1fb29c07478e6c06a46b867e43b0bcdb241b44cc52be9bc25ce5944eed4648e7", + "sha256:24fadc71218ad2b8ffe437b54876c9382b4a29e030a05a9879f615091f42ffc2", + "sha256:2cdc65a46e74514ce742c2013cd4a2d12e8553e3a2563c64879f7c7e4d28bce7", + "sha256:2ef6721c97894a7aa77723740a09547197533146fba8355e86d6d9a4a1056b14", + "sha256:3b834f4b16173e5b92ab6566f0473bfb09f939ba14b23b8da1f54fa63e4b623f", + "sha256:3d929a19f5469b3f4df33a3df2983db070ebb2088a1e145e18facbc28cae5b27", + "sha256:41f67248d92a5e0a2076d3517d8d4b1e41a97e2df10eb8f93106c89107f38b57", + "sha256:47e5bf85b80abc03be7455c95b6d6e4896a62f6541c1f2ce77a7d2bb832af262", + "sha256:4d0152565c6aa6ebbfb1e5d8624140a440f2b99bf7afaafbdbf6430426497f28", + "sha256:50d08cd0a2ecd2a8657bd3d82c71efd5a58edb04d9308185d66c3a5a5bed9610", + "sha256:61f1a9d247317fa08a308daaa8ee7b3f760ab1809ca2da14ecc88ae4257d6172", + "sha256:6932a7652464746fcb484f7fc3618e6503d2066d853f68a4bd97193a3996e273", + "sha256:7a7e3daa202beb61821c06d2517428e8e7c1aab08943e92ec9e5755c2fc9ba5e", + "sha256:7dbaa3c7de82ef37e7708521be41db5565004258ca76945ad74a8e998c30af8d", + "sha256:7df5608bc38bd37ef585ae9c38c9cd46d7c81498f086915b0f97255ea60c2818", + "sha256:806abdd8249ba3953c33742506fe414880bad78ac25cc9a9b1c6ae97bedd573f", + "sha256:883f216eac8712b83a63f41b76ddfb7b2afab1b74abbb413c5df6680f071a6b9", + "sha256:912e3812a1dbbc834da2b32299b124b5ddcb664ed354916fd1ed6f193f0e2d01", + "sha256:937bdc5a7f5343d1c97dc98149a0be7eb9704e937fe3dc7140e229ae4fc572a7", + "sha256:9882a7451c680c12f232a422730f986a1fcd808da0fd428f08b671237237d651", + "sha256:9a92109192b360634a4489c0c756364c0c3a2992906752165ecb50544c251312", + "sha256:9d7bc666bd8c5a4225e7ac71f2f9d12466ec555e89092728ea0f5c0c2422ea80", + "sha256:a5f63b5a68daedc54c7c3464508d8c12075e56dcfbd42f8c1bf40169061ae666", + "sha256:a646e48de237d860c36e0db37ecaecaa3619e6f3e9d5319e527ccbc8151df061", + "sha256:a89b8312d51715b510a4fe9fc13686283f376cfd5abca8cd1c65e4c76e21081b", + "sha256:a92386125e9ee90381c3369f57a2a50fa9e6aa8b1cf1d9c4b200d41a7dd8e992", + "sha256:ae88931f93214777c7a3aa0a8f92a683f83ecde27f65a45f95f22d289a69e593", + "sha256:afc8eef765d948543a4775f00b7b8c079b3321d6b675dde0d02afa2ee23000b4", + "sha256:b0eb01ca85b2361b09480784a7931fc648ed8b7836f01fb9241141b968feb1db", + "sha256:b1c25762197144e211efb5f4e8ad656f36c8d214d390585d1d21281f46d556ba", + "sha256:b4005fee46ed9be0b8fb42be0c20e79411533d1fd58edabebc0dd24626882cfd", + "sha256:b920e4d028f6442bea9a75b7491c063f0b9a3972520731ed26c83e254302eb1e", + "sha256:baada14941c83079bf84c037e2d8b7506ce201e92e3d2fa0d1303507a8538212", + "sha256:bb40c011447712d2e19cc261c82655f75f32cb724788df315ed992a4d65696bb", + "sha256:c0949b55eb607898e28eaccb525ab104b2d86542a85c74baf3a6dc24002edec2", + "sha256:c9aeea7b63edb7884b031a35305629a7593272b54f429a9869a4f63a1bf04c34", + "sha256:cfe96560c6ce2f4c07d6647af2d0f3c54cc33289894ebd88cfbb3bcd5391e256", + "sha256:d27b5997bdd2eb9fb199982bb7eb6164db0426904020dc38c10203187ae2ff2f", + "sha256:d921bc90b1defa55c9917ca6b6b71430e4286fc9e44c55ead78ca1a9f9eba5f2", + "sha256:e6bf8de6c36ed96c86ea3b6e1d5273c53f46ef518a062464cd7ef5dd2cf92e38", + "sha256:eaed6977fa73408b7b8a24e8b14e59e1668cfc0f4c40193ea7ced8e210adf996", + "sha256:fa1d323703cfdac2036af05191b969b910d8f115cf53093125e4058f62012c9a", + "sha256:fe1e26e1ffc38be097f0ba1d0d07fcade2bcfd1d023cda5b29935ae8052bd793" ], "markers": "python_version >= '3.8'", - "version": "==10.0.1" + "version": "==10.1.0" }, "pkginfo": { "hashes": [ @@ -1446,31 +1446,33 @@ }, "pre-commit": { "hashes": [ - "sha256:6bbd5129a64cad4c0dfaeeb12cd8f7ea7e15b77028d985341478c8af3c759522", - "sha256:96d529a951f8b677f730a7212442027e8ba53f9b04d217c4c67dc56c393ad945" + "sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32", + "sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660" ], "markers": "python_version >= '3.8'", - "version": "==3.4.0" + "version": "==3.5.0" }, "psutil": { "hashes": [ - "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d", - "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217", - "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4", - "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c", - "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f", - "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da", - "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4", - "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42", - "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5", - "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4", - "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9", - "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f", - "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30", - "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==5.9.5" + "sha256:10e8c17b4f898d64b121149afb136c53ea8b68c7531155147867b7b1ac9e7e28", + "sha256:18cd22c5db486f33998f37e2bb054cc62fd06646995285e02a51b1e08da97017", + "sha256:3ebf2158c16cc69db777e3c7decb3c0f43a7af94a60d72e87b2823aebac3d602", + "sha256:51dc3d54607c73148f63732c727856f5febec1c7c336f8f41fcbd6315cce76ac", + "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a", + "sha256:70cb3beb98bc3fd5ac9ac617a327af7e7f826373ee64c80efd4eb2856e5051e9", + "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4", + "sha256:91ecd2d9c00db9817a4b4192107cf6954addb5d9d67a969a4f436dbc9200f88c", + "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c", + "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c", + "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a", + "sha256:ca2780f5e038379e520281e4c032dddd086906ddff9ef0d1b9dcf00710e5071c", + "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57", + "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a", + "sha256:fb8a697f11b0f5994550555fcfe3e69799e5b060c8ecf9e2f75c69302cc35c0d", + "sha256:ff18b8d1a784b810df0b0fff3bcb50ab941c3b8e2c8de5726f9c71c601c611aa" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==5.9.6" }, "pwntools-elf-only": { "hashes": [ @@ -1481,11 +1483,11 @@ }, "pycodestyle": { "hashes": [ - "sha256:259bcc17857d8a8b3b4a2327324b79e5f020a13c16074670f9c8c8f872ea76d0", - "sha256:5d1013ba8dc7895b548be5afb05740ca82454fd899971563d2ef625d090326f8" + "sha256:41ba0e7afc9752dfb53ced5489e89f8186be00e599e712660695b7a75ff2663f", + "sha256:44fe31000b2d866f2e41841b18528a505fbd7fef9017b04eff4e2648a0fadc67" ], "markers": "python_version >= '3.8'", - "version": "==2.11.0" + "version": "==2.11.1" }, "pycparser": { "hashes": [ @@ -1623,11 +1625,11 @@ }, "pyfakefs": { "hashes": [ - "sha256:3e040f3792086086a0dc2191b05fe709438e168aafe2e94fcdbef8e3859208d8", - "sha256:8eb95f1dd1c4b8bdce30448fe169875e3a4451c32d3f9c37799157bd4eb7b789" + "sha256:33c1f891078c727beec465e75cb314120635e2298456493cc2cc0539e2130cbb", + "sha256:e3e35f65ce55ee8ecc5e243d55cfdbb5d0aa24938f6e04e19f0fab062f255020" ], "markers": "python_version >= '3.7'", - "version": "==5.2.4" + "version": "==5.3.0" }, "pyflakes": { "hashes": [ @@ -1679,11 +1681,11 @@ }, "pyright": { "hashes": [ - "sha256:2e9e0878298685b66485b340a0aaa16342129eb03ff9ed0e3c1ab66b8bfbe914", - "sha256:8e5b09cc5d1cfa0bcbf8824b0316d21c43fe229da7cef0a09cd12fcf6cb3eedd" + "sha256:4802bdc603f165ccfb84338ef850e90181abbb621028b09b81ec8aa5e97dfae2", + "sha256:a47f760c2f00aa9f593f78a7b22f98b22a9b9e1952d6a31dbed91842ca47b0b3" ], "markers": "python_version >= '3.7'", - "version": "==1.1.330.post0" + "version": "==1.1.332" }, "pyserial": { "hashes": [ @@ -2027,19 +2029,19 @@ "shepherd-core": { "extras": [], "hashes": [ - "sha256:282de4adddbeb91ae59e6df56c42a7a8393b70769bdefa3be2893d0d96eb95c7", - "sha256:82fa1d7ca94b99568a268787568fe908d851e6ecc19f1a69886aa80396a812a9" + "sha256:1cb27b408e56577e3625b6a4605bfef4f54a64c533b1df7aaead1677e4173e61", + "sha256:f6939ad4fdf18f29f15034ec0fa2231486d315b0c57e61ad26d2edd0504f26b4" ], - "markers": "python_version >= '3.7'", - "version": "==2023.9.9" + "markers": "python_version >= '3.8'", + "version": "==2023.10.1" }, "shepherd-data": { "hashes": [ - "sha256:a15fe66008302cdaabc0e416e829fd7c08963d87667afc50bfa6378be3e2bf59", - "sha256:da62948467d8bdabc95790f1ee86794cd267c86a43b4a02c18476a3407d5ace5" + "sha256:28567d89412a5beef79a5a16f48df5275f858a20229aeab697e2c1c28b8f4f0d", + "sha256:52a38ba7f1531d89bb10c77bc53632b6719f80228818731bd830323ff00774d3" ], - "markers": "python_version >= '3.7'", - "version": "==2023.9.9" + "markers": "python_version >= '3.8'", + "version": "==2023.10.1" }, "shepherd-herd": { "editable": true, @@ -2251,11 +2253,11 @@ }, "urllib3": { "hashes": [ - "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2", - "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564" + "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", + "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" ], "markers": "python_version >= '3.7'", - "version": "==2.0.6" + "version": "==2.0.7" }, "virtualenv": { "hashes": [ @@ -2464,39 +2466,35 @@ "develop": { "astroid": { "hashes": [ - "sha256:1defdbca052635dd29657ea674edfc45e4b5be9cd53630c5b084fcfed94344a8", - "sha256:f2510e7fdcd6cfda4ec50014726d4857abf79acfc010084ce8c26091913f1b25" + "sha256:7d5895c9825e18079c5aeac0572bc2e4c83205c95d416e0b4fee8bc361d2d9ca", + "sha256:86b0bb7d7da0be1a7c4aedb7974e391b32d4ed89e33de6ed6902b4b15c97577e" ], "markers": "python_full_version >= '3.8.0'", - "version": "==3.0.0" + "version": "==3.0.1" }, "black": { "hashes": [ - "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f", - "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7", - "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100", - "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573", - "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d", - "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f", - "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9", - "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300", - "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948", - "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325", - "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9", - "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71", - "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186", - "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f", - "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe", - "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855", - "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80", - "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393", - "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c", - "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204", - "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377", - "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301" + "sha256:0e232f24a337fed7a82c1185ae46c56c4a6167fb0fe37411b43e876892c76699", + "sha256:30b78ac9b54cf87bcb9910ee3d499d2bc893afd52495066c49d9ee6b21eee06e", + "sha256:31946ec6f9c54ed7ba431c38bc81d758970dd734b96b8e8c2b17a367d7908171", + "sha256:31b9f87b277a68d0e99d2905edae08807c007973eaa609da5f0c62def6b7c0bd", + "sha256:47c4510f70ec2e8f9135ba490811c071419c115e46f143e4dce2ac45afdcf4c9", + "sha256:481167c60cd3e6b1cb8ef2aac0f76165843a374346aeeaa9d86765fe0dd0318b", + "sha256:6901631b937acbee93c75537e74f69463adaf34379a04eef32425b88aca88a23", + "sha256:76baba9281e5e5b230c9b7f83a96daf67a95e919c2dfc240d9e6295eab7b9204", + "sha256:7fb5fc36bb65160df21498d5a3dd330af8b6401be3f25af60c6ebfe23753f747", + "sha256:960c21555be135c4b37b7018d63d6248bdae8514e5c55b71e994ad37407f45b8", + "sha256:a3c2ddb35f71976a4cfeca558848c2f2f89abc86b06e8dd89b5a65c1e6c0f22a", + "sha256:c870bee76ad5f7a5ea7bd01dc646028d05568d33b0b09b7ecfc8ec0da3f3f39c", + "sha256:d3d9129ce05b0829730323bdcb00f928a448a124af5acf90aa94d9aba6969604", + "sha256:db451a3363b1e765c172c3fd86213a4ce63fb8524c938ebd82919bf2a6e28c6a", + "sha256:e223b731a0e025f8ef427dd79d8cd69c167da807f5710add30cdf131f13dd62e", + "sha256:f20ff03f3fdd2fd4460b4f631663813e57dc277e37fb216463f3b907aa5a9bdd", + "sha256:f74892b4b836e5162aa0452393112a574dac85e13902c57dfbaaf388e4eda37c", + "sha256:f8dc7d50d94063cdfd13c82368afd8588bac4ce360e4224ac399e769d6704e98" ], "markers": "python_version >= '3.8'", - "version": "==23.9.1" + "version": "==23.10.0" }, "build": { "hashes": [ @@ -2937,41 +2935,41 @@ }, "numpy": { "hashes": [ - "sha256:020cdbee66ed46b671429c7265cf00d8ac91c046901c55684954c3958525dab2", - "sha256:0621f7daf973d34d18b4e4bafb210bbaf1ef5e0100b5fa750bd9cde84c7ac292", - "sha256:0792824ce2f7ea0c82ed2e4fecc29bb86bee0567a080dacaf2e0a01fe7654369", - "sha256:09aaee96c2cbdea95de76ecb8a586cb687d281c881f5f17bfc0fb7f5890f6b91", - "sha256:166b36197e9debc4e384e9c652ba60c0bacc216d0fc89e78f973a9760b503388", - "sha256:186ba67fad3c60dbe8a3abff3b67a91351100f2661c8e2a80364ae6279720299", - "sha256:306545e234503a24fe9ae95ebf84d25cba1fdc27db971aa2d9f1ab6bba19a9dd", - "sha256:436c8e9a4bdeeee84e3e59614d38c3dbd3235838a877af8c211cfcac8a80b8d3", - "sha256:4a873a8180479bc829313e8d9798d5234dfacfc2e8a7ac188418189bb8eafbd2", - "sha256:4acc65dd65da28060e206c8f27a573455ed724e6179941edb19f97e58161bb69", - "sha256:51be5f8c349fdd1a5568e72713a21f518e7d6707bcf8503b528b88d33b57dc68", - "sha256:546b7dd7e22f3c6861463bebb000646fa730e55df5ee4a0224408b5694cc6148", - "sha256:5671338034b820c8d58c81ad1dafc0ed5a00771a82fccc71d6438df00302094b", - "sha256:637c58b468a69869258b8ae26f4a4c6ff8abffd4a8334c830ffb63e0feefe99a", - "sha256:767254ad364991ccfc4d81b8152912e53e103ec192d1bb4ea6b1f5a7117040be", - "sha256:7d484292eaeb3e84a51432a94f53578689ffdea3f90e10c8b203a99be5af57d8", - "sha256:7f6bad22a791226d0a5c7c27a80a20e11cfe09ad5ef9084d4d3fc4a299cca505", - "sha256:86f737708b366c36b76e953c46ba5827d8c27b7a8c9d0f471810728e5a2fe57c", - "sha256:8c6adc33561bd1d46f81131d5352348350fc23df4d742bb246cdfca606ea1208", - "sha256:914b28d3215e0c721dc75db3ad6d62f51f630cb0c277e6b3bcb39519bed10bd8", - "sha256:b44e6a09afc12952a7d2a58ca0a2429ee0d49a4f89d83a0a11052da696440e49", - "sha256:bb0d9a1aaf5f1cb7967320e80690a1d7ff69f1d47ebc5a9bea013e3a21faec95", - "sha256:c0b45c8b65b79337dee5134d038346d30e109e9e2e9d43464a2970e5c0e93229", - "sha256:c2e698cb0c6dda9372ea98a0344245ee65bdc1c9dd939cceed6bb91256837896", - "sha256:c78a22e95182fb2e7874712433eaa610478a3caf86f28c621708d35fa4fd6e7f", - "sha256:e062aa24638bb5018b7841977c360d2f5917268d125c833a686b7cbabbec496c", - "sha256:e5e18e5b14a7560d8acf1c596688f4dfd19b4f2945b245a71e5af4ddb7422feb", - "sha256:eae430ecf5794cb7ae7fa3808740b015aa80747e5266153128ef055975a72b99", - "sha256:ee84ca3c58fe48b8ddafdeb1db87388dce2c3c3f701bf447b05e4cfcc3679112", - "sha256:f042f66d0b4ae6d48e70e28d487376204d3cbf43b84c03bac57e28dac6151581", - "sha256:f8db2f125746e44dce707dd44d4f4efeea8d7e2b43aace3f8d1f235cfa2733dd", - "sha256:f93fc78fe8bf15afe2b8d6b6499f1c73953169fad1e9a8dd086cdff3190e7fdf" + "sha256:06934e1a22c54636a059215d6da99e23286424f316fddd979f5071093b648668", + "sha256:1c59c046c31a43310ad0199d6299e59f57a289e22f0f36951ced1c9eac3665b9", + "sha256:1d1bd82d539607951cac963388534da3b7ea0e18b149a53cf883d8f699178c0f", + "sha256:1e11668d6f756ca5ef534b5be8653d16c5352cbb210a5c2a79ff288e937010d5", + "sha256:3649d566e2fc067597125428db15d60eb42a4e0897fc48d28cb75dc2e0454e53", + "sha256:59227c981d43425ca5e5c01094d59eb14e8772ce6975d4b2fc1e106a833d5ae2", + "sha256:6081aed64714a18c72b168a9276095ef9155dd7888b9e74b5987808f0dd0a974", + "sha256:6965888d65d2848e8768824ca8288db0a81263c1efccec881cb35a0d805fcd2f", + "sha256:76ff661a867d9272cd2a99eed002470f46dbe0943a5ffd140f49be84f68ffc42", + "sha256:78ca54b2f9daffa5f323f34cdf21e1d9779a54073f0018a3094ab907938331a2", + "sha256:82e871307a6331b5f09efda3c22e03c095d957f04bf6bc1804f30048d0e5e7af", + "sha256:8ab9163ca8aeb7fd32fe93866490654d2f7dda4e61bc6297bf72ce07fdc02f67", + "sha256:9696aa2e35cc41e398a6d42d147cf326f8f9d81befcb399bc1ed7ffea339b64e", + "sha256:97e5d6a9f0702c2863aaabf19f0d1b6c2628fbe476438ce0b5ce06e83085064c", + "sha256:9f42284ebf91bdf32fafac29d29d4c07e5e9d1af862ea73686581773ef9e73a7", + "sha256:a03fb25610ef560a6201ff06df4f8105292ba56e7cdd196ea350d123fc32e24e", + "sha256:a5b411040beead47a228bde3b2241100454a6abde9df139ed087bd73fc0a4908", + "sha256:af22f3d8e228d84d1c0c44c1fbdeb80f97a15a0abe4f080960393a00db733b66", + "sha256:afd5ced4e5a96dac6725daeb5242a35494243f2239244fad10a90ce58b071d24", + "sha256:b9d45d1dbb9de84894cc50efece5b09939752a2d75aab3a8b0cef6f3a35ecd6b", + "sha256:bb894accfd16b867d8643fc2ba6c8617c78ba2828051e9a69511644ce86ce83e", + "sha256:c8c6c72d4a9f831f328efb1312642a1cafafaa88981d9ab76368d50d07d93cbe", + "sha256:cd7837b2b734ca72959a1caf3309457a318c934abef7a43a14bb984e574bbb9a", + "sha256:cdd9ec98f0063d93baeb01aad472a1a0840dee302842a2746a7a8e92968f9575", + "sha256:d1cfc92db6af1fd37a7bb58e55c8383b4aa1ba23d012bdbba26b4bcca45ac297", + "sha256:d1d2c6b7dd618c41e202c59c1413ef9b2c8e8a15f5039e344af64195459e3104", + "sha256:d2984cb6caaf05294b8466966627e80bf6c7afd273279077679cb010acb0e5ab", + "sha256:d58e8c51a7cf43090d124d5073bc29ab2755822181fcad978b12e144e5e5a4b3", + "sha256:d78f269e0c4fd365fc2992c00353e4530d274ba68f15e968d8bc3c69ce5f5244", + "sha256:dcfaf015b79d1f9f9c9fd0731a907407dc3e45769262d657d754c3a028586124", + "sha256:e44ccb93f30c75dfc0c3aa3ce38f33486a75ec9abadabd4e59f114994a9c4617", + "sha256:e509cbc488c735b43b5ffea175235cec24bbc57b227ef1acc691725beb230d1c" ], "index": "pypi", - "version": "==1.26.0" + "version": "==1.26.1" }, "packaging": { "hashes": [ @@ -3015,11 +3013,11 @@ }, "pre-commit": { "hashes": [ - "sha256:6bbd5129a64cad4c0dfaeeb12cd8f7ea7e15b77028d985341478c8af3c759522", - "sha256:96d529a951f8b677f730a7212442027e8ba53f9b04d217c4c67dc56c393ad945" + "sha256:5804465c675b659b0862f07907f96295d490822a450c4c40e747d0b1c6ebcb32", + "sha256:841dc9aef25daba9a0238cd27984041fa0467b4199fc4852e27950664919f660" ], "markers": "python_version >= '3.8'", - "version": "==3.4.0" + "version": "==3.5.0" }, "pycparser": { "hashes": [ @@ -3054,11 +3052,11 @@ }, "pyright": { "hashes": [ - "sha256:2e9e0878298685b66485b340a0aaa16342129eb03ff9ed0e3c1ab66b8bfbe914", - "sha256:8e5b09cc5d1cfa0bcbf8824b0316d21c43fe229da7cef0a09cd12fcf6cb3eedd" + "sha256:4802bdc603f165ccfb84338ef850e90181abbb621028b09b81ec8aa5e97dfae2", + "sha256:a47f760c2f00aa9f593f78a7b22f98b22a9b9e1952d6a31dbed91842ca47b0b3" ], "markers": "python_version >= '3.7'", - "version": "==1.1.330.post0" + "version": "==1.1.332" }, "pyyaml": { "hashes": [ @@ -3156,6 +3154,29 @@ "markers": "python_full_version >= '3.7.0'", "version": "==13.6.0" }, + "ruff": { + "hashes": [ + "sha256:2a909d3930afdbc2e9fd893b0034479e90e7981791879aab50ce3d9f55205bd6", + "sha256:2d68367d1379a6b47e61bc9de144a47bcdb1aad7903bbf256e4c3d31f11a87ae", + "sha256:3305d1cb4eb8ff6d3e63a48d1659d20aab43b49fe987b3ca4900528342367145", + "sha256:3521bf910104bf781e6753282282acc145cbe3eff79a1ce6b920404cd756075a", + "sha256:3ff3006c97d9dc396b87fb46bb65818e614ad0181f059322df82bbfe6944e264", + "sha256:620d4b34302538dbd8bbbe8fdb8e8f98d72d29bd47e972e2b59ce6c1e8862257", + "sha256:6aa7e63c3852cf8fe62698aef31e563e97143a4b801b57f920012d0e07049a8d", + "sha256:8f5b24daddf35b6c207619301170cae5d2699955829cda77b6ce1e5fc69340df", + "sha256:b7cdc893aef23ccc14c54bd79a8109a82a2c527e11d030b62201d86f6c2b81c5", + "sha256:ba3208543ab91d3e4032db2652dcb6c22a25787b85b8dc3aeff084afdc612e5c", + "sha256:bc11955f6ce3398d2afe81ad7e49d0ebf0a581d8bcb27b8c300281737735e3a3", + "sha256:c34ae501d0ec71acf19ee5d4d889e379863dcc4b796bf8ce2934a9357dc31db7", + "sha256:c90461ae4abec261609e5ea436de4a4b5f2822921cf04c16d2cc9327182dbbcc", + "sha256:cbbd8eead88ea83a250499074e2a8e9d80975f0b324b1e2e679e4594da318c25", + "sha256:d3f9ac658ba29e07b95c80fa742b059a55aefffa8b1e078bc3c08768bdd4b11a", + "sha256:e140bd717c49164c8feb4f65c644046fe929c46f42493672853e3213d7bdbce2", + "sha256:f4780e2bb52f3863a565ec3f699319d3493b83ff95ebbb4993e59c62aaf6e75e" + ], + "index": "pypi", + "version": "==0.1.1" + }, "secretstorage": { "hashes": [ "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", @@ -3203,6 +3224,14 @@ ], "version": "==2023.3.1.1" }, + "types-pyyaml": { + "hashes": [ + "sha256:334373d392fde0fdf95af5c3f1661885fa10c52167b14593eb856289e1855062", + "sha256:c05bc6c158facb0676674b7f11fe3960db4f389718e19e62bd2b84d6205cfd24" + ], + "index": "pypi", + "version": "==6.0.12.12" + }, "typing-extensions": { "hashes": [ "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0", @@ -3213,11 +3242,11 @@ }, "urllib3": { "hashes": [ - "sha256:7a7c7003b000adf9e7ca2a377c9688bbc54ed41b985789ed576570342a375cd2", - "sha256:b19e1a85d206b56d7df1d5e683df4a7725252a964e3993648dd0fb5a1c157564" + "sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84", + "sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e" ], "markers": "python_version >= '3.7'", - "version": "==2.0.6" + "version": "==2.0.7" }, "virtualenv": { "hashes": [ From 888c1853ca033c90c664c2cc80825069f55e4040 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 01:11:24 +0200 Subject: [PATCH 05/35] Update phc2sys@.service --- deploy/roles/ptp_host/files/phc2sys@.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/roles/ptp_host/files/phc2sys@.service b/deploy/roles/ptp_host/files/phc2sys@.service index 05c63513..aa664a9a 100644 --- a/deploy/roles/ptp_host/files/phc2sys@.service +++ b/deploy/roles/ptp_host/files/phc2sys@.service @@ -25,4 +25,4 @@ StartLimitBurst=10 WantedBy=multi-user.target # check with -# sudo systemctl status phc2sys@eth0.service \ No newline at end of file +# sudo systemctl status phc2sys@eth0.service From cad84b204f4a4d7d9f781db9be648628d8baa2d1 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 01:11:50 +0200 Subject: [PATCH 06/35] forbid additional ssh cypher --- deploy/roles/secure_testbed/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/roles/secure_testbed/tasks/main.yml b/deploy/roles/secure_testbed/tasks/main.yml index 5781f12c..65df3686 100644 --- a/deploy/roles/secure_testbed/tasks/main.yml +++ b/deploy/roles/secure_testbed/tasks/main.yml @@ -31,10 +31,10 @@ - {regex: '^.*ChallengeResponseAuthentication.*$', line: 'ChallengeResponseAuthentication no'} - {regex: '^.*X11Forwarding.*$', line: 'X11Forwarding no'} - {regex: '^.*AllowUsers.*$', line: 'AllowUsers {{ ansible_user }}'} - # select good AND exclude weak algorithms: + # select good AND exclude weak algorithms: TODO: just forbid - {regex: '^.*KexAlgorithms.*$', line: 'KexAlgorithms -ecdh-sha2*,diffie-hellman-group-exchange*,diffie-hellman-group14-sha1'} - {regex: '^.*HostKeyAlgorithms.*$', line: 'HostKeyAlgorithms -ecda-sha2*,ecdsa-sha2*'} - - {regex: '^.*Ciphers.*$', line: 'Ciphers chacha20-poly1305@openssh.com,aes128-ctr,aes192-ctr,aes256-ctr'} + - {regex: '^.*Ciphers.*$', line: 'Ciphers -arcfour*,chacha20-poly1305@openssh.com,aes128-ctr,aes192-ctr,aes256-ctr'} - {regex: '^.*MACs.*$', line: 'MACs -umac-64*,hmac-sha1*,hmac-sha2-256,hmac-sha2-512,umac-128@open*'} # TODO: x11Forwarding is twice in file, one yes and one no From 8f4d033acd22569983d30e7d894f9f271f538874 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 01:17:02 +0200 Subject: [PATCH 07/35] clangformat 17 --- software/firmware/device-tree/am335x_pinctrl.h | 2 +- software/gps-overlay/am335x_pinctrl.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/software/firmware/device-tree/am335x_pinctrl.h b/software/firmware/device-tree/am335x_pinctrl.h index 508ee9ce..a6a69600 100644 --- a/software/firmware/device-tree/am335x_pinctrl.h +++ b/software/firmware/device-tree/am335x_pinctrl.h @@ -38,7 +38,7 @@ * Macros to allow using the absolute physical address instead of the * padconf registers instead of the offset from padconf base. */ -#define OMAP_IOPAD_OFFSET(pa, offset) (((pa) &0xffff) - (offset)) +#define OMAP_IOPAD_OFFSET(pa, offset) (((pa) & 0xffff) - (offset)) #define OMAP2420_CORE_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0030)(val) #define OMAP2430_CORE_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x2030)(val) diff --git a/software/gps-overlay/am335x_pinctrl.h b/software/gps-overlay/am335x_pinctrl.h index 508ee9ce..a6a69600 100644 --- a/software/gps-overlay/am335x_pinctrl.h +++ b/software/gps-overlay/am335x_pinctrl.h @@ -38,7 +38,7 @@ * Macros to allow using the absolute physical address instead of the * padconf registers instead of the offset from padconf base. */ -#define OMAP_IOPAD_OFFSET(pa, offset) (((pa) &0xffff) - (offset)) +#define OMAP_IOPAD_OFFSET(pa, offset) (((pa) & 0xffff) - (offset)) #define OMAP2420_CORE_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x0030)(val) #define OMAP2430_CORE_IOPAD(pa, val) OMAP_IOPAD_OFFSET((pa), 0x2030)(val) From 2eaf71cbc0f6fb81ed9c80d75883c8caadf1eebd Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 11:16:20 +0200 Subject: [PATCH 08/35] raise min version to py310 --- .github/workflows/qa_tests.yml | 4 +- .pre-commit-config.yaml | 11 ++++- docs/dev/contributing.rst | 2 +- software/python-package/pyproject.toml | 2 +- software/python-package/setup.cfg | 4 +- .../python-package/shepherd_sheep/__init__.py | 2 +- software/python-package/shepherd_sheep/cli.py | 6 +-- .../python-package/shepherd_sheep/eeprom.py | 2 +- .../shepherd_sheep/h5_writer.py | 16 ++++---- .../python-package/shepherd_sheep/logger.py | 2 +- .../shepherd_sheep/monitor_abc.py | 4 +- .../shepherd_sheep/monitor_kernel.py | 2 +- .../shepherd_sheep/monitor_ptp.py | 2 +- .../shepherd_sheep/monitor_sheep.py | 2 +- .../shepherd_sheep/monitor_sysutil.py | 2 +- .../shepherd_sheep/monitor_uart.py | 4 +- .../shepherd_sheep/shared_memory.py | 14 +++---- .../shepherd_sheep/shepherd_debug.py | 12 +++--- .../shepherd_sheep/shepherd_emulator.py | 4 +- .../shepherd_sheep/shepherd_io.py | 20 ++++----- .../shepherd_sheep/sysfs_interface.py | 10 ++--- .../shepherd_sheep/target_io.py | 8 ++-- software/shepherd-calibration/pyproject.toml | 2 +- software/shepherd-calibration/setup.cfg | 4 +- .../shepherd_cal/calibrator.py | 12 +++--- .../shepherd_cal/calibrator_cli.py | 16 ++++---- .../shepherd_cal/logger.py | 2 +- .../shepherd_cal/profile.py | 18 ++++---- .../shepherd_cal/profile_analyzer.py | 4 +- .../shepherd_cal/profile_calibration.py | 12 +++--- .../shepherd_cal/profiler.py | 8 ++-- .../shepherd_cal/profiler_cli.py | 8 ++-- software/shepherd-herd/pyproject.toml | 2 +- software/shepherd-herd/setup.cfg | 4 +- software/shepherd-herd/shepherd_herd/herd.py | 41 ++++++++++--------- .../shepherd-herd/shepherd_herd/herd_cli.py | 8 ++-- .../shepherd-herd/shepherd_herd/logger.py | 2 +- 37 files changed, 140 insertions(+), 138 deletions(-) diff --git a/.github/workflows/qa_tests.yml b/.github/workflows/qa_tests.yml index 1fbe60a7..76adffe3 100644 --- a/.github/workflows/qa_tests.yml +++ b/.github/workflows/qa_tests.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-22.04 # TODO: -latest was 20.04 and had old packages strategy: matrix: - python-version: ["3.11", "3.10", "3.9"] + python-version: ["3.12", "3.11", "3.10"] steps: - name: Checkout 🛎️ uses: actions/checkout@v4 @@ -46,7 +46,7 @@ jobs: # needs sudo because it installs packages - name: PyLint the shepherd py-package 🐏 run: "pylint $(git ls-files '*.py') || pylint-exit $?" - # -E --py-version 3.8 + # -E --py-version 3.10 working-directory: "software/python-package/" - name: Install the shepherd-herd py-package 𓋿 𓀍 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c330f3a..ead2f24d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -62,7 +62,7 @@ repos: rev: v3.15.0 hooks: - id: pyupgrade - args: ["--py38-plus", "--keep-runtime-typing"] + args: ["--py310-plus", "--keep-runtime-typing"] - repo: https://github.com/pycqa/flake8 rev: 6.1.0 @@ -98,6 +98,13 @@ repos: ### Load Config from ... - flake8-pyproject + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: 'v0.1.1' + hooks: + - id: ruff + args: ["--fix", "--exit-non-zero-on-fix"] + # manual run: ruff check . --preview + # TODO: disable, until bug is resolved, maybe connected to # https://github.com/ansible/ansible-lint/issues/3636 # - repo: https://github.com/ansible-community/ansible-lint.git @@ -180,4 +187,4 @@ repos: # args: ["--fix", "--exit-non-zero-on-fix"] # default_language_version: -# python: python3.8 +# python: python3.10 diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index b4f164e6..ed8b0008 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -8,7 +8,7 @@ Codestyle Please stick to the C and Python codestyle guidelines provided with the source code. -All **Python code** uses the feature-set of **version 3.8** is supposed to be formatted using `Black `_ in default mode and is tested with the `Flake8 `_ linter including some addons for cleaner and more secure code. +All included **Python code** uses the feature-set of **version 3.10** is supposed to be formatted using `Black `_ in default mode and is tested with the `Flake8 `_ linter including some addons for cleaner and more secure code. **C code** uses the feature-set of **C99** and shall be formatted based on *LLVM*-Style with some alterations to make it easier to read, similar to python code. We provide the corresponding ``clang-format`` config as ``.clang-format`` in the repository's root directory. diff --git a/software/python-package/pyproject.toml b/software/python-package/pyproject.toml index cd682b52..5eb25a46 100644 --- a/software/python-package/pyproject.toml +++ b/software/python-package/pyproject.toml @@ -5,6 +5,6 @@ build-backend = "setuptools.build_meta" [tool.pyright] root = "./" include = ['./shepherd', ] -pythonVersion = "3.8" +pythonVersion = "3.10" pythonPlatform = "Linux" reportMissingParameterType = true diff --git a/software/python-package/setup.cfg b/software/python-package/setup.cfg index 04b5edfd..1b15d2f4 100644 --- a/software/python-package/setup.cfg +++ b/software/python-package/setup.cfg @@ -21,8 +21,6 @@ classifiers = Intended Audience :: Information Technology Intended Audience :: Science/Research Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 @@ -36,7 +34,7 @@ package_dir = =. zip_safe = True include_package_data = True -python_requires = >= 3.8 +python_requires = >= 3.10 install_requires = shepherd-core[elf,inventory] click>=8.1.3 diff --git a/software/python-package/shepherd_sheep/__init__.py b/software/python-package/shepherd_sheep/__init__.py index 7a6f7b12..e72429b1 100644 --- a/software/python-package/shepherd_sheep/__init__.py +++ b/software/python-package/shepherd_sheep/__init__.py @@ -245,7 +245,7 @@ def run_programmer(cfg: ProgrammingTask) -> bool: return failed # TODO: all run_() should emit error and abort_on_error should decide -def run_task(cfg: Union[ShpModel, Path, str]) -> bool: +def run_task(cfg: ShpModel | Path | str) -> bool: observer_name = platform.node().strip() try: wrapper = prepare_task(cfg, observer_name) diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index 4d86828a..66717e5b 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -172,7 +172,7 @@ def eeprom(): type=click.Path(exists=True, readable=True, file_okay=True, dir_okay=False), ) def write( - cal_file: Optional[Path], + cal_file: Path | None, ): cal_cape = CalibrationCape.from_file(cal_file) try: @@ -192,7 +192,7 @@ def write( default=None, help="If provided, calibration data is dumped to this file", ) -def read(cal_file: Optional[Path]): +def read(cal_file: Path | None): set_verbosity() try: @@ -215,7 +215,7 @@ def read(cal_file: Optional[Path]): @cli.command(short_help="Start zerorpc server") @click.option("--port", "-p", type=click.INT, default=4242) -def rpc(port: Optional[int]): +def rpc(port: int | None): shepherd_io = ShepherdDebug() shepherd_io.__enter__() log.info("Shepherd Debug Interface: Initialized") diff --git a/software/python-package/shepherd_sheep/eeprom.py b/software/python-package/shepherd_sheep/eeprom.py index 3b32b676..c4b35d02 100644 --- a/software/python-package/shepherd_sheep/eeprom.py +++ b/software/python-package/shepherd_sheep/eeprom.py @@ -150,7 +150,7 @@ def __setitem__(self, key: str, value): # type: ignore else: self._write(eeprom_format[key]["offset"], value) - def _write_cape_data(self, cape_data: Optional[CapeData]) -> None: + def _write_cape_data(self, cape_data: CapeData | None) -> None: """Writes complete BeagleBone cape data to EEPROM Args: diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index 17db7478..5f69e933 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -59,14 +59,14 @@ class Writer(CoreWriter): def __init__( self, file_path: Path, - mode: Optional[str] = None, - datatype: Optional[str] = None, - window_samples: Optional[int] = None, - cal_data: Union[CalSeries, CalEmu, CalHrv, None] = None, + mode: str | None = None, + datatype: str | None = None, + window_samples: int | None = None, + cal_data: CalSeries | CalEmu | CalHrv | None = None, compression: Compression = Compression.default, modify_existing: bool = False, force_overwrite: bool = False, - verbose: Optional[bool] = True, + verbose: bool | None = True, samples_per_buffer: int = 10_000, samplerate_sps: int = 100_000, ): @@ -110,7 +110,7 @@ def __init__( # prepare Monitors self.sysutil_log_enabled: bool = True - self.monitors: List[Monitor] = [] + self.monitors: list[Monitor] = [] def __enter__(self): """Initializes the structure of the HDF5 file @@ -234,8 +234,8 @@ def write_buffer(self, buffer: DataBuffer) -> None: def start_monitors( self, - sys: Optional[SystemLogging] = None, - gpio: Optional[GpioTracing] = None, + sys: SystemLogging | None = None, + gpio: GpioTracing | None = None, ) -> None: if sys is not None and sys.dmesg: self.monitors.append(KernelMonitor(self.kernel_grp, self._compression)) diff --git a/software/python-package/shepherd_sheep/logger.py b/software/python-package/shepherd_sheep/logger.py index 7cebe4a1..b7fa9678 100644 --- a/software/python-package/shepherd_sheep/logger.py +++ b/software/python-package/shepherd_sheep/logger.py @@ -30,7 +30,7 @@ def get_verbosity() -> bool: return verbosity_state -def set_verbosity(state: Union[bool, int] = True, temporary: bool = False) -> None: +def set_verbosity(state: bool | int = True, temporary: bool = False) -> None: if isinstance(state, bool): # strange solution -> bool is also int, so it falls through below in elif if not state: diff --git a/software/python-package/shepherd_sheep/monitor_abc.py b/software/python-package/shepherd_sheep/monitor_abc.py index d1448661..05631201 100644 --- a/software/python-package/shepherd_sheep/monitor_abc.py +++ b/software/python-package/shepherd_sheep/monitor_abc.py @@ -13,7 +13,7 @@ class Monitor(ABC): def __init__( self, target: h5py.Group, - compression: Optional[Compression] = Compression.default, + compression: Compression | None = Compression.default, poll_intervall: float = 0.25, ): self.data = target @@ -21,7 +21,7 @@ def __init__( self.position = 0 self.increment = 100 self.event = threading.Event() - self.thread: Optional[threading.Thread] = None + self.thread: threading.Thread | None = None # create time, others have to be created in main class self.data.create_dataset( diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index adf60ab1..17d7014a 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -15,7 +15,7 @@ class KernelMonitor(Monitor): def __init__( self, target: h5py.Group, - compression: Optional[Compression] = Compression.default, + compression: Compression | None = Compression.default, backlog: int = 60, ): super().__init__(target, compression, poll_intervall=0.52) diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index 592dd345..5f4ec299 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -15,7 +15,7 @@ class PTPMonitor(Monitor): # TODO: also add phc2sys def __init__( self, target: h5py.Group, - compression: Optional[Compression] = Compression.default, + compression: Compression | None = Compression.default, ): super().__init__(target, compression, poll_intervall=0.51) self.data.create_dataset( diff --git a/software/python-package/shepherd_sheep/monitor_sheep.py b/software/python-package/shepherd_sheep/monitor_sheep.py index c3da2250..ecd2529a 100644 --- a/software/python-package/shepherd_sheep/monitor_sheep.py +++ b/software/python-package/shepherd_sheep/monitor_sheep.py @@ -13,7 +13,7 @@ class SheepMonitor(Monitor): def __init__( self, target: h5py.Group, - compression: Optional[Compression] = Compression.default, + compression: Compression | None = Compression.default, ): super().__init__(target, compression, poll_intervall=0.25) self.queue = get_message_queue() diff --git a/software/python-package/shepherd_sheep/monitor_sysutil.py b/software/python-package/shepherd_sheep/monitor_sysutil.py index a735f622..cd665b70 100644 --- a/software/python-package/shepherd_sheep/monitor_sysutil.py +++ b/software/python-package/shepherd_sheep/monitor_sysutil.py @@ -15,7 +15,7 @@ class SysUtilMonitor(Monitor): def __init__( self, target: h5py.Group, - compression: Optional[Compression] = Compression.default, + compression: Compression | None = Compression.default, ): super().__init__(target, compression, poll_intervall=0.3) self.log_interval_ns: int = 1 * (10**9) # step-size is 1 s diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index f1552041..07dedb78 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -15,9 +15,9 @@ class UARTMonitor(Monitor): def __init__( self, target: h5py.Group, - compression: Optional[Compression] = Compression.default, + compression: Compression | None = Compression.default, uart: str = "/dev/ttyS1", - baudrate: Optional[int] = None, + baudrate: int | None = None, ): super().__init__(target, compression, poll_intervall=0.05) self.uart = uart diff --git a/software/python-package/shepherd_sheep/shared_memory.py b/software/python-package/shepherd_sheep/shared_memory.py index 7d2e41ff..f21ca063 100644 --- a/software/python-package/shepherd_sheep/shared_memory.py +++ b/software/python-package/shepherd_sheep/shared_memory.py @@ -25,8 +25,8 @@ class GPIOEdges: def __init__( self, - timestamps_ns: Optional[np.ndarray] = None, - values: Optional[np.ndarray] = None, + timestamps_ns: np.ndarray | None = None, + values: np.ndarray | None = None, ): self.timestamps_ns = timestamps_ns if timestamps_ns is not None else np.empty(0) self.values = values if values is not None else np.empty(0) @@ -47,8 +47,8 @@ def __init__( self, voltage: np.ndarray, current: np.ndarray, - timestamp_ns: Optional[int] = None, - gpio_edges: Optional[GPIOEdges] = None, + timestamp_ns: int | None = None, + gpio_edges: GPIOEdges | None = None, util_mean: float = 0, util_max: float = 0, ): @@ -83,8 +83,8 @@ def __init__( size: int, n_buffers: int, samples_per_buffer: int, - trace_iv: Optional[PowerTracing], - trace_gpio: Optional[GpioTracing], + trace_iv: PowerTracing | None, + trace_gpio: GpioTracing | None, ): """Initializes relevant parameters for shared memory area. @@ -168,7 +168,7 @@ def __exit__(self, *args): # type: ignore os.close(self.devmem_fd) @staticmethod - def timedelta_to_ns(delta: Optional[timedelta], default_s: int = 0) -> int: + def timedelta_to_ns(delta: timedelta | None, default_s: int = 0) -> int: if isinstance(delta, timedelta): return int(delta.total_seconds() * 10**9) else: diff --git a/software/python-package/shepherd_sheep/shepherd_debug.py b/software/python-package/shepherd_sheep/shepherd_debug.py index 7958e841..9ddeff20 100644 --- a/software/python-package/shepherd_sheep/shepherd_debug.py +++ b/software/python-package/shepherd_sheep/shepherd_debug.py @@ -40,7 +40,7 @@ class ShepherdDebug(ShepherdIO): def __init__(self, use_io: bool = True): super().__init__("debug", trace_iv=PowerTracing(), trace_gpio=GpioTracing()) - self._io: Optional[TargetIO] = TargetIO() if use_io else None + self._io: TargetIO | None = TargetIO() if use_io else None # offer a default cali for debugging self._cal: CalibrationCape = CalibrationCape() @@ -137,7 +137,7 @@ def dac_write(self, channels: int, value: int): def get_buffer( self, - timeout_n: Optional[float] = None, + timeout_n: float | None = None, verbose: bool = False, ) -> NoReturn: raise NotImplementedError("Method not implemented for debugging mode") @@ -158,7 +158,7 @@ def vsource_init( cal_emu: CalibrationEmulator, log_intermediate: bool = False, dtype_in: EnergyDType = EnergyDType.ivsample, - window_size: Optional[int] = None, + window_size: int | None = None, ): super().send_calibration_settings(cal_emu) src_pru = ConverterPRUConfig.from_vsrc(src_cfg, log_intermediate) @@ -211,7 +211,7 @@ def cnv_charge( self, input_voltage_uV: int, input_current_nA: int, - ) -> Tuple[int, int]: + ) -> tuple[int, int]: self._send_msg( commons.MSG_DBG_VSRC_CHARGE, [int(input_voltage_uV), int(input_current_nA)], @@ -234,7 +234,7 @@ def cnv_calc_out_power(self, current_adc_raw: int) -> int: ) return values[0] * (2**32) + values[1] # P_out_pW - def cnv_drain(self, current_adc_raw: int) -> Tuple[int, int]: + def cnv_drain(self, current_adc_raw: int) -> tuple[int, int]: self._send_msg(commons.MSG_DBG_VSRC_DRAIN, int(current_adc_raw)) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_DRAIN: @@ -303,7 +303,7 @@ def set_shepherd_pcb_power(self, state: bool) -> None: def select_port_for_power_tracking( self, - target: Union[TargetPort, bool, None], + target: TargetPort | bool | None, ) -> None: super().select_port_for_power_tracking(target) diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index d33e61b9..3eb96eaf 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -112,7 +112,7 @@ def __init__( ) log.info("Virtual Source will be initialized to:\n%s", cfg.virtual_source) - self.writer: Optional[Writer] = None + self.writer: Writer | None = None if cfg.output_path is not None: store_path = cfg.output_path.resolve() if store_path.is_dir(): @@ -133,7 +133,7 @@ def __init__( ) # hard-wire pin-direction until they are configurable - self._io: Optional[TargetIO] = TargetIO() + self._io: TargetIO | None = TargetIO() log.info("Setting variable GPIO to INPUT (actuation is not implemented yet)") for pin in range(len(target_pins)): self._io.set_pin_direction(pin, pdir=True) # True = Inp diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index ee8110a7..141622ec 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -80,8 +80,8 @@ def __new__(cls, *args, **kwds): def __init__( self, mode: str, - trace_iv: Optional[PowerTracing], - trace_gpio: Optional[GpioTracing], + trace_iv: PowerTracing | None, + trace_gpio: GpioTracing | None, ): """Initializes relevant variables. @@ -152,7 +152,7 @@ def __exit__(self, *args): # type: ignore ShepherdIO._instance = None @staticmethod - def _send_msg(msg_type: int, values: Union[int, list]) -> None: + def _send_msg(msg_type: int, values: int | list) -> None: """Sends a formatted message to PRU0. Args: @@ -189,7 +189,7 @@ def _flush_msgs(): def start( self, - start_time: Optional[float] = None, + start_time: float | None = None, wait_blocking: bool = True, ) -> None: """Starts sampling either now or at later point in time. @@ -331,7 +331,7 @@ def set_power_state_emulator(self, state: bool) -> None: time.sleep(0.3) # time to stabilize voltage-drop @staticmethod - def convert_target_port_to_bool(target: Union[TargetPort, str, bool, None]) -> bool: + def convert_target_port_to_bool(target: TargetPort | str | bool | None) -> bool: if target is None: return True elif isinstance(target, str): @@ -346,7 +346,7 @@ def convert_target_port_to_bool(target: Union[TargetPort, str, bool, None]) -> b def select_port_for_power_tracking( self, - target: Union[TargetPort, bool, None], + target: TargetPort | bool | None, ) -> None: """ choose which targets (A or B) gets the supply with current-monitor, @@ -371,7 +371,7 @@ def select_port_for_power_tracking( def select_port_for_io_interface( self, - target: Union[TargetPort, bool, None], + target: TargetPort | bool | None, ) -> None: """choose which targets (A or B) gets the io-connection (serial, swd, gpio) from beaglebone, @@ -405,7 +405,7 @@ def set_io_level_converter(self, state: bool) -> None: @staticmethod def set_aux_target_voltage( voltage: float, - cal_emu: Optional[CalibrationEmulator] = None, + cal_emu: CalibrationEmulator | None = None, ) -> None: """Enables or disables the voltage for the second target @@ -420,7 +420,7 @@ def set_aux_target_voltage( sfs.write_dac_aux_voltage(voltage, cal_emu) @staticmethod - def get_aux_voltage(cal_emu: Optional[CalibrationEmulator] = None) -> float: + def get_aux_voltage(cal_emu: CalibrationEmulator | None = None) -> float: """Reads the auxiliary voltage (dac channel B) from the PRU core. Args: @@ -434,7 +434,7 @@ def get_aux_voltage(cal_emu: Optional[CalibrationEmulator] = None) -> float: @validate_call def send_calibration_settings( self, - cal_: Union[CalibrationEmulator, CalibrationHarvester, None], + cal_: CalibrationEmulator | CalibrationHarvester | None, ) -> None: """Sends calibration settings to PRU core diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index ad3ef4f0..fa271f75 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -144,7 +144,7 @@ def wait_for_state(wanted_state: str, timeout: float) -> float: time.sleep(0.1) -def set_start(start_time: Union[float, int, None] = None) -> None: +def set_start(start_time: float | int | None = None) -> None: """Starts shepherd. Writes 'start' to the 'state' sysfs attribute in order to transition from @@ -212,8 +212,8 @@ def write_mode(mode: str, force: bool = False) -> None: @validate_call def write_dac_aux_voltage( - voltage: Union[float, str, None], - cal_emu: Optional[CalibrationEmulator] = None, + voltage: float | str | None, + cal_emu: CalibrationEmulator | None = None, ) -> None: """Sends the auxiliary voltage (dac channel B) to the PRU core. @@ -278,7 +278,7 @@ def write_dac_aux_voltage_raw( f.write(str(voltage_raw)) -def read_dac_aux_voltage(cal_emu: Optional[CalibrationEmulator] = None) -> float: +def read_dac_aux_voltage(cal_emu: CalibrationEmulator | None = None) -> float: """Reads the auxiliary voltage (dac channel B) from the PRU core. Args: @@ -444,7 +444,7 @@ def read_virtual_harvester_settings() -> list: return int_settings -def write_pru_msg(msg_type: int, values: Union[list, float, int]) -> None: +def write_pru_msg(msg_type: int, values: list | float | int) -> None: """ :param msg_type: :param values: diff --git a/software/python-package/shepherd_sheep/target_io.py b/software/python-package/shepherd_sheep/target_io.py index 47652101..5e8c7d09 100644 --- a/software/python-package/shepherd_sheep/target_io.py +++ b/software/python-package/shepherd_sheep/target_io.py @@ -41,7 +41,7 @@ from periphery import GPIO -target_pins: List[Dict] = [ # pin-order from target-connector +target_pins: list[dict] = [ # pin-order from target-connector {"name": "gpio0", "pin": 26, "dir": 78}, {"name": "gpio1", "pin": 27, "dir": 78}, {"name": "gpio2", "pin": 46, "dir": 78}, @@ -66,12 +66,12 @@ def __init__(self): """ dir_pins = {pin["dir"] for pin in target_pins if isinstance(pin["dir"], int)} - self.dirs: Dict[int, GPIO] = {} + self.dirs: dict[int, GPIO] = {} for pin in dir_pins: self.dirs[pin] = GPIO(pin, "out") self.dirs[pin].write(True) # True == Output to target - self.gpios: Dict[str, GPIO] = {} + self.gpios: dict[str, GPIO] = {} for pin_info in target_pins: if pin_info["dir"] == "I": self.gpios[pin_info["name"]] = GPIO(pin_info["pin"], "in") @@ -79,7 +79,7 @@ def __init__(self): self.gpios[pin_info["name"]] = GPIO(pin_info["pin"], "out") self.gpios[pin_info["name"]].write(False) # init LOW - self.pin_names: List[str] = [pin["name"] for pin in target_pins] + self.pin_names: list[str] = [pin["name"] for pin in target_pins] self.pin_count: int = len(target_pins) def one_high(self, num: int) -> None: diff --git a/software/shepherd-calibration/pyproject.toml b/software/shepherd-calibration/pyproject.toml index 622f7926..88bdc4ee 100644 --- a/software/shepherd-calibration/pyproject.toml +++ b/software/shepherd-calibration/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [tool.pyright] root = "./" include = ['./shepherd_cal', ] -pythonVersion = "3.8" +pythonVersion = "3.10" pythonPlatform = "All" # strict = ["./"] #reportUnknownParameterType = true diff --git a/software/shepherd-calibration/setup.cfg b/software/shepherd-calibration/setup.cfg index 9bb5a218..c9845afd 100644 --- a/software/shepherd-calibration/setup.cfg +++ b/software/shepherd-calibration/setup.cfg @@ -20,8 +20,6 @@ classifiers = Intended Audience :: Information Technology Intended Audience :: Science/Research Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 @@ -35,7 +33,7 @@ package_dir = =. zip_safe = True include_package_data = True -python_requires = >= 3.8 +python_requires = >= 3.10 install_requires = typer[all] fabric diff --git a/software/shepherd-calibration/shepherd_cal/calibrator.py b/software/shepherd-calibration/shepherd_cal/calibrator.py index 972a3d54..e8cb0230 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator.py @@ -50,12 +50,12 @@ def __init__( self, host: str, user: str, - password: Optional[str] = None, - smu_ip: Optional[str] = None, + password: str | None = None, + smu_ip: str | None = None, mode_4wire: bool = True, pwrline_cycles: float = 16, ): - fabric_args: Dict[str, str] = {} + fabric_args: dict[str, str] = {} if password is not None: fabric_args["password"] = password @@ -337,7 +337,7 @@ def measure_dac_voltage( return results def measure_harvester(self) -> CalMeasurementHarvester: - results: Dict[str, CalMeasPairs] = {} + results: dict[str, CalMeasPairs] = {} logger.info("Measurement - Harvester - ADC . Voltage") results["adc_V_Sense"] = self.measure_harvester_adc_voltage(self.kth.smub) @@ -352,7 +352,7 @@ def measure_harvester(self) -> CalMeasurementHarvester: return CalMeasurementHarvester(**results) def measure_emulator(self) -> CalMeasurementEmulator: - results: Dict[str, CalMeasPairs] = {} + results: dict[str, CalMeasPairs] = {} logger.info("Measurement - Emulator - ADC . Current - Target A") # targetA-Port will get the monitored dac-channel-b self.sheep.select_port_for_power_tracking(TargetPort.A) @@ -383,7 +383,7 @@ def measure_emulator(self) -> CalMeasurementEmulator: def write( self, - cal_file: Union[str, Path], + cal_file: str | Path, ): temp_file = "/tmp/calib.yaml" # noqa: S108 if isinstance(cal_file, str): diff --git a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py index 1883ef5f..3109ee73 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py @@ -52,8 +52,8 @@ def measure( host: str = host_arg_t, user: str = user_opt_t, - password: Optional[str] = pass_opt_t, - outfile: Optional[Path] = ofile_opt_t, + password: str | None = pass_opt_t, + outfile: Path | None = ofile_opt_t, smu_ip: str = smu_ip_opt_t, smu_2wire: bool = smu_2w_opt_t, smu_nplc: float = smu_nc_opt_t, @@ -61,7 +61,7 @@ def measure( emulator: bool = emu_opt_t, cape_serial: str = serial_opt_t, write: bool = write_opt_t, - version: Optional[str] = version_opt_t, + version: str | None = version_opt_t, verbose: bool = verbose_opt_t, ): """Measure calibration-data for shepherd cape""" @@ -131,9 +131,9 @@ def measure( def write( host: str = host_arg_t, user: str = user_opt_t, - password: Optional[str] = pass_opt_t, - cal_file: Optional[Path] = ifile_opt_t, - measurement_file: Optional[Path] = ifile_opt_t, + password: str | None = pass_opt_t, + cal_file: Path | None = ifile_opt_t, + measurement_file: Path | None = ifile_opt_t, verbose: bool = verbose_opt_t, ): """Write calibration-data to shepherd cape eeprom (choose cal- or measurement-file)""" @@ -160,8 +160,8 @@ def write( def read( host: str = host_arg_t, user: str = user_opt_t, - password: Optional[str] = pass_opt_t, - cal_file: Optional[Path] = ofile_opt_t, + password: str | None = pass_opt_t, + cal_file: Path | None = ofile_opt_t, verbose: bool = verbose_opt_t, ): """Read calibration-data from shepherd cape""" diff --git a/software/shepherd-calibration/shepherd_cal/logger.py b/software/shepherd-calibration/shepherd_cal/logger.py index d59cc066..80a1c0ef 100644 --- a/software/shepherd-calibration/shepherd_cal/logger.py +++ b/software/shepherd-calibration/shepherd_cal/logger.py @@ -5,7 +5,7 @@ logger.setLevel(logging.INFO) -def set_verbosity(state: Union[bool, int] = True) -> None: +def set_verbosity(state: bool | int = True) -> None: if isinstance(state, bool): # strange solution -> bool is also int, so it falls through below in elif if not state: diff --git a/software/shepherd-calibration/shepherd_cal/profile.py b/software/shepherd-calibration/shepherd_cal/profile.py index 2e24c7ec..3a56db85 100644 --- a/software/shepherd-calibration/shepherd_cal/profile.py +++ b/software/shepherd-calibration/shepherd_cal/profile.py @@ -9,7 +9,7 @@ from .logger import logger from .profile_calibration import ProfileCalibration -component_dict: Dict[str, str] = { +component_dict: dict[str, str] = { "a": "emu_a", "emu_a": "emu_a", "b": "emu_b", @@ -18,7 +18,7 @@ "hrv": "hrv", } -elem_dict: Dict[str, int] = { +elem_dict: dict[str, int] = { "voltage_shp_V": 0, "voltage_shp_raw": 1, "voltage_ref_V": 2, @@ -26,7 +26,7 @@ "current_shp_raw": 4, "current_ref_A": 5, } -elem_list: List[str] = [ +elem_list: list[str] = [ "v_shp_V", "v_shp_raw", "v_ref_V", @@ -50,14 +50,14 @@ def __init__(self, file: Path): meas_file = np.load(str(file), allow_pickle=True) self.file_name: str = file.stem - self.data: Dict[str, pd.DataFrame] = {} - self.cals: Dict[str, ProfileCalibration] = {} + self.data: dict[str, pd.DataFrame] = {} + self.cals: dict[str, ProfileCalibration] = {} - self.results: Dict[str, pd.DataFrame] = {} - self.stats: List[pd.DataFrame] = [] + self.results: dict[str, pd.DataFrame] = {} + self.stats: list[pd.DataFrame] = [] - self.data_filters: Dict[str, pd.Series] = {} - self.res_filters: Dict[str, pd.Series] = {} + self.data_filters: dict[str, pd.Series] = {} + self.res_filters: dict[str, pd.Series] = {} for comp_i in component_dict: if comp_i not in meas_file: diff --git a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py index bf5127aa..3e25ca2d 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py +++ b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py @@ -10,7 +10,7 @@ def analyze_directory( folder_path: Path, - stats_path: Optional[Path] = None, + stats_path: Path | None = None, do_plots: bool = False, ) -> None: stats_list = [] @@ -27,7 +27,7 @@ def analyze_directory( if "origin" in stats_base.columns: stat_names = stats_base["origin"].tolist() - files: List[str] = [] + files: list[str] = [] if folder_path.is_file(): files.append(str(folder_path)) elif folder_path.is_dir(): diff --git a/software/shepherd-calibration/shepherd_cal/profile_calibration.py b/software/shepherd-calibration/shepherd_cal/profile_calibration.py index 9d80c9e8..90c86705 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_calibration.py +++ b/software/shepherd-calibration/shepherd_cal/profile_calibration.py @@ -19,7 +19,7 @@ class ProfileCalibration(CalibrationSeries): @classmethod def from_measurement(cls, result: pd.DataFrame): - values: Dict[str, CalibrationPair] = {} + values: dict[str, CalibrationPair] = {} cal_c = cls._determine_current_cal(result) cal_v = cls._determine_voltage_cal(result) if cal_c is not None: @@ -30,9 +30,9 @@ def from_measurement(cls, result: pd.DataFrame): @staticmethod def _measurements_to_calibration( - ref: Union[pd.Series, float], - raw: Union[pd.Series, float], - ) -> Tuple[float, float]: + ref: pd.Series | float, + raw: pd.Series | float, + ) -> tuple[float, float]: result = stats.linregress(raw, ref) offset = float(result.intercept) gain = float(result.slope) @@ -44,7 +44,7 @@ def _measurements_to_calibration( return float(gain), float(offset) @staticmethod - def _determine_current_cal(result: pd.DataFrame) -> Optional[CalibrationPair]: + def _determine_current_cal(result: pd.DataFrame) -> CalibrationPair | None: # chose first voltage above 2.4 V as base, currents range from 60 uA to 14 mA result = ( result.groupby(by=["c_ref_A", "v_shp_V"]).mean().reset_index(drop=False) @@ -79,7 +79,7 @@ def _determine_current_cal(result: pd.DataFrame) -> Optional[CalibrationPair]: return CalibrationPair(gain=gain, offset=offset) @staticmethod - def _determine_voltage_cal(result: pd.DataFrame) -> Optional[CalibrationPair]: + def _determine_voltage_cal(result: pd.DataFrame) -> CalibrationPair | None: # chose first current above 60 uA as base, voltages range from 0.3 V to 2.6 V result = ( result.groupby(by=["c_ref_A", "v_shp_V"]).mean().reset_index(drop=False) diff --git a/software/shepherd-calibration/shepherd_cal/profiler.py b/software/shepherd-calibration/shepherd_cal/profiler.py index a281fa0c..19e286a0 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler.py +++ b/software/shepherd-calibration/shepherd_cal/profiler.py @@ -46,7 +46,7 @@ def __init__(self, calibrator: Calibrator, short: bool = False): [5.0, 0.05], np.arange(0.0, 5.1, 0.4), ) - self.currents_A: List[float] = [ + self.currents_A: list[float] = [ 0e-6, 1e-6, 2e-6, @@ -69,7 +69,7 @@ def __init__(self, calibrator: Calibrator, short: bool = False): ] else: self.voltages_V: np.ndarray = np.append([0.05], np.arange(0.0, 5.1, 0.2)) - self.currents_A: List[float] = [ + self.currents_A: list[float] = [ 0e-6, 1e-6, 2e-6, @@ -99,7 +99,7 @@ def measure_emulator_setpoint( smu: KeithleyClass, voltage_V: float, current_A: float = 0, - ) -> Tuple[np.ndarray, float, float]: + ) -> tuple[np.ndarray, float, float]: voltage_V = min(max(voltage_V, 0.0), 5.0) # negative current, because smu acts as a drain @@ -150,7 +150,7 @@ def measure_harvester_setpoint( smu: KeithleyClass, voltage_V: float, current_A: float = 0, - ) -> Tuple[np.ndarray, np.ndarray, float, float]: + ) -> tuple[np.ndarray, np.ndarray, float, float]: voltage_V = min(max(voltage_V, 0.0), 5.0) # SMU as current-source diff --git a/software/shepherd-calibration/shepherd_cal/profiler_cli.py b/software/shepherd-calibration/shepherd_cal/profiler_cli.py index e23bf1a8..7713c696 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler_cli.py +++ b/software/shepherd-calibration/shepherd_cal/profiler_cli.py @@ -54,8 +54,8 @@ def measure( host: str = host_arg_t, user: str = user_opt_t, - password: Optional[str] = pass_opt_t, - outfile: Optional[Path] = ofile_opt_t, + password: str | None = pass_opt_t, + outfile: Path | None = ofile_opt_t, smu_ip: str = smu_ip_opt_t, smu_2wire: bool = smu_2w_opt_t, smu_nplc: float = smu_nc_opt_t, @@ -86,7 +86,7 @@ def measure( shpcal = Calibrator(host, user, password, smu_ip, smu_4wire, smu_nplc) profiler = Profiler(shpcal, short) - results: Dict[str, np.ndarray] = {"cape": cape_serial} + results: dict[str, np.ndarray] = {"cape": cape_serial} if not quiet: click.echo(INSTR_PROFILE_SHP) @@ -137,7 +137,7 @@ def measure( @cli_pro.command() def analyze( infiles: Path = in_files_arg_t, - outfile: Optional[Path] = out_file_opt_t, + outfile: Path | None = out_file_opt_t, plot: bool = plot_opt_t, verbose: bool = verbose_opt_t, ): diff --git a/software/shepherd-herd/pyproject.toml b/software/shepherd-herd/pyproject.toml index dfc024cd..99f50ffc 100644 --- a/software/shepherd-herd/pyproject.toml +++ b/software/shepherd-herd/pyproject.toml @@ -5,6 +5,6 @@ build-backend = "setuptools.build_meta" [tool.pyright] root = "./" include = ['./shepherd_herd', ] -pythonVersion = "3.8" +pythonVersion = "3.10" pythonPlatform = "All" reportMissingParameterType = true diff --git a/software/shepherd-herd/setup.cfg b/software/shepherd-herd/setup.cfg index 5f2a4f77..cbf613bf 100644 --- a/software/shepherd-herd/setup.cfg +++ b/software/shepherd-herd/setup.cfg @@ -20,8 +20,6 @@ classifiers = Intended Audience :: Information Technology Intended Audience :: Science/Research Programming Language :: Python :: 3 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 @@ -35,7 +33,7 @@ package_dir = =. zip_safe = True include_package_data = True -python_requires = >= 3.8 +python_requires = >= 3.10 install_requires = click numpy diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index a0b1c2fa..688e1aaa 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -46,12 +46,12 @@ class Herd: def __init__( self, - inventory: Optional[str] = None, - limit: Optional[str] = None, - user: Optional[str] = None, - key_filepath: Optional[Path] = None, + inventory: str | None = None, + limit: str | None = None, + user: str | None = None, + key_filepath: Path | None = None, ): - limits_list: Optional[List[str]] = None + limits_list: list[str] | None = None if isinstance(limit, str): limits_list = limit.split(",") limits_list = [_host for _host in limits_list if len(_host) >= 1] @@ -95,9 +95,9 @@ def __init__( logger.info("Shepherd-Inventory = '%s'", host_path.as_posix()) hostlist = [] - hostnames: Dict[str, str] = {} + hostnames: dict[str, str] = {} for hostname, hostvars in inventory_data["sheep"]["hosts"].items(): - if isinstance(limits_list, List) and (hostname not in limits_list): + if isinstance(limits_list, list) and (hostname not in limits_list): continue if "ansible_host" in hostvars: @@ -119,7 +119,7 @@ def __init__( "Provide remote hosts (either inventory empty or limit does not match)", ) - connect_kwargs: Dict[str, str] = {} + connect_kwargs: dict[str, str] = {} if key_filepath is not None: connect_kwargs["key_filename"] = str(key_filepath) @@ -129,7 +129,7 @@ def __init__( connect_timeout=5, connect_kwargs=connect_kwargs, ) - self.hostnames: Dict[str, str] = hostnames + self.hostnames: dict[str, str] = hostnames logger.info("Herd consists of %d sheep", len(self.group)) @@ -251,7 +251,7 @@ def print_output(self, replies: dict[int, Result], verbose: bool = False) -> Non @staticmethod def _thread_put( cnx: Connection, - src: Union[Path, StringIO], + src: Path | StringIO, dst: Path, force_overwrite: bool, ): @@ -283,8 +283,8 @@ def _thread_put( def put_file( self, - src: Union[StringIO, Path, str], - dst: Union[Path, str], + src: StringIO | Path | str, + dst: Path | str, force_overwrite: bool = False, ) -> None: if isinstance(src, StringIO): @@ -338,8 +338,8 @@ def _thread_get(cnx: Connection, src: Path, dst: Path): @validate_call def get_file( self, - src: Union[Path, str], - dst_dir: Union[Path, str], + src: Path | str, + dst_dir: Path | str, timestamp: bool = False, separate: bool = False, delete_src: bool = False, @@ -421,7 +421,7 @@ def get_file( del threads return failed_retrieval - def find_consensus_time(self) -> Tuple[datetime, float]: + def find_consensus_time(self) -> tuple[datetime, float]: """Finds a start time in the future when all nodes should start service In order to run synchronously, all nodes should start at the same time. @@ -451,8 +451,8 @@ def find_consensus_time(self) -> Tuple[datetime, float]: @validate_call def put_task( self, - task: Union[Path, ShpModel], - remote_path: Union[Path, str] = "/etc/shepherd/config.yaml", + task: Path | ShpModel, + remote_path: Path | str = "/etc/shepherd/config.yaml", ) -> None: """transfers shepherd tasks to the group of hosts / sheep. @@ -521,6 +521,7 @@ def check_status(self, warn: bool = False) -> bool: "shepherd still active on %s", self.hostnames[cnx.host], ) + # shepherd-herd -v shell-cmd -s "systemctl status shepherd" return active def start_measurement(self) -> int: @@ -594,7 +595,7 @@ def inventorize(self, output_path: Path) -> bool: return failed @validate_call - def run_task(self, config: Union[Path, ShpModel], attach: bool = False) -> int: + def run_task(self, config: Path | ShpModel, attach: bool = False) -> int: if attach: remote_path = Path("/etc/shepherd/config_for_herd.yaml") self.put_task(config, remote_path) @@ -617,8 +618,8 @@ def run_task(self, config: Union[Path, ShpModel], attach: bool = False) -> int: @validate_call def get_task_files( self, - config: Union[Path, ShpModel], - dst_dir: Union[Path, str], + config: Path | ShpModel, + dst_dir: Path | str, separate: bool = False, delete_src: bool = False, ) -> bool: diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index 2a433291..4971186b 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -67,10 +67,10 @@ def exit_gracefully(*args) -> None: # type: ignore @click.pass_context def cli( ctx: click.Context, - inventory: Optional[str], - limit: Optional[str], - user: Optional[str], - key_filepath: Optional[Path], + inventory: str | None, + limit: str | None, + user: str | None, + key_filepath: Path | None, verbose: bool, version: bool, ): diff --git a/software/shepherd-herd/shepherd_herd/logger.py b/software/shepherd-herd/shepherd_herd/logger.py index fe21e275..a1637c88 100644 --- a/software/shepherd-herd/shepherd_herd/logger.py +++ b/software/shepherd-herd/shepherd_herd/logger.py @@ -15,7 +15,7 @@ def get_verbosity() -> bool: return verbosity_state -def set_verbosity(state: Union[bool, int] = True) -> None: +def set_verbosity(state: bool | int = True) -> None: if isinstance(state, bool): # strange solution -> bool is also int, so it falls through below in elif if not state: From e26d38f5267e413c237823660c8f0776017c45bb Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 11:25:21 +0200 Subject: [PATCH 09/35] remove not needed imports, switch to sets and more efficient code --- .gitmodules | 6 ++++++ .../firmware/pru0-cython-module/testing.py | 2 +- .../python-package/shepherd_sheep/__init__.py | 3 +-- software/python-package/shepherd_sheep/cli.py | 1 - .../python-package/shepherd_sheep/eeprom.py | 1 - .../shepherd_sheep/h5_writer.py | 3 --- .../python-package/shepherd_sheep/launcher.py | 4 ++-- .../python-package/shepherd_sheep/logger.py | 1 - .../shepherd_sheep/monitor_abc.py | 1 - .../shepherd_sheep/monitor_kernel.py | 1 - .../shepherd_sheep/monitor_ptp.py | 1 - .../shepherd_sheep/monitor_sheep.py | 1 - .../shepherd_sheep/monitor_sysutil.py | 1 - .../shepherd_sheep/monitor_uart.py | 1 - .../shepherd_sheep/shared_memory.py | 1 - .../shepherd_sheep/shepherd_debug.py | 21 ++++++++----------- .../shepherd_sheep/shepherd_emulator.py | 1 - .../shepherd_sheep/shepherd_io.py | 4 +--- .../shepherd_sheep/sysfs_interface.py | 2 -- .../shepherd_sheep/target_io.py | 2 -- .../python-package/tests/test_cli_misc.py | 6 +++--- .../shepherd_cal/calibrator.py | 3 --- .../shepherd_cal/calibrator_cli.py | 1 - .../shepherd_cal/logger.py | 1 - .../shepherd_cal/profile.py | 2 -- .../shepherd_cal/profile_analyzer.py | 2 -- .../shepherd_cal/profile_calibration.py | 4 ---- .../shepherd_cal/profiler.py | 2 -- .../shepherd_cal/profiler_cli.py | 2 -- software/shepherd-devicetest/src/__init__.py | 2 +- software/shepherd-herd/shepherd_herd/herd.py | 7 +------ .../shepherd-herd/shepherd_herd/herd_cli.py | 1 - .../shepherd-herd/shepherd_herd/logger.py | 1 - 33 files changed, 25 insertions(+), 67 deletions(-) diff --git a/.gitmodules b/.gitmodules index a7d9bd4e..fb789b10 100644 --- a/.gitmodules +++ b/.gitmodules @@ -9,3 +9,9 @@ [submodule "targets"] path = software/shepherd-targets url = https://github.com/orgua/shepherd-targets.git + +# helpful commands: +# git submodule update --init --recursive +# git submodule update --recursive --remote +# git submodule foreach git reset --hard + diff --git a/software/firmware/pru0-cython-module/testing.py b/software/firmware/pru0-cython-module/testing.py index f1453152..21a515a3 100644 --- a/software/firmware/pru0-cython-module/testing.py +++ b/software/firmware/pru0-cython-module/testing.py @@ -10,6 +10,6 @@ vccfg = ConverterPRUConfig.from_vsrc(data=vscfg) vconv = VirtualConverterModel(cfg=vccfg.export_for_sysfs()) -x = int(0) +x = 0 print(vconv.calc_out_power(x)) print(vconv.calc_inp_power(5, 6)) diff --git a/software/python-package/shepherd_sheep/__init__.py b/software/python-package/shepherd_sheep/__init__.py index e72429b1..9241d6d6 100644 --- a/software/python-package/shepherd_sheep/__init__.py +++ b/software/python-package/shepherd_sheep/__init__.py @@ -14,7 +14,6 @@ import time from contextlib import ExitStack from pathlib import Path -from typing import Union from shepherd_core.data_models import FirmwareDType from shepherd_core.data_models import ShpModel @@ -102,7 +101,7 @@ def run_firmware_mod(cfg: FirmwareModTask) -> bool: set_verbosity(cfg.verbose, temporary=True) check_sys_access() # not really needed here file_path = extract_firmware(cfg.data, cfg.data_type, cfg.firmware_file) - if cfg.data_type in [FirmwareDType.path_elf, FirmwareDType.base64_elf]: + if cfg.data_type in {FirmwareDType.path_elf, FirmwareDType.base64_elf}: modify_uid(file_path, cfg.custom_id) file_path = firmware_to_hex(file_path) if file_path.as_posix() != cfg.firmware_file.as_posix(): diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index 66717e5b..d88f3a52 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -12,7 +12,6 @@ import sys import time from pathlib import Path -from typing import Optional import click import gevent diff --git a/software/python-package/shepherd_sheep/eeprom.py b/software/python-package/shepherd_sheep/eeprom.py index c4b35d02..cfb1d70b 100644 --- a/software/python-package/shepherd_sheep/eeprom.py +++ b/software/python-package/shepherd_sheep/eeprom.py @@ -13,7 +13,6 @@ import os import struct from contextlib import suppress -from typing import Optional from shepherd_core.data_models.base.calibration import CalibrationCape from shepherd_core.data_models.base.calibration import CapeData diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index 5f69e933..20613c29 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -9,9 +9,6 @@ :license: MIT, see LICENSE for more details. """ from pathlib import Path -from typing import List -from typing import Optional -from typing import Union import h5py import numpy as np diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index a7c54dd1..97de6f73 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -131,7 +131,7 @@ def get_state(self, timeout: float = 10) -> bool: "ActiveState", dbus_interface="org.freedesktop.DBus.Properties", ) - if systemd_state in ["deactivating", "activating"]: + if systemd_state in {"deactivating", "activating"}: time.sleep(0.1) else: break @@ -157,7 +157,7 @@ def set_service(self, requested_state: bool): if requested_state == active_state: log.debug("service already in requested state") self.gpio_led.write(active_state) - return + return None if active_state: log.info("stopping service") diff --git a/software/python-package/shepherd_sheep/logger.py b/software/python-package/shepherd_sheep/logger.py index b7fa9678..3d830e4a 100644 --- a/software/python-package/shepherd_sheep/logger.py +++ b/software/python-package/shepherd_sheep/logger.py @@ -1,7 +1,6 @@ import logging.handlers import multiprocessing import sys -from typing import Union import chromalog from shepherd_core.logger import set_log_verbose_level diff --git a/software/python-package/shepherd_sheep/monitor_abc.py b/software/python-package/shepherd_sheep/monitor_abc.py index 05631201..59a3c19e 100644 --- a/software/python-package/shepherd_sheep/monitor_abc.py +++ b/software/python-package/shepherd_sheep/monitor_abc.py @@ -1,7 +1,6 @@ import threading from abc import ABC from abc import abstractmethod -from typing import Optional import h5py from shepherd_core import Compression diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index 17d7014a..ecb0d5fd 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -2,7 +2,6 @@ import subprocess # noqa: S404 import threading import time -from typing import Optional import h5py from shepherd_core import Compression diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index 5f4ec299..ba5fbdb1 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -2,7 +2,6 @@ import subprocess # noqa: S404 import threading import time -from typing import Optional import h5py from shepherd_core import Compression diff --git a/software/python-package/shepherd_sheep/monitor_sheep.py b/software/python-package/shepherd_sheep/monitor_sheep.py index ecd2529a..d19c2a41 100644 --- a/software/python-package/shepherd_sheep/monitor_sheep.py +++ b/software/python-package/shepherd_sheep/monitor_sheep.py @@ -1,5 +1,4 @@ import threading -from typing import Optional import h5py from shepherd_core import Compression diff --git a/software/python-package/shepherd_sheep/monitor_sysutil.py b/software/python-package/shepherd_sheep/monitor_sysutil.py index cd665b70..bba8b1b8 100644 --- a/software/python-package/shepherd_sheep/monitor_sysutil.py +++ b/software/python-package/shepherd_sheep/monitor_sysutil.py @@ -1,6 +1,5 @@ import threading import time -from typing import Optional import h5py import numpy as np diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index 07dedb78..1c465358 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -1,7 +1,6 @@ import threading import time from pathlib import Path -from typing import Optional import h5py import serial diff --git a/software/python-package/shepherd_sheep/shared_memory.py b/software/python-package/shepherd_sheep/shared_memory.py index f21ca063..7f23a84e 100644 --- a/software/python-package/shepherd_sheep/shared_memory.py +++ b/software/python-package/shepherd_sheep/shared_memory.py @@ -4,7 +4,6 @@ import time from dataclasses import dataclass from datetime import timedelta -from typing import Optional import numpy as np from shepherd_core.data_models import GpioTracing diff --git a/software/python-package/shepherd_sheep/shepherd_debug.py b/software/python-package/shepherd_sheep/shepherd_debug.py index 9ddeff20..1899e72d 100644 --- a/software/python-package/shepherd_sheep/shepherd_debug.py +++ b/software/python-package/shepherd_sheep/shepherd_debug.py @@ -1,9 +1,6 @@ import contextlib import time from typing import NoReturn -from typing import Optional -from typing import Tuple -from typing import Union import msgpack import msgpack_numpy @@ -76,17 +73,17 @@ def adc_read(self, channel: str): Returns: Binary ADC value read from corresponding channel """ - if channel.lower() in ["hrv_a_in", "hrv_i_in", "a_in", "i_in"]: + if channel.lower() in {"hrv_a_in", "hrv_i_in", "a_in", "i_in"}: channel_no = 0 - elif channel.lower() in ["hrv_v_in", "v_in"]: + elif channel.lower() in {"hrv_v_in", "v_in"}: channel_no = 1 - elif channel.lower() in [ + elif channel.lower() in { "emu", "emu_a_out", "emu_i_out", "a_out", "i_out", - ]: + }: channel_no = 2 else: raise ValueError(f"ADC channel { channel } unknown") @@ -320,33 +317,33 @@ def convert_value_to_raw(self, component: str, channel: str, value: float) -> in return self._cal[component][channel].si_to_raw(value) def set_gpio_one_high(self, num: int) -> None: - if not (self._io is None): + if self._io is not None: self._io.one_high(num) else: log.debug("Error: IO is not enabled in this shepherd-debug-instance") def get_gpio_state(self, num: int) -> bool: - if not (self._io is None): + if self._io is not None: return self._io.get_pin_state(num) else: log.debug("Error: IO is not enabled in this shepherd-debug-instance") return False def set_gpio_direction(self, num: int, pdir: bool) -> None: - if not (self._io is None): + if self._io is not None: self._io.set_pin_direction(num, pdir) else: log.debug("Error: IO is not enabled in this shepherd-debug-instance") def get_gpio_direction(self, num: int) -> bool: - if not (self._io is None): + if self._io is not None: return self._io.get_pin_direction(num) else: log.debug("Error: IO is not enabled in this shepherd-debug-instance") return True def get_gpio_info(self) -> list: - if not (self._io is None): + if self._io is not None: return self._io.pin_names else: log.debug("Error: IO is not enabled in this shepherd-debug-instance") diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index 3eb96eaf..fa7265b7 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -3,7 +3,6 @@ import time from contextlib import ExitStack from datetime import datetime -from typing import Optional from shepherd_core import CalibrationPair from shepherd_core import CalibrationSeries diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 141622ec..d93b99da 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -9,8 +9,6 @@ """ import time from contextlib import suppress -from typing import Optional -from typing import Union from pydantic import validate_call from shepherd_core import CalibrationEmulator @@ -94,7 +92,7 @@ def __init__( sfs.load_pru0_firmware("shepherd") self.mode = mode - if mode in ["harvester", "emulator"]: + if mode in {"harvester", "emulator"}: self.component = mode # TODO: still needed? else: self.component = "emulator" diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index fa271f75..5be47190 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -12,8 +12,6 @@ import sys import time from pathlib import Path -from typing import Optional -from typing import Union from pydantic import validate_call from shepherd_core import CalibrationEmulator diff --git a/software/python-package/shepherd_sheep/target_io.py b/software/python-package/shepherd_sheep/target_io.py index 5e8c7d09..9d5b607a 100644 --- a/software/python-package/shepherd_sheep/target_io.py +++ b/software/python-package/shepherd_sheep/target_io.py @@ -31,8 +31,6 @@ :license: MIT, see LICENSE for more details. """ from contextlib import suppress -from typing import Dict -from typing import List from .logger import log diff --git a/software/python-package/tests/test_cli_misc.py b/software/python-package/tests/test_cli_misc.py index 26e33765..b35b70da 100644 --- a/software/python-package/tests/test_cli_misc.py +++ b/software/python-package/tests/test_cli_misc.py @@ -108,7 +108,7 @@ def test_cli_eeprom_read_min_arg_a( "read", ], ) - assert res.exit_code in [0, 2, 3] + assert res.exit_code in {0, 2, 3} @pytest.mark.hardware @@ -131,7 +131,7 @@ def test_cli_eeprom_read_min_arg_b( path_cal.as_posix(), ], ) - assert res.exit_code in [0, 2, 3] + assert res.exit_code in {0, 2, 3} @pytest.mark.hardware @@ -154,7 +154,7 @@ def test_cli_eeprom_read_explicit( path_cal.as_posix(), ], ) - assert res.exit_code in [0, 2, 3] + assert res.exit_code in {0, 2, 3} @pytest.mark.hardware diff --git a/software/shepherd-calibration/shepherd_cal/calibrator.py b/software/shepherd-calibration/shepherd_cal/calibrator.py index e8cb0230..74580dc5 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator.py @@ -1,9 +1,6 @@ #!/usr/bin/env python3 import time from pathlib import Path -from typing import Dict -from typing import Optional -from typing import Union import msgpack import msgpack_numpy diff --git a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py index 3109ee73..4d1275a3 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py @@ -1,7 +1,6 @@ from datetime import datetime from pathlib import Path from time import time -from typing import Optional import click import typer diff --git a/software/shepherd-calibration/shepherd_cal/logger.py b/software/shepherd-calibration/shepherd_cal/logger.py index 80a1c0ef..8bbd7b18 100644 --- a/software/shepherd-calibration/shepherd_cal/logger.py +++ b/software/shepherd-calibration/shepherd_cal/logger.py @@ -1,5 +1,4 @@ import logging -from typing import Union logger = logging.getLogger("shp.calTool") logger.setLevel(logging.INFO) diff --git a/software/shepherd-calibration/shepherd_cal/profile.py b/software/shepherd-calibration/shepherd_cal/profile.py index 3a56db85..cb3f1e4b 100644 --- a/software/shepherd-calibration/shepherd_cal/profile.py +++ b/software/shepherd-calibration/shepherd_cal/profile.py @@ -1,6 +1,4 @@ from pathlib import Path -from typing import Dict -from typing import List import numpy as np import pandas as pd diff --git a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py index 3e25ca2d..79f2d418 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py +++ b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py @@ -1,7 +1,5 @@ import os from pathlib import Path -from typing import List -from typing import Optional import pandas as pd diff --git a/software/shepherd-calibration/shepherd_cal/profile_calibration.py b/software/shepherd-calibration/shepherd_cal/profile_calibration.py index 90c86705..af5e485a 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_calibration.py +++ b/software/shepherd-calibration/shepherd_cal/profile_calibration.py @@ -1,8 +1,4 @@ -from typing import Dict -from typing import Optional -from typing import Tuple from typing import TypeVar -from typing import Union import numpy as np import pandas as pd diff --git a/software/shepherd-calibration/shepherd_cal/profiler.py b/software/shepherd-calibration/shepherd_cal/profiler.py index 19e286a0..e34e24a6 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler.py +++ b/software/shepherd-calibration/shepherd_cal/profiler.py @@ -10,8 +10,6 @@ """ import itertools import time -from typing import List -from typing import Tuple import msgpack import msgpack_numpy diff --git a/software/shepherd-calibration/shepherd_cal/profiler_cli.py b/software/shepherd-calibration/shepherd_cal/profiler_cli.py index 7713c696..2a3fde22 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler_cli.py +++ b/software/shepherd-calibration/shepherd_cal/profiler_cli.py @@ -1,8 +1,6 @@ from datetime import datetime from pathlib import Path from time import time -from typing import Dict -from typing import Optional import click import numpy as np diff --git a/software/shepherd-devicetest/src/__init__.py b/software/shepherd-devicetest/src/__init__.py index 26282c1a..0f102137 100644 --- a/software/shepherd-devicetest/src/__init__.py +++ b/software/shepherd-devicetest/src/__init__.py @@ -274,7 +274,7 @@ def assemble_window(): dpg.add_text(tag="text_B_gpio", default_value="PRU Input") dpg.add_spacer(width=16) dpg.add_input_text( - tag=f"gpio_output", + tag="gpio_output", default_value="0", readonly=True, label="", diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 688e1aaa..38c285e9 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -8,11 +8,6 @@ from datetime import timedelta from io import StringIO from pathlib import Path -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple -from typing import Union import yaml from fabric import Connection @@ -69,7 +64,7 @@ def __init__( hostnames = {hostname: hostname for hostname in hostlist} else: # look at all these directories for inventory-file - if inventory in ["", None]: + if inventory in {"", None}: inventories = [ "/etc/shepherd/herd.yml", "~/herd.yml", diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index 4971186b..f468dcde 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -2,7 +2,6 @@ import sys from datetime import datetime from pathlib import Path -from typing import Optional import click import shepherd_core diff --git a/software/shepherd-herd/shepherd_herd/logger.py b/software/shepherd-herd/shepherd_herd/logger.py index a1637c88..583608f7 100644 --- a/software/shepherd-herd/shepherd_herd/logger.py +++ b/software/shepherd-herd/shepherd_herd/logger.py @@ -1,5 +1,4 @@ import logging -from typing import Union from shepherd_core.logger import set_log_verbose_level From a905b31175b1c5ce5d3a89eadf63f889de9213be Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 15:09:33 +0200 Subject: [PATCH 10/35] py linting - part 1 --- .gitmodules | 1 - docs/conf.py | 1 - software/firmware/pru0-cython-module/setup.py | 8 +++-- .../firmware/pru0-cython-module/testing.py | 2 -- .../python-package/shepherd_sheep/__init__.py | 6 ++-- software/python-package/shepherd_sheep/cli.py | 8 ++--- .../python-package/shepherd_sheep/commons.py | 21 +++++++------- .../shepherd_sheep/h5_writer.py | 10 +++++-- .../python-package/shepherd_sheep/logger.py | 2 +- .../shepherd_sheep/monitor_abc.py | 4 +-- .../shepherd_sheep/monitor_kernel.py | 10 +++---- .../shepherd_sheep/monitor_ptp.py | 8 ++--- .../shepherd_sheep/monitor_sheep.py | 4 +-- .../shepherd_sheep/monitor_sysutil.py | 6 ++-- .../shepherd_sheep/shepherd_emulator.py | 17 ++++++----- .../shepherd_sheep/shepherd_harvester.py | 9 ++++-- .../shepherd_sheep/shepherd_io.py | 10 +++---- software/python-package/tests/conftest.py | 2 +- .../tests/test_cli_programmer.py | 2 +- .../python-package/tests/test_cli_run_task.py | 8 +++-- .../python-package/tests/test_datalogging.py | 3 +- software/python-package/tests/test_eeprom.py | 4 +-- .../python-package/tests/test_emulation.py | 3 +- .../tests/test_sysfs_interface.py | 2 +- .../tests/test_virtual_source.py | 9 ++---- .../python-package/tests_manual/__init__.py | 0 .../tests_manual/testbench_gen_fake.py | 3 +- .../shepherd_cal/calibrator_cli.py | 3 +- .../shepherd_cal/cli_helper.py | 2 +- .../shepherd_cal/profile_analyzer.py | 2 +- .../shepherd_cal/profiler_cli.py | 3 +- software/shepherd-devicetest/src/__init__.py | 12 ++++---- software/shepherd-herd/shepherd_herd/herd.py | 29 ++++++++++--------- .../shepherd-herd/shepherd_herd/herd_cli.py | 2 +- .../shepherd-herd/shepherd_herd/logger.py | 2 +- software/shepherd-herd/tests/conftest.py | 8 ++--- .../tests/test_cli_programmer.py | 2 +- 37 files changed, 122 insertions(+), 106 deletions(-) create mode 100644 software/python-package/tests_manual/__init__.py diff --git a/.gitmodules b/.gitmodules index fb789b10..97359710 100644 --- a/.gitmodules +++ b/.gitmodules @@ -14,4 +14,3 @@ # git submodule update --init --recursive # git submodule update --recursive --remote # git submodule foreach git reset --hard - diff --git a/docs/conf.py b/docs/conf.py index 32d432c2..2691eceb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -66,6 +66,5 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ["_static"] # noqa: E800 sitemap_url_scheme = "{link}" diff --git a/software/firmware/pru0-cython-module/setup.py b/software/firmware/pru0-cython-module/setup.py index 18c95ae5..d93b168e 100644 --- a/software/firmware/pru0-cython-module/setup.py +++ b/software/firmware/pru0-cython-module/setup.py @@ -1,5 +1,6 @@ import os import shutil +from pathlib import Path from Cython.Build import cythonize from setuptools import Extension @@ -11,10 +12,11 @@ "../pru0-shepherd-fw/calibration.c", "../pru0-shepherd-fw/math64_safe.c", ] -if not os.path.isdir("./build"): - os.makedirs("./build") +build_path = Path(__file__) / "build" +if not build_path.is_dir(): + build_path.mkdir(parents=True) for src in external_src: - shutil.copy(src, "./build/" + src.split("/")[-1]) + shutil.copy(src, build_path / src.split("/")[-1]) module_vconv = Extension( diff --git a/software/firmware/pru0-cython-module/testing.py b/software/firmware/pru0-cython-module/testing.py index 21a515a3..874a04a4 100644 --- a/software/firmware/pru0-cython-module/testing.py +++ b/software/firmware/pru0-cython-module/testing.py @@ -1,6 +1,4 @@ # import sys -# print(sys.path) -# sys.path.append('/home/vishal/shepherd/software/firmware/pru0-cython-module') # import pyximport from shepherd_core.data_models import VirtualSourceConfig from shepherd_core.data_models.content.virtual_source import ConverterPRUConfig diff --git a/software/python-package/shepherd_sheep/__init__.py b/software/python-package/shepherd_sheep/__init__.py index 9241d6d6..79cf8574 100644 --- a/software/python-package/shepherd_sheep/__init__.py +++ b/software/python-package/shepherd_sheep/__init__.py @@ -144,7 +144,7 @@ def run_programmer(cfg: ProgrammingTask) -> bool: file_tmp = Path(path_tmp.name) / "aligned.hex" # tmp_path because firmware can be in readonly content-dir cmd = [ - "srec_cat", + "/usr/bin/srec_cat", # BL51 hex files are not sorted for ascending addresses. Suppress this warning "-disable-sequence-warning", # load input HEX file @@ -167,13 +167,13 @@ def run_programmer(cfg: ProgrammingTask) -> bool: file_tmp.as_posix(), "-Intel", ] - ret = subprocess.run(cmd, timeout=30) # noqa: S607 S603 + ret = subprocess.run(cmd, timeout=30, check=False) # noqa: S603 if ret.returncode > 0: log.error("Error during realignment (srec_cat): %s", ret.stderr) failed = True raise SystemExit - with open(file_tmp.as_posix(), "rb") as fw: + with file_tmp.resolve().open("rb") as fw: try: dbg.shared_mem.write_firmware(fw.read()) target = cfg.mcu_type.lower() diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index d88f3a52..daf6911d 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -59,7 +59,7 @@ # - redone programmer, emulation -def exit_gracefully(*args): # type: ignore +def exit_gracefully(*_args): # type: ignore log.warning("Aborted!") sys.exit(0) @@ -180,7 +180,7 @@ def write( storage.write_calibration(cal_cape) except FileNotFoundError: log.error("Access to EEPROM failed (FS) -> is Shepherd-Cape missing?") - exit(2) + sys.exit(2) @eeprom.command(short_help="Read cape info and calibration data from EEPROM") @@ -201,10 +201,10 @@ def read(cal_file: Path | None): log.warning( "Reading from EEPROM failed (Val) -> no plausible data found", ) - exit(2) + sys.exit(2) except FileNotFoundError: log.error("Access to EEPROM failed (FS) -> is Shepherd-Cape missing?") - exit(3) + sys.exit(3) if cal_file is None: log.info("Retrieved Cal-Data:\n\n%s", str(cal)) diff --git a/software/python-package/shepherd_sheep/commons.py b/software/python-package/shepherd_sheep/commons.py index e0873127..34511863 100644 --- a/software/python-package/shepherd_sheep/commons.py +++ b/software/python-package/shepherd_sheep/commons.py @@ -39,17 +39,18 @@ MSG_DEP_ERR_NOFREEBUF = 0xE5 # fmt: off +# ruff: noqa: E241, E501 GPIO_LOG_BIT_POSITIONS = { - 0: {"pru_reg": "r31_00", "name": "tgt_gpio0", "bb_pin": "P8_45", "sys_pin": "P8_14", "sys_reg": "26"}, # noqa: E501 - 1: {"pru_reg": "r31_01", "name": "tgt_gpio1", "bb_pin": "P8_46", "sys_pin": "P8_17", "sys_reg": "27"}, # noqa: E501 - 2: {"pru_reg": "r31_02", "name": "tgt_gpio2", "bb_pin": "P8_43", "sys_pin": "P8_16", "sys_reg": "14"}, # noqa: E501 - 3: {"pru_reg": "r31_03", "name": "tgt_gpio3", "bb_pin": "P8_44", "sys_pin": "P8_15", "sys_reg": "15"}, # noqa: E501 - 4: {"pru_reg": "r31_04", "name": "tgt_gpio4", "bb_pin": "P8_41", "sys_pin": "P8_26", "sys_reg": "29"}, # noqa: E501 - 5: {"pru_reg": "r31_05", "name": "tgt_gpio5", "bb_pin": "P8_42", "sys_pin": "P8_36", "sys_reg": "16"}, # noqa: E501 - 6: {"pru_reg": "r31_06", "name": "tgt_gpio6", "bb_pin": "P8_39", "sys_pin": "P8_34", "sys_reg": "17"}, # noqa: E501 - 7: {"pru_reg": "r31_07", "name": "tgt_uart_rx", "bb_pin": "P8_40", "sys_pin": "P9_26", "sys_reg": "14"}, # noqa: E501 - 8: {"pru_reg": "r31_08", "name": "tgt_uart_tx", "bb_pin": "P8_27", "sys_pin": "P9_24", "sys_reg": "15"}, # noqa: E501 - 9: {"pru_reg": "r31_09", "name": "tgt_bat_ok", "bb_pin": "P8_29", "sys_pin": "", "sys_reg": ""}, # noqa: E501 + 0: {"pru_reg": "r31_00", "name": "tgt_gpio0", "bb_pin": "P8_45", "sys_pin": "P8_14", "sys_reg": "26"}, + 1: {"pru_reg": "r31_01", "name": "tgt_gpio1", "bb_pin": "P8_46", "sys_pin": "P8_17", "sys_reg": "27"}, + 2: {"pru_reg": "r31_02", "name": "tgt_gpio2", "bb_pin": "P8_43", "sys_pin": "P8_16", "sys_reg": "14"}, + 3: {"pru_reg": "r31_03", "name": "tgt_gpio3", "bb_pin": "P8_44", "sys_pin": "P8_15", "sys_reg": "15"}, + 4: {"pru_reg": "r31_04", "name": "tgt_gpio4", "bb_pin": "P8_41", "sys_pin": "P8_26", "sys_reg": "29"}, + 5: {"pru_reg": "r31_05", "name": "tgt_gpio5", "bb_pin": "P8_42", "sys_pin": "P8_36", "sys_reg": "16"}, + 6: {"pru_reg": "r31_06", "name": "tgt_gpio6", "bb_pin": "P8_39", "sys_pin": "P8_34", "sys_reg": "17"}, + 7: {"pru_reg": "r31_07", "name": "tgt_uart_rx", "bb_pin": "P8_40", "sys_pin": "P9_26", "sys_reg": "14"}, + 8: {"pru_reg": "r31_08", "name": "tgt_uart_tx", "bb_pin": "P8_27", "sys_pin": "P9_24", "sys_reg": "15"}, + 9: {"pru_reg": "r31_09", "name": "tgt_bat_ok", "bb_pin": "P8_29", "sys_pin": "", "sys_reg": ""}, } # Note: this table is copied (for hdf5-reference) from pru1/main.c, HW-Rev2.4b # Note: datalib has gpio-models + data! this lives now in diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index 20613c29..3d67d1f4 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -9,8 +9,13 @@ :license: MIT, see LICENSE for more details. """ from pathlib import Path +from typing import TYPE_CHECKING +from typing import ClassVar + +if TYPE_CHECKING: + import h5py + from .monitor_abc import Monitor -import h5py import numpy as np import yaml from shepherd_core import CalibrationEmulator as CalEmu @@ -23,7 +28,6 @@ from .commons import GPIO_LOG_BIT_POSITIONS from .commons import MAX_GPIO_EVT_PER_BUFFER -from .monitor_abc import Monitor from .monitor_kernel import KernelMonitor from .monitor_ptp import PTPMonitor from .monitor_sheep import SheepMonitor @@ -48,7 +52,7 @@ class Writer(CoreWriter): nanoseconds """ - mode_dtype_dict = { + mode_dtype_dict: ClassVar[dict[str, list]] = { "harvester": ["ivsample", "ivcurve", "isc_voc"], "emulator": ["ivsample"], } diff --git a/software/python-package/shepherd_sheep/logger.py b/software/python-package/shepherd_sheep/logger.py index 3d830e4a..52fbaff8 100644 --- a/software/python-package/shepherd_sheep/logger.py +++ b/software/python-package/shepherd_sheep/logger.py @@ -39,7 +39,7 @@ def set_verbosity(state: bool | int = True, temporary: bool = False) -> None: set_log_verbose_level(console_handler, 3) if temporary: return - global verbosity_state + global verbosity_state # noqa: PLW0603 verbosity_state = True diff --git a/software/python-package/shepherd_sheep/monitor_abc.py b/software/python-package/shepherd_sheep/monitor_abc.py index 59a3c19e..036f1f52 100644 --- a/software/python-package/shepherd_sheep/monitor_abc.py +++ b/software/python-package/shepherd_sheep/monitor_abc.py @@ -14,7 +14,7 @@ def __init__( target: h5py.Group, compression: Compression | None = Compression.default, poll_intervall: float = 0.25, - ): + ) -> None: self.data = target self.poll_intervall = poll_intervall self.position = 0 @@ -42,7 +42,7 @@ def __init__( type(self).__name__, ) - def __exit__(self, *exc): # type: ignore + def __exit__(self, *exc) -> None: # type: ignore self.data["time"].resize((self.position,)) log.info( "[%s] recorded %d events", diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index ecb0d5fd..b995074b 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -16,7 +16,7 @@ def __init__( target: h5py.Group, compression: Compression | None = Compression.default, backlog: int = 60, - ): + ) -> None: super().__init__(target, compression, poll_intervall=0.52) self.backlog = backlog @@ -30,14 +30,14 @@ def __init__( command = [ "sudo", - "journalctl", + "/usr/bin/journalctl", "--dmesg", "--follow", f"--lines={self.backlog}", "--output=short-precise", ] - self.process = subprocess.Popen( # noqa: S603 - command, + self.process = subprocess.Popen( + command, # noqa: S603 stdout=subprocess.PIPE, universal_newlines=True, ) @@ -49,7 +49,7 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc): # type: ignore + def __exit__(self, *exc) -> None: # type: ignore self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index ba5fbdb1..d0332ebb 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -15,7 +15,7 @@ def __init__( self, target: h5py.Group, compression: Compression | None = Compression.default, - ): + ) -> None: super().__init__(target, compression, poll_intervall=0.51) self.data.create_dataset( "values", @@ -37,8 +37,8 @@ def __init__( "--lines=60", "--output=short-precise", ] # for client - self.process = subprocess.Popen( # noqa: S603 - command, + self.process = subprocess.Popen( + command, # noqa: S603 stdout=subprocess.PIPE, universal_newlines=True, ) @@ -50,7 +50,7 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc): # type: ignore + def __exit__(self, *exc) -> None: # type: ignore self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_sheep.py b/software/python-package/shepherd_sheep/monitor_sheep.py index d19c2a41..3c6c8149 100644 --- a/software/python-package/shepherd_sheep/monitor_sheep.py +++ b/software/python-package/shepherd_sheep/monitor_sheep.py @@ -13,7 +13,7 @@ def __init__( self, target: h5py.Group, compression: Compression | None = Compression.default, - ): + ) -> None: super().__init__(target, compression, poll_intervall=0.25) self.queue = get_message_queue() self.data.create_dataset( @@ -38,7 +38,7 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc): # type: ignore + def __exit__(self, *exc) -> None: # type: ignore self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_sysutil.py b/software/python-package/shepherd_sheep/monitor_sysutil.py index bba8b1b8..54531fdf 100644 --- a/software/python-package/shepherd_sheep/monitor_sysutil.py +++ b/software/python-package/shepherd_sheep/monitor_sysutil.py @@ -15,7 +15,7 @@ def __init__( self, target: h5py.Group, compression: Compression | None = Compression.default, - ): + ) -> None: super().__init__(target, compression, poll_intervall=0.3) self.log_interval_ns: int = 1 * (10**9) # step-size is 1 s self.log_timestamp_ns: int = 0 @@ -70,7 +70,7 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc): # type: ignore + def __exit__(self, *exc) -> None: # type: ignore self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) @@ -81,7 +81,7 @@ def __exit__(self, *exc): # type: ignore self.data["net"].resize((self.position, 2)) super().__exit__() - def thread_fn(self, backlog: int = 40): + def thread_fn(self) -> None: """captures state of system in a fixed interval https://psutil.readthedocs.io/en/latest/#cpu :return: none diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index fa7265b7..12312ae1 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -7,6 +7,7 @@ from shepherd_core import CalibrationPair from shepherd_core import CalibrationSeries from shepherd_core import Reader as CoreReader +from shepherd_core import local_tz from shepherd_core.data_models import EnergyDType from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig from shepherd_core.data_models.content.virtual_source import ConverterPRUConfig @@ -115,7 +116,7 @@ def __init__( if cfg.output_path is not None: store_path = cfg.output_path.resolve() if store_path.is_dir(): - timestamp = datetime.fromtimestamp(self.start_time) + timestamp = datetime.fromtimestamp(self.start_time, tz=local_tz()) timestring = timestamp.strftime("%Y-%m-%d_%H-%M-%S") # ⤷ closest to ISO 8601, avoids ":" store_path = store_path / f"emu_{timestring}.h5" @@ -217,10 +218,11 @@ def run(self): try: idx, emu_buf = self.get_buffer(verbose=self.verbose_extra) except ShepherdIOException as e: - log.warning("Caught an Exception", exc_info=e) - if self.cfg.abort_on_error: - raise RuntimeError("Caught unforgivable ShepherdIO-Exception") + raise RuntimeError( + "Caught unforgivable ShepherdIO-Exception" + ) from e + log.warning("Caught an Exception", exc_info=e) continue if emu_buf.timestamp_ns / 1e9 >= ts_end: @@ -244,6 +246,7 @@ def run(self): # We're done when the PRU has processed all emulation data buffers if e.id_num == commons.MSG_DEP_ERR_NOFREEBUF: break - else: - if self.cfg.abort_on_error: - raise RuntimeError("Caught unforgivable ShepherdIO-Exception") + if self.cfg.abort_on_error: + raise RuntimeError( + "Caught unforgivable ShepherdIO-Exception" + ) from e diff --git a/software/python-package/shepherd_sheep/shepherd_harvester.py b/software/python-package/shepherd_sheep/shepherd_harvester.py index de61abe2..cd92f88f 100644 --- a/software/python-package/shepherd_sheep/shepherd_harvester.py +++ b/software/python-package/shepherd_sheep/shepherd_harvester.py @@ -4,6 +4,7 @@ import time from contextlib import ExitStack +from shepherd_core import local_tz from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig from shepherd_core.data_models.task import HarvestTask @@ -65,7 +66,7 @@ def __init__( store_path = cfg.output_path.resolve() if store_path.is_dir(): - timestamp = datetime.datetime.fromtimestamp(self.start_time) + timestamp = datetime.datetime.fromtimestamp(self.start_time, tz=local_tz()) timestring = timestamp.strftime("%Y-%m-%d_%H-%M-%S") # ⤷ closest to ISO 8601, avoids ":" store_path = store_path / f"hrv_{timestring}.h5" @@ -143,9 +144,11 @@ def run(self) -> None: try: idx, hrv_buf = self.get_buffer(verbose=self.verbose_extra) except ShepherdIOException as e: - log.warning("Caught an Exception", exc_info=e) if self.cfg.abort_on_error: - raise RuntimeError("Caught unforgivable ShepherdIO-Exception") + raise RuntimeError( + "Caught unforgivable ShepherdIO-Exception" + ) from e + log.warning("Caught an Exception", exc_info=e) continue if (hrv_buf.timestamp_ns / 1e9) >= ts_end: diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index d93b99da..7c12d5fe 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -98,7 +98,6 @@ def __init__( self.component = "emulator" self.gpios = {} - # self.shared_mem: Optional[SharedMem] = None # noqa: E800 self._buffer_period: float = 0.1 # placeholder value self.trace_iv = trace_iv @@ -171,7 +170,7 @@ def _get_msg(self, timeout_n: int = 5): for _ in range(timeout_n): try: return sfs.read_pru_msg() - except sfs.SysfsInterfaceException: + except sfs.SysfsInterfaceException: # noqa: PERF203 time.sleep(self._buffer_period) continue raise ShepherdIOException("Timeout waiting for message", ID_ERR_TIMEOUT) @@ -182,7 +181,7 @@ def _flush_msgs(): while True: try: sfs.read_pru_msg() - except sfs.SysfsInterfaceException: + except sfs.SysfsInterfaceException: # noqa: PERF203 break def start( @@ -196,7 +195,7 @@ def start( start_time (int): Desired start time in unix time wait_blocking (bool): If true, block until start has completed """ - if isinstance(start_time, (float, int)): + if isinstance(start_time, float | int): log.debug("asking kernel module for start at %.2f", start_time) sfs.set_start(start_time) if wait_blocking: @@ -211,7 +210,8 @@ def wait_for_start(timeout: float) -> None: """ sfs.wait_for_state("running", timeout) - def reinitialize_prus(self) -> None: + @staticmethod + def reinitialize_prus() -> None: sfs.set_stop(force=True) # forces idle sfs.wait_for_state("idle", 5) diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 1e4f79a1..92c081ce 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -8,7 +8,7 @@ def check_beagleboard() -> bool: - with suppress(Exception), open("/proc/cpuinfo") as info: + with suppress(Exception), Path("/proc/cpuinfo").open(encoding="utf-8-sig") as info: if "AM33XX" in info.read(): return True return False diff --git a/software/python-package/tests/test_cli_programmer.py b/software/python-package/tests/test_cli_programmer.py index d99e2c42..bbd610fe 100644 --- a/software/python-package/tests/test_cli_programmer.py +++ b/software/python-package/tests/test_cli_programmer.py @@ -24,7 +24,7 @@ def fw_msp() -> Path: @pytest.fixture def fw_empty(tmp_path: Path) -> Path: store_path = tmp_path / "firmware_null.hex" - with open(store_path.resolve(), "w") as f: + with store_path.resolve().open("w", encoding="utf-8-sig") as f: f.write("") return store_path diff --git a/software/python-package/tests/test_cli_run_task.py b/software/python-package/tests/test_cli_run_task.py index 244b078c..657d8ee6 100644 --- a/software/python-package/tests/test_cli_run_task.py +++ b/software/python-package/tests/test_cli_run_task.py @@ -18,6 +18,7 @@ import pytest from pydantic import ValidationError from shepherd_core import CalibrationHarvester +from shepherd_core import local_tz from shepherd_core.data_models import Firmware from shepherd_core.data_models import GpioTracing from shepherd_core.data_models import PowerTracing @@ -33,7 +34,8 @@ def random_data(length: int) -> np.ndarray: - return np.random.randint(0, high=2**18, size=length, dtype="u4") + rng = np.random.default_rng() + return rng.integers(low=0, high=2**18, size=length, dtype="u4") @pytest.fixture @@ -100,7 +102,7 @@ def test_cli_harvest_parameters_most( force_overwrite=True, duration=10, use_cal_default=True, - time_start=datetime.fromtimestamp(round(time.time() + 20)), + time_start=datetime.fromtimestamp(round(time.time() + 20), tz=local_tz()), abort_on_error=False, verbose=3, ).to_file(tmp_yaml) @@ -282,7 +284,7 @@ def test_cli_emulate_parameters_long( input_path=data_h5.as_posix(), output_path=path_h5.as_posix(), voltage_aux=2.5, - time_start=datetime.fromtimestamp(round(time.time() + 20)), + time_start=datetime.fromtimestamp(round(time.time() + 20), tz=local_tz()), use_cal_default=True, enable_io=True, io_port="B", diff --git a/software/python-package/tests/test_datalogging.py b/software/python-package/tests/test_datalogging.py index 7be8059a..a81a66de 100644 --- a/software/python-package/tests/test_datalogging.py +++ b/software/python-package/tests/test_datalogging.py @@ -14,7 +14,8 @@ def random_data(length: int) -> np.ndarray: - return np.random.randint(0, high=2**18, size=length, dtype="u4") + rng = np.random.default_rng() + return rng.integers(low=0, high=2**18, size=length, dtype="u4") @pytest.fixture() diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index e5e1591d..d464c645 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -94,7 +94,7 @@ def test_write_value(eeprom_retained, cape_data) -> None: def test_write_capedata(eeprom_retained, cape_data) -> None: eeprom_retained._write_cape_data(cape_data) for key, value in cape_data.items(): - if type(value) is str: + if isinstance(value, str): assert eeprom_retained[key] == value.rstrip("\0") else: assert eeprom_retained[key] == value @@ -102,7 +102,7 @@ def test_write_capedata(eeprom_retained, cape_data) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_read_capedata(eeprom_with_data, cape_data) -> None: +def test_read_capedata(eeprom_with_data, cape_data: CapeData) -> None: cape_data = eeprom_with_data._read_cape_data() for key in cape_data.keys(): assert cape_data[key] == cape_data[key] diff --git a/software/python-package/tests/test_emulation.py b/software/python-package/tests/test_emulation.py index a4c34455..6e1eeebb 100644 --- a/software/python-package/tests/test_emulation.py +++ b/software/python-package/tests/test_emulation.py @@ -20,7 +20,8 @@ def random_data(length) -> np.ndarray: - return np.random.randint(0, high=2**18, size=length, dtype="u4") + rng = np.random.default_rng() + return rng.integers(low=0, high=2**18, size=length, dtype="u4") @pytest.fixture diff --git a/software/python-package/tests/test_sysfs_interface.py b/software/python-package/tests/test_sysfs_interface.py index 2176584a..67515c08 100644 --- a/software/python-package/tests/test_sysfs_interface.py +++ b/software/python-package/tests/test_sysfs_interface.py @@ -147,7 +147,7 @@ def test_initial_calibration_settings(shepherd_up, cal4sysfs): @pytest.mark.hardware def test_initial_harvester_settings(shepherd_up): - hrv_list = [0] + list(range(200, 211)) + hrv_list = [0, *list(range(200, 211))] assert sysfs_interface.read_virtual_harvester_settings() == hrv_list diff --git a/software/python-package/tests/test_virtual_source.py b/software/python-package/tests/test_virtual_source.py index 444ce9ec..8c78fd80 100644 --- a/software/python-package/tests/test_virtual_source.py +++ b/software/python-package/tests/test_virtual_source.py @@ -12,10 +12,7 @@ @pytest.fixture def src_cfg(request) -> VirtualSourceConfig: marker = request.node.get_closest_marker("src_name") - if marker is None: - src_name = None - else: - src_name = marker.args[0] + src_name = None if marker is None else marker.args[0] if isinstance(src_name, str): if ".yaml" in src_name: @@ -200,7 +197,7 @@ def test_vsource_drain_charge( if (v_raw1 < 1) or (v_raw2 < 1): print( f"Stopped Drain-loop after {index}/{n_samples} samples " - f"({round(100*index/n_samples)} %), because output was disabled", + f"({round(100 * index / n_samples)} %), because output was disabled", ) break @@ -345,7 +342,7 @@ def test_vsource_diodecap( V_settle_mV = (V_inp_uV * 10**-3 - 300) / 2 # how many steps? charging took 9 steps at 200mA, so roughly 9 * 200 / (10 - 5) print( - f"DiodeCap Drain #### Inp = 5mA @ {V_inp_uV/10**3} mV , Out = 10mA " + f"DiodeCap Drain #### Inp = 5mA @ {V_inp_uV / 10**3} mV , Out = 10mA " f"-> V_out should settle @ {V_settle_mV} mV ", ) for _ in range(25): diff --git a/software/python-package/tests_manual/__init__.py b/software/python-package/tests_manual/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/software/python-package/tests_manual/testbench_gen_fake.py b/software/python-package/tests_manual/testbench_gen_fake.py index cdf8ece7..a278fb77 100644 --- a/software/python-package/tests_manual/testbench_gen_fake.py +++ b/software/python-package/tests_manual/testbench_gen_fake.py @@ -11,7 +11,8 @@ def random_data(length): - return np.random.randint(0, high=2**18, size=length, dtype="u4") + rng = np.random.default_rng() + return rng.integers(low=0, high=2**18, size=length, dtype="u4") with Writer(store_path, cal_data=CalibrationHarvester()) as store: diff --git a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py index 4d1275a3..9294eb04 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py @@ -4,6 +4,7 @@ import click import typer +from shepherd_core import local_tz from shepherd_core.data_models.base.cal_measurement import CalMeasurementCape from shepherd_core.data_models.base.calibration import CapeData @@ -98,7 +99,7 @@ def measure( msr_cape = CalMeasurementCape(**results) if outfile is None: - timestamp = datetime.fromtimestamp(time()) + timestamp = datetime.fromtimestamp(time(), tz=local_tz()) timestring = timestamp.strftime("%Y-%m-%d_%H-%M") outfile = Path(f"./{timestring}_shepherd_cape_{cape_serial}.measurement.yaml") logger.debug("No filename provided -> set to '%s'.", outfile) diff --git a/software/shepherd-calibration/shepherd_cal/cli_helper.py b/software/shepherd-calibration/shepherd_cal/cli_helper.py index 55a9158c..1c5fadf1 100644 --- a/software/shepherd-calibration/shepherd_cal/cli_helper.py +++ b/software/shepherd-calibration/shepherd_cal/cli_helper.py @@ -10,7 +10,7 @@ from .logger import set_verbosity -def exit_gracefully(*args): # type: ignore +def exit_gracefully(*_args): # type: ignore logger.warning("Aborted!") sys.exit(0) diff --git a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py index 79f2d418..5c04b37c 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py +++ b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py @@ -35,7 +35,7 @@ def analyze_directory( for file in files: fpath = Path(file) - if not os.path.isfile(file): + if not fpath.is_file(): continue if "npz" not in fpath.suffix.lower(): continue diff --git a/software/shepherd-calibration/shepherd_cal/profiler_cli.py b/software/shepherd-calibration/shepherd_cal/profiler_cli.py index 2a3fde22..2b5b4fc4 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler_cli.py +++ b/software/shepherd-calibration/shepherd_cal/profiler_cli.py @@ -5,6 +5,7 @@ import click import numpy as np import typer +from shepherd_core import local_tz from .calibrator import INSTR_4WIRE from .calibrator import Calibrator @@ -74,7 +75,7 @@ def measure( time_now = time() components = ("_emu" if emulator else "") + ("_hrv" if harvester else "") if outfile is None: - timestamp = datetime.fromtimestamp(time_now) + timestamp = datetime.fromtimestamp(time_now, tz=local_tz()) timestring = timestamp.strftime("%Y-%m-%d_%H-%M") outfile = Path(f"./{timestring}_shepherd_cape_{cape_serial}") if short: diff --git a/software/shepherd-devicetest/src/__init__.py b/software/shepherd-devicetest/src/__init__.py index 0f102137..55f8d4f7 100644 --- a/software/shepherd-devicetest/src/__init__.py +++ b/software/shepherd-devicetest/src/__init__.py @@ -85,7 +85,7 @@ def assemble_window(): dpg.add_radio_button( tag="target_pwr", items=[*tgt_dict], - default_value=[*tgt_dict][0], + default_value=next(iter(tgt_dict)), callback=target_power_callback, show=True, ) @@ -98,7 +98,7 @@ def assemble_window(): dpg.add_radio_button( tag="target_io", items=[*tgt_dict], - default_value=[*tgt_dict][0], + default_value=next(iter(tgt_dict)), callback=target_io_callback, show=True, ) @@ -107,7 +107,7 @@ def assemble_window(): dpg.add_radio_button( tag="io_lvl_converter", items=[*able_dict], - default_value=[*able_dict][0], + default_value=next(iter(able_dict)), callback=io_level_converter_callback, show=True, ) @@ -126,7 +126,8 @@ def assemble_window(): ) with dpg.tooltip("gpio_nRes_REC_ADC"): dpg.add_text( - "Option to reset this ADC - it has to be reinitialized afterwards (with PRU re-init)", + "Option to reset this ADC - " + "it has to be reinitialized afterwards (with PRU re-init)", ) dpg.add_spacer(width=5) @@ -138,7 +139,8 @@ def assemble_window(): ) with dpg.tooltip("gpio_nRes_EMU_ADC"): dpg.add_text( - "Option to reset this ADC - it has to be configured afterwards (with PRU re-init)", + "Option to reset this ADC - " + "it has to be configured afterwards (with PRU re-init)", ) dpg.add_spacer(width=15) diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 38c285e9..44468e5e 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -8,6 +8,7 @@ from datetime import timedelta from io import StringIO from pathlib import Path +from typing import ClassVar import yaml from fabric import Connection @@ -17,6 +18,7 @@ from paramiko.ssh_exception import SSHException from pydantic import validate_call from shepherd_core import Inventory +from shepherd_core import local_tz from shepherd_core import tb_client from shepherd_core.data_models import ShpModel from shepherd_core.data_models import Wrapper @@ -28,13 +30,13 @@ class Herd: - _remote_paths_allowed = [ - Path("/var/shepherd/recordings/"), # default + path_default = Path("/var/shepherd/recordings/") + _remote_paths_allowed: ClassVar[set] = { + path_default, # default Path("/var/shepherd/"), Path("/etc/shepherd/"), Path("/tmp/"), # noqa: S108 - ] - path_default = _remote_paths_allowed[0] + } timestamp_diff_allowed = 10 start_delay_s = 30 @@ -80,13 +82,13 @@ def __init__( if host_path is None: raise FileNotFoundError(", ".join(inventories)) - with open(host_path) as stream: + with host_path.open(encoding="utf-8-sig") as stream: try: inventory_data = yaml.safe_load(stream) - except yaml.YAMLError: + except yaml.YAMLError as _xpt: raise FileNotFoundError( f"Couldn't read inventory file {host_path}, please provide a valid one", - ) + ) from _xpt logger.info("Shepherd-Inventory = '%s'", host_path.as_posix()) hostlist = [] @@ -256,7 +258,7 @@ def _thread_put( filename = src.name src = str(src) - if dst.suffix == "" and not str(dst).endswith("/"): + if not dst.suffix and not str(dst).endswith("/"): dst = str(dst) + "/" if not cnx.is_connected: @@ -347,10 +349,9 @@ def get_file( dst_paths = {} # assemble file-names - if Path(src).is_absolute(): - src_path = Path(src) - else: - src_path = Path(self.path_default) / src + src_path = ( + Path(src) if Path(src).is_absolute() else Path(self.path_default) / src + ) for i, cnx in enumerate(self.group): hostname = self.hostnames[cnx.host] @@ -459,7 +460,7 @@ def put_task( task_dict = task.model_dump(exclude_unset=True) task_wrap = Wrapper( datatype=type(task).__name__, - created=datetime.now(), + created=datetime.now(tz=local_tz()), parameters=task_dict, ) task_yaml = yaml.safe_dump( @@ -471,7 +472,7 @@ def put_task( elif isinstance(task, Path): if not task.is_file() or not task.exists(): raise ValueError("Task-Path must be existing file") - with open(task) as stream: + with task.open(encoding="utf-8-sig") as stream: task_yaml = yaml.safe_load(stream) else: raise ValueError("Task must either be model or path to a model") diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index f468dcde..fbb97416 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -23,7 +23,7 @@ # - arguments can be configured in a dict and standardized across tools -def exit_gracefully(*args) -> None: # type: ignore +def exit_gracefully(*_args) -> None: # type: ignore log.warning("Aborted!") sys.exit(0) diff --git a/software/shepherd-herd/shepherd_herd/logger.py b/software/shepherd-herd/shepherd_herd/logger.py index 583608f7..e00a3261 100644 --- a/software/shepherd-herd/shepherd_herd/logger.py +++ b/software/shepherd-herd/shepherd_herd/logger.py @@ -22,5 +22,5 @@ def set_verbosity(state: bool | int = True) -> None: elif isinstance(state, int) and state < 3: return # old format, will be replaced set_log_verbose_level(logger, 3) - global verbosity_state + global verbosity_state # noqa: PLW0603 verbosity_state = True diff --git a/software/shepherd-herd/tests/conftest.py b/software/shepherd-herd/tests/conftest.py index ad03deb1..03529a3e 100644 --- a/software/shepherd-herd/tests/conftest.py +++ b/software/shepherd-herd/tests/conftest.py @@ -16,12 +16,12 @@ def cli_runner() -> CliRunner: def extract_first_sheep(herd_path: Path) -> str: - with open(herd_path) as stream: + with herd_path.open(encoding="utf-8-sig") as stream: try: inventory_data = yaml.safe_load(stream) - except yaml.YAMLError: - raise TypeError(f"Couldn't read inventory file {herd_path}") - return list(inventory_data["sheep"]["hosts"].keys())[0] + except yaml.YAMLError as _xpt: + raise TypeError(f"Couldn't read inventory file {herd_path}") from _xpt + return next(iter(inventory_data["sheep"]["hosts"].keys())) def wait_for_end(cli_runner: CliRunner, tmin: float = 0, timeout: float = 999) -> bool: diff --git a/software/shepherd-herd/tests/test_cli_programmer.py b/software/shepherd-herd/tests/test_cli_programmer.py index 908cb0b6..eb15d075 100644 --- a/software/shepherd-herd/tests/test_cli_programmer.py +++ b/software/shepherd-herd/tests/test_cli_programmer.py @@ -18,7 +18,7 @@ def fw_example() -> Path: @pytest.fixture def fw_empty(tmp_path: Path) -> Path: store_path = tmp_path / "firmware_null.hex" - with open(store_path, "w") as f: + with store_path.open("w", encoding="utf-8-sig") as f: f.write("") return store_path From a0a43be736ee435e085deaf8ca87f11425f2b872 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 15:22:06 +0200 Subject: [PATCH 11/35] simplify program logic --- .../python-package/shepherd_sheep/eeprom.py | 35 +++++++++---------- .../python-package/shepherd_sheep/launcher.py | 2 +- .../shepherd_sheep/shared_memory.py | 3 +- .../shepherd_sheep/shepherd_debug.py | 13 +++---- .../shepherd_sheep/shepherd_emulator.py | 3 +- .../shepherd_sheep/shepherd_io.py | 31 ++++++++-------- .../shepherd_sheep/target_io.py | 11 +++--- .../tests/test_sysfs_interface.py | 6 ++-- .../tests/test_virtual_source.py | 9 ++--- software/shepherd-herd/shepherd_herd/herd.py | 15 ++++---- 10 files changed, 56 insertions(+), 72 deletions(-) diff --git a/software/python-package/shepherd_sheep/eeprom.py b/software/python-package/shepherd_sheep/eeprom.py index cfb1d70b..bda95fc6 100644 --- a/software/python-package/shepherd_sheep/eeprom.py +++ b/software/python-package/shepherd_sheep/eeprom.py @@ -112,8 +112,7 @@ def __getitem__(self, key: str): if eeprom_format[key]["type"] == "str": str_data = raw_data.split(b"\x00") return str_data[0].decode("utf-8") - else: - return raw_data + return raw_data def __setitem__(self, key: str, value): # type: ignore """Writes attribute to EEPROM. @@ -213,19 +212,19 @@ def read_calibration(self) -> CalibrationCape: def retrieve_calibration(use_default_cal: bool = False) -> CalibrationCape: if use_default_cal: return CalibrationCape() - else: - try: - with EEPROM() as storage: - return storage.read_calibration() - except ValueError: - log.warning( - "Couldn't read calibration from EEPROM (ValueError). " - "Falling back to default values.", - ) - return CalibrationCape() - except FileNotFoundError: - log.warning( - "Couldn't read calibration from EEPROM (FileNotFoundError). " - "Falling back to default values.", - ) - return CalibrationCape() + + try: + with EEPROM() as storage: + return storage.read_calibration() + except ValueError: + log.warning( + "Couldn't read calibration from EEPROM (ValueError). " + "Falling back to default values.", + ) + return CalibrationCape() + except FileNotFoundError: + log.warning( + "Couldn't read calibration from EEPROM (FileNotFoundError). " + "Falling back to default values.", + ) + return CalibrationCape() diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index 97de6f73..a4ce9e21 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -142,7 +142,7 @@ def get_state(self, timeout: float = 10) -> bool: if systemd_state == "active": return True - elif systemd_state == "inactive": + if systemd_state == "inactive": return False raise Exception(f"Unknown state { systemd_state }") diff --git a/software/python-package/shepherd_sheep/shared_memory.py b/software/python-package/shepherd_sheep/shared_memory.py index 7f23a84e..593600f8 100644 --- a/software/python-package/shepherd_sheep/shared_memory.py +++ b/software/python-package/shepherd_sheep/shared_memory.py @@ -170,8 +170,7 @@ def __exit__(self, *args): # type: ignore def timedelta_to_ns(delta: timedelta | None, default_s: int = 0) -> int: if isinstance(delta, timedelta): return int(delta.total_seconds() * 10**9) - else: - return int(timedelta(seconds=default_s).total_seconds() * 10**9) + return int(timedelta(seconds=default_s).total_seconds() * 10**9) def config_tracers(self, timestamp_ns: int) -> None: if self.trace_iv is not None: diff --git a/software/python-package/shepherd_sheep/shepherd_debug.py b/software/python-package/shepherd_sheep/shepherd_debug.py index 1899e72d..c901ac45 100644 --- a/software/python-package/shepherd_sheep/shepherd_debug.py +++ b/software/python-package/shepherd_sheep/shepherd_debug.py @@ -325,8 +325,7 @@ def set_gpio_one_high(self, num: int) -> None: def get_gpio_state(self, num: int) -> bool: if self._io is not None: return self._io.get_pin_state(num) - else: - log.debug("Error: IO is not enabled in this shepherd-debug-instance") + log.debug("Error: IO is not enabled in this shepherd-debug-instance") return False def set_gpio_direction(self, num: int, pdir: bool) -> None: @@ -338,16 +337,14 @@ def set_gpio_direction(self, num: int, pdir: bool) -> None: def get_gpio_direction(self, num: int) -> bool: if self._io is not None: return self._io.get_pin_direction(num) - else: - log.debug("Error: IO is not enabled in this shepherd-debug-instance") - return True + log.debug("Error: IO is not enabled in this shepherd-debug-instance") + return True def get_gpio_info(self) -> list: if self._io is not None: return self._io.pin_names - else: - log.debug("Error: IO is not enabled in this shepherd-debug-instance") - return [] + log.debug("Error: IO is not enabled in this shepherd-debug-instance") + return [] def set_power_state_emulator(self, state: bool) -> None: super().set_power_state_emulator(state) diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index 12312ae1..0ba5973d 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -60,8 +60,7 @@ def __init__( msg = f"Input-File has wrong mode ({self.reader.get_mode()} != harvester)" if self.cfg.abort_on_error: raise ValueError(msg) - else: - log.error(msg) + log.error(msg) if not self.reader.is_valid() and self.cfg.abort_on_error: raise RuntimeError("Input-File is not valid!") diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 7c12d5fe..44ed2844 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -64,16 +64,15 @@ class ShepherdIO: _instance = None @classmethod - def __new__(cls, *args, **kwds): + def __new__(cls, *_args, **_kwds): """Implements singleton class.""" if ShepherdIO._instance is None: new_class = object.__new__(cls) ShepherdIO._instance = new_class # was raising on reuse and stored weakref.ref before return new_class - else: - log.debug("ShepherdIO-Singleton reused") - return ShepherdIO._instance + log.debug("ShepherdIO-Singleton reused") + return ShepherdIO._instance def __init__( self, @@ -332,11 +331,11 @@ def set_power_state_emulator(self, state: bool) -> None: def convert_target_port_to_bool(target: TargetPort | str | bool | None) -> bool: if target is None: return True - elif isinstance(target, str): + if isinstance(target, str): return TargetPort[target] == TargetPort.A - elif isinstance(target, TargetPort): + if isinstance(target, TargetPort): return target == TargetPort.A - elif isinstance(target, bool): + if isinstance(target, bool): return target raise ValueError( f"Parameter 'target' must be A or B (was {target}, type {type(target)})", @@ -521,31 +520,31 @@ def get_buffer(self, timeout_n: int = 10, verbose: bool = False): ) return value, buf - elif msg_type == commons.MSG_DBG_PRINT: + if msg_type == commons.MSG_DBG_PRINT: log.info("Received cmd to print: %d", value) continue - elif msg_type == commons.MSG_DEP_ERR_INCMPLT: + if msg_type == commons.MSG_DEP_ERR_INCMPLT: raise ShepherdIOException( "Got incomplete buffer", commons.MSG_DEP_ERR_INCMPLT, value, ) - elif msg_type == commons.MSG_DEP_ERR_INVLDCMD: + if msg_type == commons.MSG_DEP_ERR_INVLDCMD: raise ShepherdIOException( "PRU received invalid command", commons.MSG_DEP_ERR_INVLDCMD, value, ) - elif msg_type == commons.MSG_DEP_ERR_NOFREEBUF: + if msg_type == commons.MSG_DEP_ERR_NOFREEBUF: raise ShepherdIOException( "PRU ran out of buffers", commons.MSG_DEP_ERR_NOFREEBUF, value, ) - else: - raise ShepherdIOException( - f"Expected msg type { commons.MSG_BUF_FROM_PRU } " - f"got { msg_type }[{ value }]", - ) + + raise ShepherdIOException( + f"Expected msg type { commons.MSG_BUF_FROM_PRU } " + f"got { msg_type }[{ value }]", + ) diff --git a/software/python-package/shepherd_sheep/target_io.py b/software/python-package/shepherd_sheep/target_io.py index 9d5b607a..c65d47e6 100644 --- a/software/python-package/shepherd_sheep/target_io.py +++ b/software/python-package/shepherd_sheep/target_io.py @@ -126,13 +126,12 @@ def get_pin_direction(self, num: int) -> bool: dir_param = target_pins[num]["dir"] if isinstance(dir_param, str): return dir_param == "I" - elif isinstance(dir_param, int): + if isinstance(dir_param, int): dir_pin = self.dirs[dir_param] return not dir_pin.read() - else: - raise RuntimeError( - "Something went wrong - could not determine pin-direction", - ) + raise RuntimeError( + "Something went wrong - could not determine pin-direction", + ) def set_pin_direction(self, num: int, pdir: bool) -> bool: """ @@ -148,7 +147,7 @@ def set_pin_direction(self, num: int, pdir: bool) -> bool: # not changeable pin_state = dir_param == "I" return pin_state == pdir - elif isinstance(dir_param, int): + if isinstance(dir_param, int): pins_affected = [ pin["name"] for pin in target_pins if pin["dir"] == dir_param ] diff --git a/software/python-package/tests/test_sysfs_interface.py b/software/python-package/tests/test_sysfs_interface.py index 67515c08..37c2cd0a 100644 --- a/software/python-package/tests/test_sysfs_interface.py +++ b/software/python-package/tests/test_sysfs_interface.py @@ -18,16 +18,14 @@ def cnv_cfg() -> ConverterPRUConfig: name = "_test_config_virtsource.yaml" path = here.parent / name src_cfg = VirtualSourceConfig.from_file(path) - cnv_pru = ConverterPRUConfig.from_vsrc(src_cfg, log_intermediate_node=False) - return cnv_pru + return ConverterPRUConfig.from_vsrc(src_cfg, log_intermediate_node=False) @pytest.fixture def hrv_cfg() -> HarvesterPRUConfig: path = Path(__file__).parent / "_test_config_harvest.yaml" hrv_cfg = HarvestTask.from_file(path.as_posix()) - hrv_pru = HarvesterPRUConfig.from_vhrv(hrv_cfg.virtual_harvester) - return hrv_pru + return HarvesterPRUConfig.from_vhrv(hrv_cfg.virtual_harvester) @pytest.fixture() diff --git a/software/python-package/tests/test_virtual_source.py b/software/python-package/tests/test_virtual_source.py index 8c78fd80..7f183680 100644 --- a/software/python-package/tests/test_virtual_source.py +++ b/software/python-package/tests/test_virtual_source.py @@ -22,10 +22,8 @@ def src_cfg(request) -> VirtualSourceConfig: here = Path(__file__).resolve() path = here.parent / src_name return VirtualSourceConfig.from_file(path) - else: - return VirtualSourceConfig(name=src_name) - else: - assert 0 + return VirtualSourceConfig(name=src_name) + raise AssertionError @pytest.fixture @@ -74,7 +72,7 @@ def pyt_vsource( @pytest.fixture def reference_vss() -> dict: # keep in sync with "_test_config_virtsource.yaml" - vss = { + return { "C_intermediate_uF": 100 * (10**0), "V_intermediate_init_mV": 3000, "eta_in": 0.5, @@ -84,7 +82,6 @@ def reference_vss() -> dict: "V_output_mV": 2000, "t_sample_s": 10 * (10**-6), } - return vss def difference_percent(val1: float, val2: float, offset: float) -> float: diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 44468e5e..70bac8de 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -525,10 +525,10 @@ def start_measurement(self) -> int: if self.check_status(warn=True): logger.info("-> won't start while shepherd-instances are active") return 1 - else: - replies = self.run_cmd(sudo=True, cmd="systemctl start shepherd") - self.print_output(replies) - return max([reply.exited for reply in replies.values()]) + + replies = self.run_cmd(sudo=True, cmd="systemctl start shepherd") + self.print_output(replies) + return max([reply.exited for reply in replies.values()]) def stop_measurement(self) -> int: logger.debug("Shepherd-nodes affected: %s", self.hostnames.values()) @@ -548,8 +548,7 @@ def poweroff(self, restart: bool) -> int: else: replies = self.run_cmd(sudo=True, cmd="poweroff") logger.info("Command for powering off nodes was issued") - exit_code = max([reply.exited for reply in replies.values()]) - return exit_code + return max([reply.exited for reply in replies.values()]) @validate_call def await_stop(self, timeout: int = 30) -> bool: @@ -580,7 +579,7 @@ def inventorize(self, output_path: Path) -> bool: path=Path(output_path) / "inventory_server.yaml", minimal=True, ) - failed = self.get_file( + return self.get_file( file_path, output_path, timestamp=False, @@ -588,7 +587,6 @@ def inventorize(self, output_path: Path) -> bool: delete_src=True, ) # TODO: best case - add all to one file or a new inventories-model? - return failed @validate_call def run_task(self, config: Path | ShpModel, attach: bool = False) -> int: @@ -637,4 +635,3 @@ def get_task_files( logger.info("Remote path of '%s' is: %s, WON'T COPY", host, path) raise RuntimeError("FN not finished, not needed ATM") # TODO return failed - pass From a27c6098af727e76c8fb1fc4c1d65fb54f6c69d7 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 15:23:12 +0200 Subject: [PATCH 12/35] Update sysfs_interface.py --- .../shepherd_sheep/sysfs_interface.py | 175 ++++++++++-------- 1 file changed, 94 insertions(+), 81 deletions(-) diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index 5be47190..5b9e1a45 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -53,33 +53,46 @@ def flatten_list(dl: list) -> list: if len(dl) == 1: if isinstance(dl[0], list): return flatten_list(dl[0]) - else: - return dl - elif isinstance(dl[0], list): + return dl + if isinstance(dl[0], list): return flatten_list(dl[0]) + flatten_list(dl[1:]) - else: - return [dl[0]] + flatten_list(dl[1:]) - else: - return [dl] + return [dl[0], *flatten_list(dl[1:])] + return [dl] -def load_kernel_module(): - subprocess.run(["modprobe", "-a", "shepherd"], timeout=60) # noqa: S607 S603 - log.debug("activated shepherd kernel module") - time.sleep(3) +def load_kernel_module() -> None: + _try = 6 + while _try > 0: + ret = subprocess.run( + ["/usr/sbin/modprobe", "-a", "shepherd"], # noqa: S603 + timeout=60, + check=False, + ).returncode + if ret == 0: + log.debug("Activated shepherd kernel module") + time.sleep(3) + return + _try -= 1 + time.sleep(1) + raise SystemError("Failed to load shepherd kernel module.") -def remove_kernel_module(): - ret = 1 - while ret > 0: - ret = subprocess.run( # noqa: S607 S603 - ["modprobe", "-rf", "shepherd"], +def remove_kernel_module() -> None: + _try = 6 + while _try > 0: + ret = subprocess.run( + ["/usr/sbin/modprobe", "-rf", "shepherd"], # noqa: S603 timeout=60, capture_output=True, + check=False, ).returncode + if ret == 0: + log.debug("Deactivated shepherd kernel module") + time.sleep(1) + return + _try -= 1 time.sleep(1) - log.debug("deactivated shepherd kernel module") - time.sleep(1) + raise SystemError("Failed to unload shepherd kernel module.") def reload_kernel_module(): @@ -142,7 +155,7 @@ def wait_for_state(wanted_state: str, timeout: float) -> float: time.sleep(0.1) -def set_start(start_time: float | int | None = None) -> None: +def set_start(start_time: float | int | None = None) -> None: # noqa: PYI041 """Starts shepherd. Writes 'start' to the 'state' sysfs attribute in order to transition from @@ -157,7 +170,7 @@ def set_start(start_time: float | int | None = None) -> None: if current_state != "idle": raise SysfsInterfaceException(f"Cannot start from state { current_state }") - with open(sysfs_path / "state", "w") as f: + with (sysfs_path / "state").open("w", encoding="utf-8") as f: if isinstance(start_time, float): start_time = int(start_time) if isinstance(start_time, int): @@ -179,7 +192,7 @@ def set_stop(force: bool = False) -> None: if current_state != "running": raise SysfsInterfaceException(f"Cannot stop from state { current_state }") - with open(sysfs_path / "state", "w") as f: + with (sysfs_path / "state").open("w", encoding="utf-8") as f: f.write("stop") @@ -197,14 +210,13 @@ def write_mode(mode: str, force: bool = False) -> None: if force: set_stop(force=True) wait_for_state("idle", 5) - else: - if get_state() != "idle": - raise SysfsInterfaceException( - f"Cannot set mode when shepherd is { get_state() }", - ) + elif get_state() != "idle": + raise SysfsInterfaceException( + f"Cannot set mode when shepherd is { get_state() }", + ) log.debug("sysfs/mode: '%s'", mode) - with open(sysfs_path / "mode", "w") as f: + with (sysfs_path / "mode").open("w", encoding="utf-8") as f: f.write(mode) @@ -271,7 +283,7 @@ def write_dac_aux_voltage_raw( voltage_raw = min(voltage_raw, 2**16 - 1) voltage_raw |= int(ch_link) << 20 voltage_raw |= int(cap_out) << 21 - with open(sysfs_path / "dac_auxiliary_voltage_raw", "w") as f: + with (sysfs_path / "dac_auxiliary_voltage_raw").open("w", encoding="utf-8") as f: log.debug("Sending raw auxiliary voltage (dac channel B): %d", voltage_raw) f.write(str(voltage_raw)) @@ -288,8 +300,7 @@ def read_dac_aux_voltage(cal_emu: CalibrationEmulator | None = None) -> float: value_raw = read_dac_aux_voltage_raw() if not cal_emu: cal_emu = CalibrationEmulator() - voltage = cal_emu.dac_V_A.raw_to_si(value_raw) - return voltage + return cal_emu.dac_V_A.raw_to_si(value_raw) def read_dac_aux_voltage_raw() -> int: @@ -299,7 +310,7 @@ def read_dac_aux_voltage_raw() -> int: Returns: voltage as dac_raw """ - with open(sysfs_path / "dac_auxiliary_voltage_raw") as f: + with (sysfs_path / "dac_auxiliary_voltage_raw").open(encoding="utf-8") as f: settings = f.read().rstrip() int_settings = [int(x) for x in settings.split()] @@ -328,7 +339,7 @@ def write_calibration_settings( ) wait_for_state("idle", 3.0) - with open(sysfs_path / "calibration_settings", "w") as f: + with (sysfs_path / "calibration_settings").open("w", encoding="utf-8") as f: output = ( f"{int(cal_pru['adc_current_gain'])} {int(cal_pru['adc_current_offset'])} \n" f"{int(cal_pru['adc_voltage_gain'])} {int(cal_pru['adc_voltage_offset'])} \n" @@ -346,11 +357,11 @@ def read_calibration_settings() -> ( The virtual-source algorithms use adc measurements and dac-output """ - with open(sysfs_path / "calibration_settings") as f: + with (sysfs_path / "calibration_settings").open(encoding="utf-8") as f: settings = f.read().rstrip() int_settings = [int(x) for x in settings.split()] - cal_pru = { + return { "adc_current_gain": int_settings[0], "adc_current_offset": int_settings[1], "adc_voltage_gain": int_settings[2], @@ -358,7 +369,6 @@ def read_calibration_settings() -> ( "dac_voltage_gain": int_settings[4], "dac_voltage_offset": int_settings[5], } - return cal_pru @validate_call @@ -379,16 +389,18 @@ def write_virtual_converter_settings(settings: ConverterPRUConfig) -> None: if isinstance(setting, int): output += f"{setting} \n" elif isinstance(setting, list): - setting = flatten_list(setting) - setting = [str(i) for i in setting] - output += " ".join(setting) + " \n" + _set = flatten_list(setting) + _set = [str(i) for i in _set] + output += " ".join(_set) + " \n" else: raise SysfsInterfaceException( f"virtual-converter value {setting} has wrong type ({type(setting)})", ) wait_for_state("idle", 3.0) - with open(sysfs_path / "virtual_converter_settings", "w") as file: + with (sysfs_path / "virtual_converter_settings").open( + "w", encoding="utf-8" + ) as file: file.write(output) @@ -398,10 +410,9 @@ def read_virtual_converter_settings() -> list: The pru-algorithm uses these settings to configure emulator. """ - with open(sysfs_path / "virtual_converter_settings") as f: + with (sysfs_path / "virtual_converter_settings").open(encoding="utf-8") as f: settings = f.read().rstrip() - int_settings = [int(x) for x in settings.split()] - return int_settings + return [int(x) for x in settings.split()] @validate_call @@ -426,7 +437,9 @@ def write_virtual_harvester_settings(settings: HarvesterPRUConfig) -> None: ) wait_for_state("idle", 3.0) - with open(sysfs_path / "virtual_harvester_settings", "w") as file: + with (sysfs_path / "virtual_harvester_settings").open( + "w", encoding="utf-8" + ) as file: file.write(output) @@ -436,13 +449,12 @@ def read_virtual_harvester_settings() -> list: The pru-algorithm uses these settings to configure emulator. """ - with open(sysfs_path / "virtual_harvester_settings") as f: + with (sysfs_path / "virtual_harvester_settings").open(encoding="utf-8") as f: settings = f.read().rstrip() - int_settings = [int(x) for x in settings.split()] - return int_settings + return [int(x) for x in settings.split()] -def write_pru_msg(msg_type: int, values: list | float | int) -> None: +def write_pru_msg(msg_type: int, values: list | float | int) -> None: # noqa: PYI041 """ :param msg_type: :param values: @@ -454,7 +466,7 @@ def write_pru_msg(msg_type: int, values: list | float | int) -> None: f"and content (={msg_type})", ) - if isinstance(values, (int, float)): + if isinstance(values, int | float): # catch all single ints and floats values = [int(values), 0] elif not isinstance(values, list): @@ -467,7 +479,7 @@ def write_pru_msg(msg_type: int, values: list | float | int) -> None: f"expected u32 for type (={type(value)}) and content (={value})", ) - with open(sysfs_path / "pru_msg_box", "w") as file: + with (sysfs_path / "pru_msg_box").open("w", encoding="utf-8") as file: file.write(f"{msg_type} {values[0]} {values[1]}") @@ -475,7 +487,7 @@ def read_pru_msg() -> tuple: """ Returns: """ - with open(sysfs_path / "pru_msg_box") as f: + with (sysfs_path / "pru_msg_box").open(encoding="utf-8") as f: message = f.read().rstrip() msg_parts = [int(x) for x in message.split()] if len(msg_parts) < 2: @@ -529,7 +541,9 @@ def write_programmer_ctrl( raise SysfsInterfaceException( f"at least one parameter out of u32-bounds, value={value}", ) - with open(sysfs_path / "programmer" / attribute, "w") as file: + with (sysfs_path / "programmer" / attribute).open( + "w", encoding="utf-8" + ) as file: log.debug("\t%s = '%s'", attribute, value) file.write(str(value)) @@ -537,25 +551,25 @@ def write_programmer_ctrl( def read_programmer_ctrl() -> list: parameters = [] for attribute in prog_attribs: - with open(sysfs_path / "programmer" / attribute) as file: + with (sysfs_path / "programmer" / attribute).open(encoding="utf-8") as file: parameters.append(file.read().rstrip()) return parameters def write_programmer_datasize(value: int) -> None: - with open(sysfs_path / "programmer/datasize", "w") as file: + with (sysfs_path / "programmer/datasize").open("w", encoding="utf-8") as file: file.write(str(value)) def start_programmer() -> None: - with open(sysfs_path / "programmer/state", "w") as file: + with (sysfs_path / "programmer/state").open("w", encoding="utf-8") as file: file.write("start") # force a pru-reset to jump into programming routine set_stop(force=True) def check_programmer() -> str: - with open(sysfs_path / "programmer/state") as file: + with (sysfs_path / "programmer/state").open(encoding="utf-8") as file: return file.read().rstrip() @@ -579,30 +593,29 @@ def load_pru0_firmware(value: str = "shepherd") -> None: if value.lower() in firmware.lower(): request = firmware log.debug("Will set pru0-firmware to '%s'", request) - count = 1 - while count < 6: + _count = 1 + while _count < 6: try: - with open(sysfs_path / "pru0_firmware", "w") as file: + with (sysfs_path / "pru0_firmware").open("w", encoding="utf-8") as file: file.write(request) time.sleep(2) - with open(sysfs_path / "pru0_firmware") as file: + with (sysfs_path / "pru0_firmware").open(encoding="utf-8") as file: result = file.read().rstrip() if result == request: return - else: - log.error( - "Requested PRU-FW (%s) was not set (is '%s')", - request, - result, - ) - except OSError: + log.error( + "Requested PRU-FW (%s) was not set (is '%s')", + request, + result, + ) + except OSError: # noqa: PERF203 log.warning( "PRU-Driver is locked up (during pru-fw change)" " -> will restart kernel-module (n=%d)", - count, + _count, ) reload_kernel_module() - count += 1 + _count += 1 raise OSError( "PRU-Driver still locked up (during pru-fw change)" " -> consider restarting node", @@ -610,19 +623,19 @@ def load_pru0_firmware(value: str = "shepherd") -> None: def pru0_firmware_is_default() -> bool: - count = 1 - while count < 6: + _count = 1 + while _count < 6: try: - with open(sysfs_path / "pru0_firmware") as file: + with (sysfs_path / "pru0_firmware").open(encoding="utf-8") as file: return file.read().rstrip() in pru0_firmwares[0] - except OSError: + except OSError: # noqa: PERF203 log.warning( "PRU-Driver is locked up (during pru-fw read)" " -> will restart kernel-module (n=%d)", - count, + _count, ) reload_kernel_module() - count += 1 + _count += 1 raise OSError( "PRU-Driver still locked up (during pru-fw read)" " -> consider restarting node", @@ -641,35 +654,35 @@ def pru0_firmware_is_default() -> bool: def get_mode() -> str: - with open(sysfs_path / "mode") as f: + with (sysfs_path / "mode").open(encoding="utf-8") as f: return str(f.read().rstrip()) def get_state() -> str: - with open(sysfs_path / "state") as f: + with (sysfs_path / "state").open(encoding="utf-8") as f: return str(f.read().rstrip()) def get_n_buffers() -> int: - with open(sysfs_path / "n_buffers") as f: + with (sysfs_path / "n_buffers").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_buffer_period_ns() -> int: - with open(sysfs_path / "buffer_period_ns") as f: + with (sysfs_path / "buffer_period_ns").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_samples_per_buffer() -> int: - with open(sysfs_path / "samples_per_buffer") as f: + with (sysfs_path / "samples_per_buffer").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_mem_address() -> int: - with open(sysfs_path / "memory/address") as f: + with (sysfs_path / "memory/address").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_mem_size() -> int: - with open(sysfs_path / "memory/size") as f: + with (sysfs_path / "memory/size").open(encoding="utf-8") as f: return int(f.read().rstrip()) From 3dfd332a0516bd4fe6fc6eb16a1ae4f227c19fe4 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 15:23:23 +0200 Subject: [PATCH 13/35] Update shepherd_callbacks.py --- .../src/shepherd_callbacks.py | 77 +++++++------------ 1 file changed, 28 insertions(+), 49 deletions(-) diff --git a/software/shepherd-devicetest/src/shepherd_callbacks.py b/software/shepherd-devicetest/src/shepherd_callbacks.py index 441f75a6..f71aa463 100644 --- a/software/shepherd-devicetest/src/shepherd_callbacks.py +++ b/software/shepherd-devicetest/src/shepherd_callbacks.py @@ -1,12 +1,14 @@ -import os +from pathlib import Path import dearpygui.dearpygui as dpg import zerorpc from past.builtins import execfile +from shepherd_core.data_models.testbed import TargetPort +from shepherd_sheep import ShepherdDebug -def include(filename): - if os.path.exists(filename): +def include(filename: str) -> None: + if Path(filename).exists(): execfile(filename) else: raise OSError(f"File {filename} not found") @@ -28,7 +30,7 @@ def program_start_callback(sender, data) -> None: def schedule_refresh() -> None: - global refresh_interval, refresh_next + global refresh_next if refresh_next <= dpg.get_frame_count(): refresh_next = round( dpg.get_frame_count() + refresh_interval * dpg.get_frame_rate(), @@ -37,7 +39,6 @@ def schedule_refresh() -> None: def window_refresh_callback(sender, data, userdata) -> None: - global shepherd_io, shepherd_state if (shepherd_io is not None) and (shepherd_state is True): gpio_refresh() adc_refresh() @@ -46,7 +47,7 @@ def window_refresh_callback(sender, data, userdata) -> None: def update_gui_elements() -> None: # TODO: DPG 0.8.x has trouble disabling items -> no themed feedback - global shepherd_io, shepherd_state, state_dict + global shepherd_state if shepherd_io is not None: update_power_state_shepherd() update_power_state_emulator() @@ -100,12 +101,12 @@ def refresh_rate_callback(sender, element_data, user_data) -> None: ######################## -shepherd_io = None +shepherd_io: ShepherdDebug | None = None shepherd_cal = None shepherd_state = True -def connect_to_node(host: str): +def connect_to_node(host: str) -> ShepherdDebug | None: # todo: could also use fabric/connection to start rpc server on node rpc_client = zerorpc.Client(timeout=60, heartbeat=20) rpc_client.connect(f"tcp://{host}:4242") @@ -115,13 +116,11 @@ def connect_to_node(host: str): # shepherd_io.__enter__() if check_connection(rpc_client): return rpc_client - else: - return None + return None def check_connection(rpc_client=None) -> bool: if rpc_client is None: - global shepherd_io rpc_client = shepherd_io if rpc_client is None: return False @@ -133,7 +132,7 @@ def check_connection(rpc_client=None) -> bool: def connect_button_callback(sender, element_data, user_data) -> None: - global shepherd_io, shepherd_cal + global shepherd_io host = dpg.get_value("host_name") if shepherd_io is None: shepherd_io = connect_to_node(host) @@ -151,93 +150,81 @@ def connect_button_callback(sender, element_data, user_data) -> None: ################################# -state_dict = {"Stop": False, "Running": True} -stateTrans_dict = {"stop": "Stop", "idle": "Stop", "running": "Running"} -able_dict = {"Disabled": False, "Enabled": True} -tgt_dict = {"Target A": True, "Target B": False} +state_dict: dict[str, bool] = {"Stop": False, "Running": True} +stateTrans_dict: dict[str, str] = {"stop": "Stop", "idle": "Stop", "running": "Running"} +able_dict: dict[str, bool] = {"Disabled": False, "Enabled": True} +tgt_dict: dict[str, TargetPort] = {"Target A": TargetPort.A, "Target B": TargetPort.B} def shepherd_power_callback(sender, element_data, user_data) -> None: - global shepherd_io, able_dict - shepherd_io.set_shepherd_pcb_power(able_dict[element_data]) + if shepherd_io is not None: + shepherd_io.set_shepherd_pcb_power(able_dict[element_data]) def update_power_state_shepherd(): - global shepherd_io, able_dict - value = int(shepherd_io.get_power_state_shepherd()) - dpg.set_value("shepherd_pwr", list(able_dict.keys())[value]) + if shepherd_io is not None: + value = int(shepherd_io.get_power_state_shepherd()) + dpg.set_value("shepherd_pwr", list(able_dict.keys())[value]) def shepherd_state_callback(sender, element_data, user_data) -> None: - global shepherd_io, shepherd_state, state_dict + global shepherd_state shepherd_state = state_dict[element_data] shepherd_io.set_shepherd_state(shepherd_state) update_gui_elements() def update_shepherd_state(): - global stateTrans_dict, shepherd_io dpg.set_value("shepherd_state", stateTrans_dict[shepherd_io.get_shepherd_state()]) def target_power_callback(sender, element_data, user_data) -> None: - global shepherd_io, tgt_dict - sel_a = tgt_dict[element_data] - shepherd_io.select_target_for_power_tracking(sel_a) + port = tgt_dict[element_data] + shepherd_io.select_port_for_power_tracking(port) def update_target_power(): - global shepherd_io, tgt_dict value = int(not shepherd_io.get_main_target_for_power()) - dpg.set_value("target_pwr", list(tgt_dict)[value]) + dpg.set_value("target_pwr", list(tgt_dict)[value]) # TODO: twisted def target_io_callback(sender, element_data, user_data) -> None: - global shepherd_io, tgt_dict - sel_a = tgt_dict[element_data] - shepherd_io.select_target_for_io_interface(sel_a) + port = tgt_dict[element_data] + shepherd_io.select_port_for_io_interface(port) def update_target_io(): - global shepherd_io, tgt_dict value = int(not shepherd_io.get_main_target_for_io()) - dpg.set_value("target_io", list(tgt_dict)[value]) + dpg.set_value("target_io", list(tgt_dict)[value]) # TODO: twisted def io_level_converter_callback(sender, element_data, user_data) -> None: - global shepherd_io, able_dict state = able_dict[element_data] shepherd_io.set_io_level_converter(state) def update_io_level_state(): - global shepherd_io, able_dict value = int(shepherd_io.get_target_io_level_conv()) dpg.set_value("io_lvl_converter", list(able_dict.keys())[value]) def set_power_state_emulator(sender, en_state, user_data) -> None: - global shepherd_io shepherd_io.set_power_state_emulator(en_state) def update_power_state_emulator() -> None: - global shepherd_io dpg.set_value("gpio_nRes_EMU_ADC", shepherd_io.get_power_state_emulator()) def set_power_state_recoder(sender, en_state, user_data) -> None: - global shepherd_io shepherd_io.set_power_state_recorder(en_state) def update_power_state_recorder() -> None: - global shepherd_io dpg.set_value("gpio_nRes_REC_ADC", shepherd_io.get_power_state_recorder()) def reinitialize_prus(sender, element_data, user_data) -> None: - global shepherd_io, shepherd_state shepherd_io.reinitialize_prus() shepherd_io.set_shepherd_state(shepherd_state) print("reinitialized PRUs") @@ -260,14 +247,12 @@ def reinitialize_prus(sender, element_data, user_data) -> None: def dac_en_callback(sender, en_state, user_value) -> None: - global shepherd_io, dac_channels value = dpg.get_value(f"value_raw_dac{user_value}") if en_state else 0 shepherd_io.dac_write(dac_channels[user_value][0], value) update_gui_elements() def dac_raw_callback(sender, value_raw, user_value) -> None: - global shepherd_io, dac_channels dac_cfg = dac_channels[user_value] value_si = shepherd_io.convert_raw_to_value(dac_cfg[1], dac_cfg[2], value_raw) value_si = round(value_si * 10**3, 3) @@ -276,7 +261,6 @@ def dac_raw_callback(sender, value_raw, user_value) -> None: def dac_val_callback(sender, value_mV, user_value) -> None: - global shepherd_io, dac_channels dac_cfg = dac_channels[user_value] value_raw = shepherd_io.convert_value_to_raw(dac_cfg[1], dac_cfg[2], value_mV / 1e3) dpg.set_value(f"value_raw_dac{user_value}", value_raw) @@ -295,7 +279,6 @@ def dac_val_callback(sender, value_mV, user_value) -> None: def adc_refresh() -> None: - global shepherd_io for _iter, _val in enumerate(adc_channels): adc_cfg = _val value_raw = shepherd_io.adc_read(adc_cfg[0]) @@ -330,7 +313,6 @@ def adc_refresh() -> None: def gpio_refresh() -> None: - global shepherd_io value = shepherd_io.gpi_read() dpg.set_value("gpio_output", f"{value} binary: {value:>10b}") for name, pin in gpio_dir_channels.items(): @@ -341,19 +323,16 @@ def gpio_refresh() -> None: def gpio_callback(sender, element_data, user_data) -> None: - global shepherd_io value = gpio_channels.index(element_data) shepherd_io.set_gpio_one_high(value) gpio_refresh() def gpio_dir_callback(sender, element_data, user_data) -> None: - global shepherd_io shepherd_io.set_gpio_direction(user_data, element_data) def gpio_batok_callback(sender, en_state, user_data) -> None: - global shepherd_io shepherd_io.gp_set_batok(en_state) gpio_refresh() @@ -396,5 +375,5 @@ def update_button_callback(sender, element_data, user_data) -> None: print("update_button_callback") -def save_button_callback(sender, element_data, user_data) -> None: +def save_button_callback(_sender, _element_data, _user_data) -> None: print("connect_button_callback") From 35fee1ef4cf8e5955ba1419829d8bcf9dc467006 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 15:27:54 +0200 Subject: [PATCH 14/35] replace setup.cfg by pyproject --- pyproject.toml | 168 +++++++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 125 ------------------------------------ 2 files changed, 168 insertions(+), 125 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..55fb7c8c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,168 @@ +[tool.codespell] +builtin = "clear,rare,informal,usage,code,en-GB_to_en-US" + +skip = "*.brd,*.sch,*.lbr,*.svg,*/sphinx_to_pages.yml,*/nrf52840.h,*/nrf52840_bitfields.h,*/core_cm4.h" +# TODO: sphinx-entry only included temporarily +# TODO: programmer header-files contain external code +ignore-words-list = "dout,jupyter,stdio,astroid,uint,fram,arange,ro" + +# options without argument +check-filenames = "" +check-hidden = "" + +[tool.isort] +profile = "black" +filter_files = true +force_single_line = true + +[tool.flake8] +max-line-length = 100 +require-plugins = [ + # https://github.com/DmytroLitvinov/awesome-flake8-extensions + ### Bugs & Code-Smells + "flake8-bugbear", + "flake8-secure-coding-standard", + "flake8-bandit", + "flake8-builtins", + + ### Clean Code + "flake8-comprehensions", + "flake8-simplify", + "flake8-eradicate", + "flake8-commas", + + ### Limitations + "flake8-blind-except", + "flake8-logging-format", + "flake8-print", + + ### Documentation + "flake8-comments", + # flake8-docstrings + # flake8-rst-docstrings + + ### Test-Improvements + "flake8-assertive", +] + +extend-ignore = [ + # black-formatter-specific + "E203", + # open() vs os.open() + "SCS109", + # hideous commas + "C812", + "C815", + "C813", + # DOCString TODO: reduce here + "D100", + "D102", + "D103", + "G200", +] + +per-file-ignores = """ +# asserts (in tests are ok), also allow print() instead of logger, +*/tests_manual/*: E501 +*/tests/*: SCS108, S101, T201 +# commented out code, TODO: finalize model +*_model.py: E800 +# telnet for openOCD, for now, TODO: replace by direct ssh.run() +software/shepherd-herd/shepherd_herd/open_ocd_cli.py: S401, S312 +# allow print() instead of logger, +# software/python-package/tests/test_virtual_source.py: T201 +# under development - so more +software/firmware/pru0-cython-module/*: E800, T201, F401 +""" +exclude = "software/shepherd-devicetest/*" + +count = true +statistics = true + +[tool.ruff] # TODO: once stable it replaces flake8, pyupgrade, isort, ... +line-length = 100 +select = [ + "A", # flake8-builtins +# "ANN", # flake8-annotations, TODO: turn on + "ARG", # flake8-unused-arguments + "B", # Bugbear + "C", + "COM", # flake8-commas +# "CPY", # flake8-copyright + "C4", # flake8-comprehensions + "DTZ", # flake8-datetimez +# "D", # pydocstyle, TODO: activate + "E", # pycodestyle errors + "ERA", # eradicate commented out code + "F", # pyflakes +# "FA", # flake8-future-annotations + "FBT", # boolean traps + "FLY", # flynt + "FURB", # refurb + "G", # flake8-logging-format +# "I", # incomplete isort + "INP", # flake8-no-pep420 + "LOG", # flake8-logging + "N", # naming + "NPY", # NumPy-specific rules + "PD", # pandas-vet + "PERF", # Perflint + "PL", # Pylint + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "RET", # flake8-return + "RUF", # Ruff-specific rules + "S", # bandit, security + "SLF", # flake8-self + "SIM", # flake8-simplify + "TID", # flake8-tidy-imports + "TCH", # flake8-type-checking + "T10", # flake8-print + "UP", # pyupgrade + "W", # pycodestyle warnings + "YTT", # flake8-2020 +] +ignore = [ + "N802", "N803", "N806", "N815", "N816", # naming (si-units should stay) + "PLR2004", # magic values + "TID252", # relative imports from parent + "RUF100", # noqa from other linters, TODO: replace flake8, when security is fully ported (S4**) + "PLR0904", # complexity + "PLR0911", "PLR0912", # complexity + "PLR0913", "PLR0915", # complexity + "C901", # complexity + "E203", # whitespace before ':' (black-default) + "ANN101", "ANN102", # self & cls not type-annotated + "ANN401", # Any as valid type + "COM812", # trailing comma, same line +] +target-version = "py310" +preview = true + +exclude= [ # external projects + "software/shepherd-datalib/*", + "software/shepherd-webservice/*", + "software/firmware/pru0-cython-module/*", +] + +[tool.ruff.per-file-ignores] +"*/tests/**" = ["ARG", "S", "D", "SLF001"] +"*/examples/**" = ["INP001"] # no namespace +"software/shepherd-calibration/**" = ["ERA001"] # comments +"software/shepherd-devicetest/**" = ["ERA001", "ARG001", "PLW0603", "F405", "F403"] +"docs/**" = ["INP001"] + +[tool.ruff.mccabe] +# Unlike Flake8, default to a complexity level of 10. +max-complexity = 10 + +[tool.pyright] +root = "./" +include = [ + './software/python-package', + './software/shepherd-herd', + './tests', +] +pythonVersion = "3.10" +pythonPlatform = "All" +reportMissingParameterType = true diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 7f374be4..00000000 --- a/setup.cfg +++ /dev/null @@ -1,125 +0,0 @@ -[codespell] -skip = *.brd,*.sch,*.lbr,*.svg,*/sphinx_to_pages.yml,*/nrf52840.h,*/nrf52840_bitfields.h,*/core_cm4.h -# TODO: sphinx-entry only included temporarily -# TODO: programmer header-files contain external code - -builtin = clear,rare,informal,usage,code,en-GB_to_en-US - -ignore-words-list = dout,jupyter,stdio,astroid,uint,fram,arange,ro - -# options without argument -check-filenames = -check-hidden = - -[isort] -profile=black -filter_files=true -force_single_line=true - -[flake8] -max-line-length = 100 -require-plugins = - # https://github.com/DmytroLitvinov/awesome-flake8-extensions - - ### Bugs & Code-Smells - flake8-bugbear - flake8-secure-coding-standard - flake8-bandit - flake8-builtins - - ### Clean Code - flake8-comprehensions - flake8-simplify - flake8-eradicate - flake8-commas - - ### Limitations - flake8-blind-except - flake8-logging-format - flake8-print - - ### Documentation - flake8-comments - # flake8-docstrings - # flake8-rst-docstrings - - ### Test-Improvements - flake8-assertive - -extend-ignore = - # black-formatter-specific - E203 - # open() vs os.open() - SCS109 - # hideous commas - C812 - C815 - C813 - # DOCString TODO: reduce here - D100 - D102 - D103 - -per-file-ignores = - # asserts (in tests are ok), also allow print() instead of logger - software/python-package/tests_manual/*: E501 - software/python-package/tests/*: SCS108 S101 T201 - software/shepherd-herd/tests/*: SCS108 S101 - # commented out code, TODO: finalize model - *_model.py: E800 - # telnet for openOCD, for now, TODO: replace by direct ssh.run() - software/shepherd-herd/shepherd_herd/open_ocd_cli.py: S401 S312 - # allow print() instead of logger, - # software/python-package/tests/test_virtual_source.py: T201 - # under development - so more - software/firmware/pru0-cython-module/*: E800, T201, F401 - -exclude = - software/shepherd-devicetest/* - -count = True -statistics = True - -[ruff] # TODO: once stable it replaces flake8, pyupgrade, isort, ... -line-length = 100 -select = - A - ARG - B - C - C4 - E - ERA - F -# I # incomplete isort - N - PLC - PLE - PLR - PLW - RET - S - T10 - UP - W # TODO: just a proposed starting-config - -ignore = A003 -target-version = py38 - -per-file-ignores = - __init__.py: F401 - tests/**: ARG, S - -# Unlike Flake8, default to a complexity level of 10. -mccabe = - max-complexity: 10 - -[pyright] -root = "./" -include = - ./software/python-package - ./software/shepherd-herd - ./tests -pythonVersion = 3.8 -pythonPlatform = All -reportMissingParameterType = True From 91db2a9c18f4184f5dc456e01a00adcf91130257 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 16:17:54 +0200 Subject: [PATCH 15/35] avoid some boolean traps --- .../shepherd_cal/profile_analyzer.py | 1 + .../shepherd_cal/profiler.py | 4 +++- .../shepherd_cal/profiler_cli.py | 6 +++-- software/shepherd-devicetest/src/__init__.py | 2 +- software/shepherd-herd/shepherd_herd/herd.py | 24 +++++++++---------- .../shepherd-herd/shepherd_herd/herd_cli.py | 10 ++++---- 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py index 5c04b37c..a76eb602 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py +++ b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py @@ -9,6 +9,7 @@ def analyze_directory( folder_path: Path, stats_path: Path | None = None, + *, do_plots: bool = False, ) -> None: stats_list = [] diff --git a/software/shepherd-calibration/shepherd_cal/profiler.py b/software/shepherd-calibration/shepherd_cal/profiler.py index e34e24a6..3ec939ec 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler.py +++ b/software/shepherd-calibration/shepherd_cal/profiler.py @@ -21,6 +21,8 @@ from .calibrator import Calibrator from .logger import logger +# ruff: noqa: FBT003 + INSTR_PROFILE_SHP = """ ---------------------- Characterize Shepherd-Frontend ----------------------- - remove targets from target-ports @@ -36,7 +38,7 @@ class Profiler: - def __init__(self, calibrator: Calibrator, short: bool = False): + def __init__(self, calibrator: Calibrator, *, short: bool = False): self._cal: Calibrator = calibrator if short: diff --git a/software/shepherd-calibration/shepherd_cal/profiler_cli.py b/software/shepherd-calibration/shepherd_cal/profiler_cli.py index 2b5b4fc4..0c9a58bb 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler_cli.py +++ b/software/shepherd-calibration/shepherd_cal/profiler_cli.py @@ -25,6 +25,8 @@ from .profiler import INSTR_PROFILE_SHP from .profiler import Profiler +# ruff: noqa: FBT001, FBT003 + cli_pro = typer.Typer( name="profile", help="Sub-commands for profiling the analog frontends", @@ -84,7 +86,7 @@ def measure( file_path = outfile.stem + ".profile_full" + components + ".npz" shpcal = Calibrator(host, user, password, smu_ip, smu_4wire, smu_nplc) - profiler = Profiler(shpcal, short) + profiler = Profiler(shpcal, short=short) results: dict[str, np.ndarray] = {"cape": cape_serial} if not quiet: @@ -142,4 +144,4 @@ def analyze( ): """Analyze profile-data""" cli_setup_callback(verbose) - analyze_directory(infiles, outfile, plot) + analyze_directory(infiles, outfile, do_plots=plot) diff --git a/software/shepherd-devicetest/src/__init__.py b/software/shepherd-devicetest/src/__init__.py index 55f8d4f7..63bd5109 100644 --- a/software/shepherd-devicetest/src/__init__.py +++ b/software/shepherd-devicetest/src/__init__.py @@ -311,7 +311,7 @@ def assemble_window(): dpg.setup_dearpygui() assemble_window() - dpg.set_primary_window("main", True) + dpg.set_primary_window("main", value=True) dpg.show_viewport() dpg.start_dearpygui() diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 70bac8de..17cc78b2 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -189,7 +189,7 @@ def _open(self) -> None: @staticmethod def _thread_run( cnx: Connection, - sudo: bool, + sudo: bool, # noqa: FBT001 cmd: str, results: dict[int, Result], index: int, @@ -210,7 +210,7 @@ def _thread_run( cnx.close() @validate_call - def run_cmd(self, cmd: str, sudo: bool = False) -> dict[int, Result]: + def run_cmd(self, cmd: str, *, sudo: bool = False) -> dict[int, Result]: """Run COMMAND on the shell -> Returns output-results NOTE: in case of error on a node that corresponding dict value is unavailable """ @@ -230,7 +230,7 @@ def run_cmd(self, cmd: str, sudo: bool = False) -> dict[int, Result]: raise RuntimeError("ZERO nodes answered - check your config") return results - def print_output(self, replies: dict[int, Result], verbose: bool = False) -> None: + def print_output(self, replies: dict[int, Result], *, verbose: bool = False) -> None: """Logs output-results of shell commands""" for i, hostname in enumerate(self.hostnames.values()): if not isinstance(replies.get(i), Result): @@ -250,7 +250,7 @@ def _thread_put( cnx: Connection, src: Path | StringIO, dst: Path, - force_overwrite: bool, + force_overwrite: bool, # noqa: FBT001 ): if isinstance(src, StringIO): filename = dst.name @@ -281,7 +281,7 @@ def _thread_put( def put_file( self, src: StringIO | Path | str, - dst: Path | str, + dst: Path | str, *, force_overwrite: bool = False, ) -> None: if isinstance(src, StringIO): @@ -336,7 +336,7 @@ def _thread_get(cnx: Connection, src: Path, dst: Path): def get_file( self, src: Path | str, - dst_dir: Path | str, + dst_dir: Path | str, *, timestamp: bool = False, separate: bool = False, delete_src: bool = False, @@ -493,7 +493,7 @@ def put_task( ) @validate_call - def check_status(self, warn: bool = False) -> bool: + def check_status(self, *, warn: bool = False) -> bool: """Returns true as long as one instance is still measuring :param warn: @@ -540,7 +540,7 @@ def stop_measurement(self) -> int: return exit_code @validate_call - def poweroff(self, restart: bool) -> int: + def poweroff(self, *, restart: bool) -> int: logger.debug("Shepherd-nodes affected: %s", self.hostnames.values()) if restart: replies = self.run_cmd(sudo=True, cmd="reboot") @@ -589,7 +589,7 @@ def inventorize(self, output_path: Path) -> bool: # TODO: best case - add all to one file or a new inventories-model? @validate_call - def run_task(self, config: Path | ShpModel, attach: bool = False) -> int: + def run_task(self, config: Path | ShpModel, *, attach: bool = False) -> int: if attach: remote_path = Path("/etc/shepherd/config_for_herd.yaml") self.put_task(config, remote_path) @@ -598,7 +598,7 @@ def run_task(self, config: Path | ShpModel, attach: bool = False) -> int: exit_code = max([reply.exited for reply in replies.values()]) if exit_code: logger.error("Running Task failed - will exit now!") - self.print_output(replies, True) + self.print_output(replies, verbose=True) else: remote_path = Path("/etc/shepherd/config.yaml") @@ -613,7 +613,7 @@ def run_task(self, config: Path | ShpModel, attach: bool = False) -> int: def get_task_files( self, config: Path | ShpModel, - dst_dir: Path | str, + dst_dir: Path | str, *, separate: bool = False, delete_src: bool = False, ) -> bool: @@ -629,7 +629,7 @@ def get_task_files( for task in tasks: if hasattr(task, "output_path"): logger.info("General remote path is: %s", task.output_path) - failed |= self.get_file(task.output_path, dst_dir, separate, delete_src) + failed |= self.get_file(task.output_path, dst_dir, separate=separate, delete_src=delete_src) if hasattr(task, "get_output_paths"): for host, path in task.get_output_paths().items(): logger.info("Remote path of '%s' is: %s, WON'T COPY", host, path) diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index fbb97416..bfad063e 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -16,6 +16,8 @@ from .logger import logger as log from .logger import set_verbosity +# ruff: noqa: FBT001 + # TODO: # - click.command shorthelp can also just be the first sentence of docstring # https://click.palletsprojects.com/en/8.1.x/documentation/#command-short-help @@ -102,7 +104,7 @@ def cli( @click.pass_context def poweroff(ctx: click.Context, restart: bool): with ctx.obj["herd"] as herd: - exit_code = herd.poweroff(restart) + exit_code = herd.poweroff(restart=restart) sys.exit(exit_code) @@ -152,7 +154,7 @@ def inventorize(ctx: click.Context, output_path: Path) -> None: @click.pass_context def run(ctx: click.Context, config: Path, attach: bool): with ctx.obj["herd"] as herd: - exit_code = herd.run_task(config, attach) + exit_code = herd.run_task(config, attach=attach) sys.exit(exit_code) @@ -409,7 +411,7 @@ def distribute( force_overwrite: bool, ): with ctx.obj["herd"] as herd: - herd.put_file(filename, remote_path, force_overwrite) + herd.put_file(filename, remote_path, force_overwrite=force_overwrite) @cli.command(short_help="Retrieves remote hdf file FILENAME and stores in in OUTDIR") @@ -472,7 +474,7 @@ def retrieve( if herd.await_stop(timeout=30): raise Exception("shepherd still active after timeout") - failed = herd.get_file(filename, outdir, timestamp, separate, delete) + failed = herd.get_file(filename, outdir, timestamp=timestamp, separate=separate, delete_src=delete) sys.exit(failed) From d416d3b58f48bc77e0a3fb171994f50a142db55b Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 18:21:47 +0200 Subject: [PATCH 16/35] py: add more explicit types --- software/python-package/shepherd_sheep/cli.py | 26 ++-- .../python-package/shepherd_sheep/eeprom.py | 19 ++- .../shepherd_sheep/h5_writer.py | 14 ++- .../python-package/shepherd_sheep/launcher.py | 19 ++- .../shepherd_sheep/monitor_abc.py | 9 +- .../shepherd_sheep/monitor_kernel.py | 9 +- .../shepherd_sheep/monitor_ptp.py | 9 +- .../shepherd_sheep/monitor_sheep.py | 9 +- .../shepherd_sheep/monitor_sysutil.py | 9 +- .../shepherd_sheep/monitor_uart.py | 11 +- .../shepherd_sheep/shared_memory.py | 24 ++-- .../shepherd_sheep/shepherd_debug.py | 23 ++-- .../shepherd_sheep/shepherd_emulator.py | 24 ++-- .../shepherd_sheep/shepherd_harvester.py | 16 ++- .../shepherd_sheep/shepherd_io.py | 37 ++++-- .../shepherd_sheep/sysfs_interface.py | 15 ++- .../shepherd_sheep/target_io.py | 2 +- software/python-package/tests/conftest.py | 12 +- .../python-package/tests/test_cli_misc.py | 49 ++++---- .../tests/test_cli_programmer.py | 47 +++++-- .../python-package/tests/test_cli_run_task.py | 68 ++++++----- .../python-package/tests/test_datalogging.py | 12 +- software/python-package/tests/test_eeprom.py | 35 +++--- .../python-package/tests/test_emulation.py | 115 +++++++++--------- software/python-package/tests/test_harvest.py | 34 +++--- .../tests/test_sysfs_interface.py | 44 ++++--- .../tests/test_virtual_source.py | 27 ++-- .../tests_manual/testbench_gen_fake.py | 2 +- .../shepherd_cal/calibrator.py | 12 +- .../shepherd_cal/calibrator_cli.py | 6 +- .../shepherd_cal/cli_helper.py | 2 +- .../shepherd_cal/profile.py | 10 +- .../shepherd_cal/profile_analyzer.py | 2 +- .../shepherd_cal/profile_calibration.py | 3 +- .../shepherd_cal/profiler.py | 2 +- .../shepherd_cal/profiler_cli.py | 4 +- software/shepherd-devicetest/src/__init__.py | 2 +- .../src/shepherd_callbacks.py | 10 +- software/shepherd-herd/shepherd_herd/herd.py | 48 ++++++-- .../shepherd-herd/shepherd_herd/herd_cli.py | 32 +++-- software/shepherd-herd/tests/conftest.py | 2 +- software/shepherd-herd/tests/test_cli_emu.py | 17 +-- software/shepherd-herd/tests/test_cli_hrv.py | 12 +- 43 files changed, 539 insertions(+), 345 deletions(-) diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index daf6911d..b4b62625 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -12,6 +12,8 @@ import sys import time from pathlib import Path +from typing import TypedDict +from typing import Unpack import click import gevent @@ -59,7 +61,7 @@ # - redone programmer, emulation -def exit_gracefully(*_args): # type: ignore +def exit_gracefully(*_args) -> None: log.warning("Aborted!") sys.exit(0) @@ -77,7 +79,7 @@ def exit_gracefully(*_args): # type: ignore help="Prints version-info at start (combinable with -v)", ) @click.pass_context -def cli(ctx: click.Context, verbose: bool, version: bool): +def cli(ctx: click.Context, verbose: bool, version: bool) -> None: """Shepherd: Synchronized Energy Harvesting Emulator and Recorder""" signal.signal(signal.SIGTERM, exit_gracefully) signal.signal(signal.SIGINT, exit_gracefully) @@ -114,7 +116,7 @@ def cli(ctx: click.Context, verbose: bool, version: bool): default="A", help="Choose Target-Port of Cape for powering", ) -def target_power(on: bool, voltage: float, gpio_pass: bool, target_port: str): +def target_power(on: bool, voltage: float, gpio_pass: bool, target_port: str) -> None: if not on: voltage = 0.0 # TODO: output would be nicer when this uses shepherdDebug as base @@ -150,7 +152,7 @@ def target_power(on: bool, voltage: float, gpio_pass: bool, target_port: str): type=click.Path(exists=True, readable=True, file_okay=True, dir_okay=False), default=Path("/etc/shepherd/config.yaml"), ) -def run(config: Path): +def run(config: Path) -> None: failed = run_task(config) if failed: log.debug("Tasks signaled an error (failed).") @@ -161,7 +163,7 @@ def run(config: Path): context_settings={"help_option_names": ["-h", "--help"], "obj": {}}, short_help="Read/Write data from EEPROM", ) -def eeprom(): +def eeprom() -> None: pass @@ -172,7 +174,7 @@ def eeprom(): ) def write( cal_file: Path | None, -): +) -> None: cal_cape = CalibrationCape.from_file(cal_file) try: log.debug("Will write Cal-Data:\n\n%s", str(cal_cape)) @@ -191,7 +193,7 @@ def write( default=None, help="If provided, calibration data is dumped to this file", ) -def read(cal_file: Path | None): +def read(cal_file: Path | None) -> None: set_verbosity() try: @@ -214,7 +216,7 @@ def read(cal_file: Path | None): @cli.command(short_help="Start zerorpc server") @click.option("--port", "-p", type=click.INT, default=4242) -def rpc(port: int | None): +def rpc(port: int | None) -> None: shepherd_io = ShepherdDebug() shepherd_io.__enter__() log.info("Shepherd Debug Interface: Initialized") @@ -224,7 +226,7 @@ def rpc(port: int | None): server.bind(f"tcp://0.0.0.0:{ port }") time.sleep(1) - def stop_server(): + def stop_server() -> None: server.stop() shepherd_io.__exit__() sys.exit(0) @@ -253,7 +255,7 @@ def inventorize(output_path: Path) -> None: @cli.command(short_help="Start shepherd launcher") @click.option("--led", "-l", type=click.INT, default=22) @click.option("--button", "-b", type=click.INT, default=65) -def launcher(led: int, button: int): +def launcher(led: int, button: int) -> None: with Launcher(button, led) as launch: launch.run() @@ -306,7 +308,7 @@ def launcher(led: int, button: int): is_flag=True, help="dry-run the programmer - no data gets written", ) -def program(**kwargs): +def program(**kwargs: Unpack[TypedDict]) -> None: protocol_dict = { "nrf52": ProgrammerProtocol.swd, "msp430": ProgrammerProtocol.sbw, @@ -321,7 +323,7 @@ def program(**kwargs): short_help="Reloads the shepherd-kernel-module", context_settings={"ignore_unknown_options": True}, ) -def fix(): +def fix() -> None: set_verbosity() reload_kernel_module() diff --git a/software/python-package/shepherd_sheep/eeprom.py b/software/python-package/shepherd_sheep/eeprom.py index bda95fc6..60aaa54d 100644 --- a/software/python-package/shepherd_sheep/eeprom.py +++ b/software/python-package/shepherd_sheep/eeprom.py @@ -13,6 +13,8 @@ import os import struct from contextlib import suppress +from types import TracebackType +from typing import Self from shepherd_core.data_models.base.calibration import CalibrationCape from shepherd_core.data_models.base.calibration import CapeData @@ -47,7 +49,7 @@ class EEPROM: """ - def __init__(self, bus_num: int = 2, address: int = 0x54, wp_pin: int = 49): + def __init__(self, bus_num: int = 2, address: int = 0x54, wp_pin: int = 49) -> None: """Initializes EEPROM by bus number and address. Args: @@ -59,12 +61,19 @@ def __init__(self, bus_num: int = 2, address: int = 0x54, wp_pin: int = 49): self._write_protect_pin: GPIO = GPIO(wp_pin, "out") self._write_protect_pin.write(True) - def __enter__(self): + def __enter__(self) -> Self: self.fd = os.open(self.dev_path, os.O_RDWR | os.O_SYNC) return self - def __exit__(self, *args): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: os.close(self.fd) + pass def _read(self, address: int, n_bytes: int) -> bytes: """Reads a given number of bytes from given address. @@ -95,7 +104,7 @@ def _write(self, address: int, buffer: bytes) -> None: raise self._write_protect_pin.write(True) - def __getitem__(self, key: str): + def __getitem__(self, key: str) -> bytes | str: """Retrieves attribute from EEPROM. Args: @@ -114,7 +123,7 @@ def __getitem__(self, key: str): return str_data[0].decode("utf-8") return raw_data - def __setitem__(self, key: str, value): # type: ignore + def __setitem__(self, key: str, value: str | bytes) -> None: """Writes attribute to EEPROM. Args: diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index 3d67d1f4..d40b6074 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -9,8 +9,10 @@ :license: MIT, see LICENSE for more details. """ from pathlib import Path +from types import TracebackType from typing import TYPE_CHECKING from typing import ClassVar +from typing import Self if TYPE_CHECKING: import h5py @@ -70,7 +72,7 @@ def __init__( verbose: bool | None = True, samples_per_buffer: int = 10_000, samplerate_sps: int = 100_000, - ): + ) -> None: # hopefully overwrite defaults from Reader self.samples_per_buffer: int = samples_per_buffer # TODO: test self.samplerate_sps: int = samplerate_sps @@ -113,7 +115,7 @@ def __init__( self.sysutil_log_enabled: bool = True self.monitors: list[Monitor] = [] - def __enter__(self): + def __enter__(self) -> Self: """Initializes the structure of the HDF5 file HDF5 is hierarchically structured and before writing data, we have to @@ -167,7 +169,13 @@ def __enter__(self): return self - def __exit__(self, *exc): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: # trim over-provisioned parts self.grp_data["time"].resize((self.data_pos,)) self.grp_data["voltage"].resize((self.data_pos,)) diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index a4ce9e21..c59a81a6 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -13,6 +13,7 @@ from contextlib import suppress from threading import Event from threading import Thread +from types import TracebackType from .logger import log @@ -22,10 +23,10 @@ from periphery import GPIO -def call_repeatedly(interval: float, func, *args): # type: ignore +def call_repeatedly(interval: float, func, *args): stopped = Event() - def loop(): + def loop() -> None: while not stopped.wait(interval): # the first call is in `interval` secs func(*args) @@ -51,13 +52,13 @@ def __init__( pin_led: int = 22, pin_ack_watchdog: int = 68, service_name: str = "shepherd", - ): + ) -> None: self.pin_button = pin_button self.pin_led = pin_led self.pin_ack_watchdog = pin_ack_watchdog self.service_name = service_name - def __enter__(self): + def __enter__(self) -> None: self.gpio_led = GPIO(self.pin_led, "out") self.gpio_button = GPIO(self.pin_button, "in") self.gpio_ack_watchdog = GPIO(self.pin_ack_watchdog, "out") @@ -81,7 +82,13 @@ def __enter__(self): return self - def __exit__(self, *exc): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.gpio_led.close() self.gpio_button.close() @@ -146,7 +153,7 @@ def get_state(self, timeout: float = 10) -> bool: return False raise Exception(f"Unknown state { systemd_state }") - def set_service(self, requested_state: bool): + def set_service(self, requested_state: bool) -> bool: """Changes state of shepherd service. Args: diff --git a/software/python-package/shepherd_sheep/monitor_abc.py b/software/python-package/shepherd_sheep/monitor_abc.py index 036f1f52..a990742c 100644 --- a/software/python-package/shepherd_sheep/monitor_abc.py +++ b/software/python-package/shepherd_sheep/monitor_abc.py @@ -1,6 +1,7 @@ import threading from abc import ABC from abc import abstractmethod +from types import TracebackType import h5py from shepherd_core import Compression @@ -42,7 +43,13 @@ def __init__( type(self).__name__, ) - def __exit__(self, *exc) -> None: # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.data["time"].resize((self.position,)) log.info( "[%s] recorded %d events", diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index b995074b..11c1003f 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -2,6 +2,7 @@ import subprocess # noqa: S404 import threading import time +from types import TracebackType import h5py from shepherd_core import Compression @@ -49,7 +50,13 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc) -> None: # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index d0332ebb..6ec5cb82 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -2,6 +2,7 @@ import subprocess # noqa: S404 import threading import time +from types import TracebackType import h5py from shepherd_core import Compression @@ -50,7 +51,13 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc) -> None: # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_sheep.py b/software/python-package/shepherd_sheep/monitor_sheep.py index 3c6c8149..6eab029e 100644 --- a/software/python-package/shepherd_sheep/monitor_sheep.py +++ b/software/python-package/shepherd_sheep/monitor_sheep.py @@ -1,4 +1,5 @@ import threading +from types import TracebackType import h5py from shepherd_core import Compression @@ -38,7 +39,13 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc) -> None: # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_sysutil.py b/software/python-package/shepherd_sheep/monitor_sysutil.py index 54531fdf..7af3d05e 100644 --- a/software/python-package/shepherd_sheep/monitor_sysutil.py +++ b/software/python-package/shepherd_sheep/monitor_sysutil.py @@ -1,5 +1,6 @@ import threading import time +from types import TracebackType import h5py import numpy as np @@ -70,7 +71,13 @@ def __init__( self.thread = threading.Thread(target=self.thread_fn, daemon=True) self.thread.start() - def __exit__(self, *exc) -> None: # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index 1c465358..47a3aa7d 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -1,6 +1,7 @@ import threading import time from pathlib import Path +from types import TracebackType import h5py import serial @@ -17,7 +18,7 @@ def __init__( compression: Compression | None = Compression.default, uart: str = "/dev/ttyS1", baudrate: int | None = None, - ): + ) -> None: super().__init__(target, compression, poll_intervall=0.05) self.uart = uart self.baudrate = baudrate @@ -50,7 +51,13 @@ def __init__( self.uart, ) - def __exit__(self, *exc): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.event.set() if self.thread is not None: self.thread.join(timeout=self.poll_intervall) diff --git a/software/python-package/shepherd_sheep/shared_memory.py b/software/python-package/shepherd_sheep/shared_memory.py index 593600f8..1ffd4cd6 100644 --- a/software/python-package/shepherd_sheep/shared_memory.py +++ b/software/python-package/shepherd_sheep/shared_memory.py @@ -4,6 +4,8 @@ import time from dataclasses import dataclass from datetime import timedelta +from types import TracebackType +from typing import Self import numpy as np from shepherd_core.data_models import GpioTracing @@ -26,11 +28,11 @@ def __init__( self, timestamps_ns: np.ndarray | None = None, values: np.ndarray | None = None, - ): + ) -> None: self.timestamps_ns = timestamps_ns if timestamps_ns is not None else np.empty(0) self.values = values if values is not None else np.empty(0) - def __len__(self): + def __len__(self) -> int: return min(self.values.size, self.timestamps_ns.size) @@ -50,7 +52,7 @@ def __init__( gpio_edges: GPIOEdges | None = None, util_mean: float = 0, util_max: float = 0, - ): + ) -> None: self.timestamp_ns = timestamp_ns if timestamp_ns is not None else 0 self.voltage = voltage self.current = current @@ -61,7 +63,7 @@ def __init__( self.util_mean = util_mean self.util_max = util_max - def __len__(self): + def __len__(self) -> int: return min(self.voltage.size, self.current.size) @@ -84,7 +86,7 @@ def __init__( samples_per_buffer: int, trace_iv: PowerTracing | None, trace_gpio: GpioTracing | None, - ): + ) -> None: """Initializes relevant parameters for shared memory area. Args: @@ -157,10 +159,16 @@ def __init__( offset=self.address, ) - def __enter__(self): + def __enter__(self) -> Self: return self - def __exit__(self, *args): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: if self.mapped_mem is not None: self.mapped_mem.close() if self.devmem_fd is not None: @@ -369,7 +377,7 @@ def write_buffer( self.mapped_mem.write(voltage.tobytes()) self.mapped_mem.write(current.tobytes()) - def write_firmware(self, data: bytes): + def write_firmware(self, data: bytes) -> int: data_size = len(data) if data_size > self.size: raise ValueError("firmware file is larger than the SharedMEM-Buffer") diff --git a/software/python-package/shepherd_sheep/shepherd_debug.py b/software/python-package/shepherd_sheep/shepherd_debug.py index c901ac45..edee4886 100644 --- a/software/python-package/shepherd_sheep/shepherd_debug.py +++ b/software/python-package/shepherd_sheep/shepherd_debug.py @@ -1,6 +1,7 @@ import contextlib import time from typing import NoReturn +from typing import Self import msgpack import msgpack_numpy @@ -34,7 +35,7 @@ class ShepherdDebug(ShepherdIO): with the ADC and DAC. """ - def __init__(self, use_io: bool = True): + def __init__(self, use_io: bool = True) -> None: super().__init__("debug", trace_iv=PowerTracing(), trace_gpio=GpioTracing()) self._io: TargetIO | None = TargetIO() if use_io else None @@ -57,14 +58,14 @@ def __init__(self, use_io: bool = True): self.W_inp_fWs: float = 0.0 self.W_out_fWs: float = 0.0 - def __enter__(self): + def __enter__(self) -> Self: super().__enter__() super().set_power_state_recorder(True) super().set_power_state_emulator(True) super().reinitialize_prus() return self - def adc_read(self, channel: str): + def adc_read(self, channel: str) -> int: """Reads value from specified ADC channel. Args: @@ -113,10 +114,10 @@ def gpi_read(self) -> int: ) return values[0] - def gp_set_batok(self, value: int): + def gp_set_batok(self, value: int) -> None: super()._send_msg(commons.MSG_DBG_GP_BATOK, value) - def dac_write(self, channels: int, value: int): + def dac_write(self, channels: int, value: int) -> None: """Writes value to specified DAC channel, DAC8562 Args: @@ -156,9 +157,11 @@ def vsource_init( log_intermediate: bool = False, dtype_in: EnergyDType = EnergyDType.ivsample, window_size: int | None = None, - ): + ) -> None: super().send_calibration_settings(cal_emu) - src_pru = ConverterPRUConfig.from_vsrc(src_cfg, log_intermediate) + src_pru = ConverterPRUConfig.from_vsrc( + src_cfg, log_intermediate_node=log_intermediate + ) super().send_virtual_converter_settings(src_pru) hrv_pru = HarvesterPRUConfig.from_vhrv( @@ -262,7 +265,9 @@ def cnv_update_states_and_output(self) -> int: return values[0] # V_out_dac_raw # TEST-SIMPLIFICATION - code below is also part py-vsource with same interface - def iterate_sampling(self, V_inp_uV: int = 0, A_inp_nA: int = 0, A_out_nA: int = 0): + def iterate_sampling( + self, V_inp_uV: int = 0, A_inp_nA: int = 0, A_out_nA: int = 0 + ) -> int: # NOTE: this includes the harvester P_inp_fW = self.cnv_calc_inp_power(V_inp_uV, A_inp_nA, include_hrv=True) A_out_raw = self._cal.emulator.adc_C_A.si_to_raw(A_out_nA * 10**-9) @@ -389,7 +394,7 @@ def switch_shepherd_mode(self, mode: str) -> str: super().start(wait_blocking=True) return mode_old - def sample_from_pru(self, length_n_buffers: int = 10): + def sample_from_pru(self, length_n_buffers: int = 10) -> bytes | None: length_n_buffers = int(min(max(length_n_buffers, 1), 55)) super().reinitialize_prus() time.sleep(0.1) diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index 0ba5973d..d307f4b6 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -3,6 +3,8 @@ import time from contextlib import ExitStack from datetime import datetime +from types import TracebackType +from typing import Self from shepherd_core import CalibrationPair from shepherd_core import CalibrationSeries @@ -39,7 +41,7 @@ def __init__( self, cfg: EmulationTask, mode: str = "emulator", - ): + ) -> None: log.debug("ShepherdEmulator-Init in %s-mode", mode) super().__init__( mode=mode, @@ -137,7 +139,7 @@ def __init__( for pin in range(len(target_pins)): self._io.set_pin_direction(pin, pdir=True) # True = Inp - def __enter__(self): + def __enter__(self) -> Self: super().__enter__() super().set_power_state_recorder(False) super().set_power_state_emulator(True) @@ -176,11 +178,19 @@ def __enter__(self): # ⤷ could be as low as ~ 10us return self - def __exit__(self, *args): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.stack.close() super().__exit__() - def return_buffer(self, index: int, buffer: DataBuffer, verbose: bool = False): + def return_buffer( + self, index: int, buffer: DataBuffer, verbose: bool = False + ) -> None: ts_start = time.time() if verbose else 0 # transform raw ADC data to SI-Units -> the virtual-source-emulator in PRU expects uV and nV @@ -196,7 +206,7 @@ def return_buffer(self, index: int, buffer: DataBuffer, verbose: bool = False): 1e3 * (time.time() - ts_start), ) - def run(self): + def run(self) -> None: self.start(self.start_time, wait_blocking=False) log.info("waiting %.2f s until start", self.start_time - time.time()) self.wait_for_start(self.start_time - time.time() + 15) @@ -219,7 +229,7 @@ def run(self): except ShepherdIOException as e: if self.cfg.abort_on_error: raise RuntimeError( - "Caught unforgivable ShepherdIO-Exception" + "Caught unforgivable ShepherdIO-Exception", ) from e log.warning("Caught an Exception", exc_info=e) continue @@ -247,5 +257,5 @@ def run(self): break if self.cfg.abort_on_error: raise RuntimeError( - "Caught unforgivable ShepherdIO-Exception" + "Caught unforgivable ShepherdIO-Exception", ) from e diff --git a/software/python-package/shepherd_sheep/shepherd_harvester.py b/software/python-package/shepherd_sheep/shepherd_harvester.py index cd92f88f..7de11bb9 100644 --- a/software/python-package/shepherd_sheep/shepherd_harvester.py +++ b/software/python-package/shepherd_sheep/shepherd_harvester.py @@ -3,6 +3,8 @@ import sys import time from contextlib import ExitStack +from types import TracebackType +from typing import Self from shepherd_core import local_tz from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig @@ -33,7 +35,7 @@ def __init__( self, cfg: HarvestTask, mode: str = "harvester", - ): + ) -> None: log.debug("ShepherdHarvester-Init in %s-mode", mode) super().__init__( mode=mode, @@ -84,7 +86,7 @@ def __init__( verbose=get_verbosity(), ) - def __enter__(self): + def __enter__(self) -> Self: super().__enter__() super().set_power_state_emulator(False) super().set_power_state_recorder(True) @@ -108,7 +110,13 @@ def __enter__(self): # ⤷ could be as low as ~ 10us return self - def __exit__(self, *args): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: self.stack.close() super().__exit__() @@ -146,7 +154,7 @@ def run(self) -> None: except ShepherdIOException as e: if self.cfg.abort_on_error: raise RuntimeError( - "Caught unforgivable ShepherdIO-Exception" + "Caught unforgivable ShepherdIO-Exception", ) from e log.warning("Caught an Exception", exc_info=e) continue diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 44ed2844..5668971c 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -9,6 +9,8 @@ """ import time from contextlib import suppress +from types import TracebackType +from typing import Self from pydantic import validate_call from shepherd_core import CalibrationEmulator @@ -22,6 +24,7 @@ from . import commons from . import sysfs_interface as sfs from .logger import log +from .shared_memory import DataBuffer from .shared_memory import SharedMemory from .sysfs_interface import check_sys_access @@ -43,7 +46,7 @@ class ShepherdIOException(Exception): - def __init__(self, message: str, id_num: int = 0, value: int = 0): + def __init__(self, message: str, id_num: int = 0, value: int = 0) -> None: super().__init__(message + f" [id=0x{id_num:x}, val=0x{value:x}]") self.id_num = id_num self.value = value @@ -64,7 +67,7 @@ class ShepherdIO: _instance = None @classmethod - def __new__(cls, *_args, **_kwds): + def __new__(cls, *_args, **_kwds) -> Self: """Implements singleton class.""" if ShepherdIO._instance is None: new_class = object.__new__(cls) @@ -79,7 +82,7 @@ def __init__( mode: str, trace_iv: PowerTracing | None, trace_gpio: GpioTracing | None, - ): + ) -> None: """Initializes relevant variables. Args: @@ -110,11 +113,11 @@ def __init__( self.n_buffers = 0 self.shared_mem: SharedMemory - def __del__(self): + def __del__(self) -> None: log.debug("Now deleting ShepherdIO") ShepherdIO._instance = None - def __enter__(self): + def __enter__(self) -> Self: try: for name, pin in gpio_pin_nums.items(): self.gpios[name] = GPIO(pin, "out") @@ -142,7 +145,13 @@ def __enter__(self): sfs.wait_for_state("idle", 3) return self - def __exit__(self, *args): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: log.info("Now exiting ShepherdIO") self._cleanup() ShepherdIO._instance = None @@ -158,7 +167,7 @@ def _send_msg(msg_type: int, values: int | list) -> None: """ sfs.write_pru_msg(msg_type, values) - def _get_msg(self, timeout_n: int = 5): + def _get_msg(self, timeout_n: int = 5) -> tuple[int, list[int]]: """Tries to retrieve formatted message from PRU0. Args: @@ -175,7 +184,7 @@ def _get_msg(self, timeout_n: int = 5): raise ShepherdIOException("Timeout waiting for message", ID_ERR_TIMEOUT) @staticmethod - def _flush_msgs(): + def _flush_msgs() -> None: """Flushes msg_channel by reading all available bytes.""" while True: try: @@ -214,7 +223,7 @@ def reinitialize_prus() -> None: sfs.set_stop(force=True) # forces idle sfs.wait_for_state("idle", 5) - def refresh_shared_mem(self): + def refresh_shared_mem(self) -> None: if hasattr(self, "shared_mem") and isinstance(self.shared_mem, SharedMemory): self.shared_mem.__exit__() @@ -251,7 +260,7 @@ def refresh_shared_mem(self): ) self.shared_mem.__enter__() - def _cleanup(self): + def _cleanup(self) -> None: log.debug("ShepherdIO is commanded to power down / cleanup") count = 1 while count < 6 and sfs.get_state() != "idle": @@ -487,7 +496,9 @@ def _return_buffer(self, index: int) -> None: """ self._send_msg(commons.MSG_BUF_FROM_HOST, index) - def get_buffer(self, timeout_n: int = 10, verbose: bool = False): + def get_buffer( + self, timeout_n: int = 10, verbose: bool = False + ) -> tuple[int, DataBuffer]: """Reads a data buffer from shared memory. Polls the msg-channel for a message from PRU0 and, if the message @@ -506,8 +517,8 @@ def get_buffer(self, timeout_n: int = 10, verbose: bool = False): """ while True: - msg_type, value = self._get_msg(timeout_n) - value = value[0] + msg_type, values = self._get_msg(timeout_n) + value = values[0] if msg_type == commons.MSG_BUF_FROM_PRU: ts_start = time.time() diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index 5b9e1a45..65682f4a 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -95,7 +95,7 @@ def remove_kernel_module() -> None: raise SystemError("Failed to unload shepherd kernel module.") -def reload_kernel_module(): +def reload_kernel_module() -> None: remove_kernel_module() load_kernel_module() @@ -399,7 +399,8 @@ def write_virtual_converter_settings(settings: ConverterPRUConfig) -> None: wait_for_state("idle", 3.0) with (sysfs_path / "virtual_converter_settings").open( - "w", encoding="utf-8" + "w", + encoding="utf-8", ) as file: file.write(output) @@ -438,7 +439,8 @@ def write_virtual_harvester_settings(settings: HarvesterPRUConfig) -> None: wait_for_state("idle", 3.0) with (sysfs_path / "virtual_harvester_settings").open( - "w", encoding="utf-8" + "w", + encoding="utf-8", ) as file: file.write(output) @@ -483,7 +485,7 @@ def write_pru_msg(msg_type: int, values: list | float | int) -> None: # noqa: P file.write(f"{msg_type} {values[0]} {values[1]}") -def read_pru_msg() -> tuple: +def read_pru_msg() -> tuple[int, list[int]]: """ Returns: """ @@ -516,7 +518,7 @@ def write_programmer_ctrl( pin_tdo: int = 0, pin_tms: int = 0, pin_dir_tms: int = 0, -): +) -> None: # check for validity pin_list = [pin_tck, pin_tdio, pin_dir_tdio, pin_tdo, pin_tms, pin_dir_tms] pin_set = set(pin_list) @@ -542,7 +544,8 @@ def write_programmer_ctrl( f"at least one parameter out of u32-bounds, value={value}", ) with (sysfs_path / "programmer" / attribute).open( - "w", encoding="utf-8" + "w", + encoding="utf-8", ) as file: log.debug("\t%s = '%s'", attribute, value) file.write(str(value)) diff --git a/software/python-package/shepherd_sheep/target_io.py b/software/python-package/shepherd_sheep/target_io.py index c65d47e6..5637d36e 100644 --- a/software/python-package/shepherd_sheep/target_io.py +++ b/software/python-package/shepherd_sheep/target_io.py @@ -57,7 +57,7 @@ class TargetIO: - def __init__(self): + def __init__(self) -> None: """Initializes relevant variables. Args: diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 92c081ce..9961caa9 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -1,8 +1,11 @@ import gc +from collections.abc import Generator from contextlib import suppress from pathlib import Path +from typing import Any import pytest +from click.testing import CliRunner from shepherd_sheep.sysfs_interface import load_kernel_module from shepherd_sheep.sysfs_interface import remove_kernel_module @@ -20,7 +23,7 @@ def check_beagleboard() -> bool: pytest.param("fake_hardware", marks=pytest.mark.fake_hardware), ], ) -def fake_hardware(request): +def fake_hardware(request) -> Generator[Any, None, None]: if request.param == "fake_hardware": request.fixturenames.append("fs") # needs pyfakefs installed fake_sysfs = request.getfixturevalue("fs") @@ -58,7 +61,7 @@ def pytest_collection_modifyitems(config, items) -> None: @pytest.fixture() -def shepherd_up(fake_hardware, shepherd_down): +def shepherd_up(fake_hardware, shepherd_down: None) -> Generator[None, None, None]: if fake_hardware is not None: files = [ ("/sys/shepherd/state", "idle"), @@ -99,3 +102,8 @@ def shepherd_up(fake_hardware, shepherd_down): def shepherd_down(fake_hardware) -> None: if fake_hardware is None: remove_kernel_module() + + +@pytest.fixture +def cli_runner() -> CliRunner: + return CliRunner() diff --git a/software/python-package/tests/test_cli_misc.py b/software/python-package/tests/test_cli_misc.py index b35b70da..1fc0fbf5 100644 --- a/software/python-package/tests/test_cli_misc.py +++ b/software/python-package/tests/test_cli_misc.py @@ -1,14 +1,15 @@ from pathlib import Path import pytest +from click.testing import CliRunner from shepherd_sheep.cli import cli @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_target_power_min_arg_a( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -22,8 +23,8 @@ def test_cli_target_power_min_arg_a( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_target_power_min_arg_b( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -40,8 +41,8 @@ def test_cli_target_power_min_arg_b( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_target_power_min_arg_c( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -58,8 +59,8 @@ def test_cli_target_power_min_arg_c( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_target_power_explicit_a( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -77,8 +78,8 @@ def test_cli_target_power_explicit_a( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_target_power_explicit_b( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -98,8 +99,8 @@ def test_cli_target_power_explicit_b( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_eeprom_read_min_arg_a( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -114,8 +115,8 @@ def test_cli_eeprom_read_min_arg_a( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_eeprom_read_min_arg_b( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_path: Path, ) -> None: path_inf = tmp_path / "info" @@ -137,8 +138,8 @@ def test_cli_eeprom_read_min_arg_b( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_eeprom_read_explicit( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_path: Path, ) -> None: path_inf = tmp_path / "info" @@ -160,8 +161,8 @@ def test_cli_eeprom_read_explicit( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_inventorize_min_arg_a( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, @@ -173,8 +174,8 @@ def test_cli_inventorize_min_arg_a( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_inventorize_min_arg_b( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_path: Path, ) -> None: file = tmp_path / "inv.yaml" @@ -189,8 +190,8 @@ def test_cli_inventorize_min_arg_b( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_inventorize_explicit( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_path: Path, ) -> None: file = tmp_path / "inv.yaml" @@ -205,8 +206,8 @@ def test_cli_inventorize_explicit( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_fix_kmod( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, ) -> None: res = cli_runner.invoke( cli, diff --git a/software/python-package/tests/test_cli_programmer.py b/software/python-package/tests/test_cli_programmer.py index bbd610fe..963a25f0 100644 --- a/software/python-package/tests/test_cli_programmer.py +++ b/software/python-package/tests/test_cli_programmer.py @@ -1,6 +1,7 @@ from pathlib import Path import pytest +from click.testing import CliRunner from shepherd_sheep.cli import cli # NOTE: (almost) direct copy between shepherd-herd & python-package @@ -31,7 +32,11 @@ def fw_empty(tmp_path: Path) -> Path: @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_minimal(shepherd_up, cli_runner, fw_nrf: Path) -> None: +def test_cli_program_minimal( + shepherd_up: None, + cli_runner: CliRunner, + fw_nrf: Path, +) -> None: res = cli_runner.invoke( cli, [ @@ -46,7 +51,11 @@ def test_cli_program_minimal(shepherd_up, cli_runner, fw_nrf: Path) -> None: @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_swd_explicit(shepherd_up, cli_runner, fw_nrf: Path) -> None: +def test_cli_program_swd_explicit( + shepherd_up: None, + cli_runner: CliRunner, + fw_nrf: Path, +) -> None: res = cli_runner.invoke( cli, [ @@ -72,8 +81,8 @@ def test_cli_program_swd_explicit(shepherd_up, cli_runner, fw_nrf: Path) -> None @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_swd_explicit_short( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, fw_nrf: Path, ) -> None: res = cli_runner.invoke( @@ -100,7 +109,11 @@ def test_cli_program_swd_explicit_short( @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_sbw_explicit(shepherd_up, cli_runner, fw_msp: Path) -> None: +def test_cli_program_sbw_explicit( + shepherd_up: None, + cli_runner: CliRunner, + fw_msp: Path, +) -> None: res = cli_runner.invoke( cli, [ @@ -125,7 +138,9 @@ def test_cli_program_sbw_explicit(shepherd_up, cli_runner, fw_msp: Path) -> None @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_file_defective_a(shepherd_up, cli_runner, fw_empty: Path) -> None: +def test_cli_program_file_defective_a( + shepherd_up: None, cli_runner: CliRunner, fw_empty: Path +) -> None: res = cli_runner.invoke( cli, [ @@ -140,7 +155,9 @@ def test_cli_program_file_defective_a(shepherd_up, cli_runner, fw_empty: Path) - @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_file_defective_b(shepherd_up, cli_runner, tmp_path: Path) -> None: +def test_cli_program_file_defective_b( + shepherd_up: None, cli_runner: CliRunner, tmp_path: Path +) -> None: res = cli_runner.invoke( cli, [ @@ -155,7 +172,9 @@ def test_cli_program_file_defective_b(shepherd_up, cli_runner, tmp_path: Path) - @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_file_defective_c(shepherd_up, cli_runner, tmp_path: Path) -> None: +def test_cli_program_file_defective_c( + shepherd_up: None, cli_runner: CliRunner, tmp_path: Path +) -> None: res = cli_runner.invoke( cli, [ @@ -171,8 +190,8 @@ def test_cli_program_file_defective_c(shepherd_up, cli_runner, tmp_path: Path) - @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_datarate_invalid_a( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, fw_nrf: Path, ) -> None: res = cli_runner.invoke( @@ -192,8 +211,8 @@ def test_cli_program_datarate_invalid_a( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_datarate_invalid_b( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, fw_nrf: Path, ) -> None: res = cli_runner.invoke( @@ -212,7 +231,9 @@ def test_cli_program_datarate_invalid_b( @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_program_target_invalid(shepherd_up, cli_runner, fw_nrf: Path) -> None: +def test_cli_program_target_invalid( + shepherd_up: None, cli_runner: CliRunner, fw_nrf: Path +) -> None: res = cli_runner.invoke( cli, [ diff --git a/software/python-package/tests/test_cli_run_task.py b/software/python-package/tests/test_cli_run_task.py index 657d8ee6..40dda59b 100644 --- a/software/python-package/tests/test_cli_run_task.py +++ b/software/python-package/tests/test_cli_run_task.py @@ -16,6 +16,7 @@ import numpy as np import pytest +from click.testing import CliRunner from pydantic import ValidationError from shepherd_core import CalibrationHarvester from shepherd_core import local_tz @@ -72,8 +73,8 @@ def path_here() -> Path: @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_harvest_no_cal( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_yaml: Path, path_h5: Path, ) -> None: @@ -92,8 +93,8 @@ def test_cli_harvest_no_cal( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_harvest_parameters_most( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_yaml: Path, path_h5: Path, ) -> None: @@ -114,8 +115,8 @@ def test_cli_harvest_parameters_most( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_harvest_parameters_minimal( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_yaml: Path, path_h5: Path, ) -> None: @@ -132,7 +133,11 @@ def test_cli_harvest_parameters_minimal( @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_harvest_preconfigured(shepherd_up, cli_runner, path_here: Path) -> None: +def test_cli_harvest_preconfigured( + shepherd_up: None, + cli_runner: CliRunner, + path_here: Path, +) -> None: file_path = path_here / "_test_config_harvest.yaml" res = cli_runner.invoke(cli, ["run", file_path.as_posix()]) assert res.exit_code == 0 @@ -141,8 +146,8 @@ def test_cli_harvest_preconfigured(shepherd_up, cli_runner, path_here: Path) -> @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_harvest_preconf_etc_shp_examples( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, path_here: Path, ) -> None: file_path = path_here.parent / "example_config_harvest.yaml" @@ -153,8 +158,8 @@ def test_cli_harvest_preconf_etc_shp_examples( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -181,8 +186,8 @@ def test_cli_emulate( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_with_custom_virtsource( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -214,8 +219,8 @@ def test_cli_emulate_with_custom_virtsource( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_with_bq25570( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -243,8 +248,8 @@ def test_cli_emulate_with_bq25570( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_aux_voltage( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -272,8 +277,8 @@ def test_cli_emulate_aux_voltage( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_parameters_long( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -309,8 +314,8 @@ def test_cli_emulate_parameters_long( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_parameters_minimal( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -332,7 +337,9 @@ def test_cli_emulate_parameters_minimal( @pytest.mark.hardware @pytest.mark.timeout(60) -def test_cli_emulate_preconfigured(shepherd_up, cli_runner, path_here: Path) -> None: +def test_cli_emulate_preconfigured( + shepherd_up: None, cli_runner: CliRunner, path_here: Path +) -> None: file_path = path_here / "_test_config_emulation.yaml" res = cli_runner.invoke(cli, ["run", file_path.as_posix()]) assert res.exit_code == 0 @@ -341,8 +348,8 @@ def test_cli_emulate_preconfigured(shepherd_up, cli_runner, path_here: Path) -> @pytest.mark.hardware @pytest.mark.timeout(80) def test_cli_emulate_preconf_etc_shp_examples( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, path_here: Path, ) -> None: file_path = path_here.parent / "example_config_emulation.yaml" @@ -353,8 +360,8 @@ def test_cli_emulate_preconf_etc_shp_examples( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_aux_voltage_fail( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, data_h5: Path, path_h5: Path, tmp_yaml: Path, @@ -380,8 +387,8 @@ def test_cli_emulate_aux_voltage_fail( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_fw_mod_task( - shepherd_up, - cli_runner, + shepherd_up: None, + cli_runner: CliRunner, tmp_path: Path, path_here: Path, ) -> None: @@ -415,8 +422,7 @@ def test_cli_fw_mod_task( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_programming( - shepherd_up, - cli_runner, + shepherd_up: None, path_here: Path, tmp_path: Path, ) -> None: @@ -429,7 +435,7 @@ def test_cli_programming( simulate=True, verbose=4, ).to_file(path_yaml) - res = cli_runner.invoke( + res = CliRunner().invoke( cli, [ "run", diff --git a/software/python-package/tests/test_datalogging.py b/software/python-package/tests/test_datalogging.py index a81a66de..3f513711 100644 --- a/software/python-package/tests/test_datalogging.py +++ b/software/python-package/tests/test_datalogging.py @@ -44,7 +44,7 @@ def cal_cape() -> CalibrationCape: @pytest.mark.parametrize("mode", ["harvester"]) -def test_create_h5writer(mode, tmp_path: Path, cal_cape: CalibrationCape) -> None: +def test_create_h5writer(mode: str, tmp_path: Path, cal_cape: CalibrationCape) -> None: d = tmp_path / f"{ mode }.h5" h = Writer(file_path=d, cal_data=cal_cape[mode], mode=mode) # assert not exists @@ -76,9 +76,9 @@ def test_create_h5writer_with_force(tmp_path: Path, cal_cape: CalibrationCape) - @pytest.mark.parametrize("mode", ["harvester"]) def test_h5writer_data( - mode, + mode: str, tmp_path: Path, - data_buffer, + data_buffer: DataBuffer, cal_cape: CalibrationCape, ) -> None: d = tmp_path / "harvest.h5" @@ -95,7 +95,11 @@ def test_h5writer_data( @pytest.mark.parametrize("mode", ["harvester"]) -def test_calibration_logging(mode, tmp_path: Path, cal_cape: CalibrationCape) -> None: +def test_calibration_logging( + mode: str, + tmp_path: Path, + cal_cape: CalibrationCape, +) -> None: d = tmp_path / "recording.h5" with Writer(file_path=d, mode=mode, cal_data=cal_cape.harvester) as _: pass diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index d464c645..5d0cbcf3 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -1,3 +1,5 @@ +from collections.abc import Generator + import pytest from shepherd_core import CalibrationCape from shepherd_core.data_models.base.calibration import CapeData @@ -20,7 +22,7 @@ def data_test_string() -> bytes: @pytest.fixture() -def eeprom_open(request, fake_hardware): +def eeprom_open(request, fake_hardware) -> Generator[EEPROM, None, None]: if fake_hardware is not None: fake_hardware.create_file("/sys/bus/i2c/devices/2-0054/eeprom", st_size=32768) request.applymarker( @@ -34,7 +36,7 @@ def eeprom_open(request, fake_hardware): @pytest.fixture() -def eeprom_retained(eeprom_open): +def eeprom_retained(eeprom_open: EEPROM) -> Generator[EEPROM, None, None]: data = eeprom_open._read(0, 1024) for i in range(256): eeprom_open._write(i * 4, b"\xDE\xAD\xBE\xEF") @@ -43,29 +45,31 @@ def eeprom_retained(eeprom_open): @pytest.fixture() -def eeprom_with_data(eeprom_retained: EEPROM, cape_data: CapeData) -> EEPROM: +def eeprom_with_data( + eeprom_retained: EEPROM, cape_data: CapeData +) -> Generator[EEPROM, None, None]: eeprom_retained._write_cape_data(cape_data) - return eeprom_retained + yield eeprom_retained @pytest.fixture() def eeprom_with_calibration( eeprom_retained: EEPROM, cal_cape: CalibrationCape, -) -> EEPROM: +) -> Generator[EEPROM, None, None]: eeprom_retained.write_calibration(cal_cape) - return eeprom_retained + yield eeprom_retained @pytest.mark.eeprom_write @pytest.mark.hardware -def test_read_raw(eeprom_open) -> None: +def test_read_raw(eeprom_open: EEPROM) -> None: eeprom_open._read(0, 4) @pytest.mark.eeprom_write @pytest.mark.hardware -def test_write_raw(eeprom_retained, data_test_string) -> None: +def test_write_raw(eeprom_retained: EEPROM, data_test_string: bytes) -> None: eeprom_retained._write(0, data_test_string) data = eeprom_retained._read(0, len(data_test_string)) assert data == data_test_string @@ -73,7 +77,7 @@ def test_write_raw(eeprom_retained, data_test_string) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_read_value(eeprom_with_data, cape_data: CapeData) -> None: +def test_read_value(eeprom_with_data: EEPROM, cape_data: CapeData) -> None: with pytest.raises(KeyError): _ = eeprom_with_data["some non-sense parameter"] assert eeprom_with_data["version"] == cape_data["version"] @@ -81,7 +85,7 @@ def test_read_value(eeprom_with_data, cape_data: CapeData) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_write_value(eeprom_retained, cape_data) -> None: +def test_write_value(eeprom_retained: EEPROM, cape_data: CapeData) -> None: with pytest.raises(KeyError): eeprom_retained["some non-sense parameter"] = "some data" @@ -91,7 +95,7 @@ def test_write_value(eeprom_retained, cape_data) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_write_capedata(eeprom_retained, cape_data) -> None: +def test_write_capedata(eeprom_retained: EEPROM, cape_data: CapeData) -> None: eeprom_retained._write_cape_data(cape_data) for key, value in cape_data.items(): if isinstance(value, str): @@ -102,7 +106,7 @@ def test_write_capedata(eeprom_retained, cape_data) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_read_capedata(eeprom_with_data, cape_data: CapeData) -> None: +def test_read_capedata(eeprom_with_data: EEPROM, cape_data: CapeData) -> None: cape_data = eeprom_with_data._read_cape_data() for key in cape_data.keys(): assert cape_data[key] == cape_data[key] @@ -110,7 +114,7 @@ def test_read_capedata(eeprom_with_data, cape_data: CapeData) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_write_calibration(eeprom_retained, cal_cape: CalibrationCape) -> None: +def test_write_calibration(eeprom_retained: EEPROM, cal_cape: CalibrationCape) -> None: eeprom_retained.write_calibration(cal_cape) cal_restored = eeprom_retained.read_calibration() for component in ["harvester", "emulator"]: @@ -119,7 +123,10 @@ def test_write_calibration(eeprom_retained, cal_cape: CalibrationCape) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware -def test_read_calibration(eeprom_with_calibration, cal_cape: CalibrationCape) -> None: +def test_read_calibration( + eeprom_with_calibration: EEPROM, + cal_cape: CalibrationCape, +) -> None: cal_restored = eeprom_with_calibration.read_calibration() for component in ["harvester", "emulator"]: assert cal_restored[component].get_hash() == cal_cape[component].get_hash() diff --git a/software/python-package/tests/test_emulation.py b/software/python-package/tests/test_emulation.py index 6e1eeebb..ef5f6d25 100644 --- a/software/python-package/tests/test_emulation.py +++ b/software/python-package/tests/test_emulation.py @@ -1,4 +1,5 @@ import time +from collections.abc import Generator from pathlib import Path import h5py @@ -19,7 +20,7 @@ from shepherd_sheep.shared_memory import DataBuffer -def random_data(length) -> np.ndarray: +def random_data(length: int) -> np.ndarray: rng = np.random.default_rng() return rng.integers(low=0, high=2**18, size=length, dtype="u4") @@ -49,44 +50,44 @@ def data_h5(tmp_path: Path) -> Path: @pytest.fixture() -def writer(tmp_path: Path): +def writer(tmp_path: Path) -> Generator[Writer, None, None]: cal = CalibrationCape().emulator with Writer( force_overwrite=True, file_path=tmp_path / "test.h5", mode="emulator", cal_data=cal, - ) as lw: - yield lw + ) as _w: + yield _w @pytest.fixture() -def shp_reader(data_h5: Path): - with CoreReader(data_h5) as lr: - yield lr +def shp_reader(data_h5: Path) -> Generator[CoreReader, None, None]: + with CoreReader(data_h5) as _r: + yield _r @pytest.fixture() def emulator( - request, - shepherd_up, + shepherd_up: None, data_h5: Path, src_cfg: VirtualSourceConfig, -) -> ShepherdEmulator: +) -> Generator[ShepherdEmulator, None, None]: cfg_emu = EmulationTask( input_path=data_h5, virtual_source=src_cfg, verbose=3, ) - emu = ShepherdEmulator(cfg_emu) - request.addfinalizer(emu.__del__) - emu.__enter__() - request.addfinalizer(emu.__exit__) - return emu + with ShepherdEmulator(cfg_emu) as _e: + yield _e @pytest.mark.hardware -def test_emulation(writer, shp_reader, emulator: ShepherdEmulator) -> None: +def test_emulation( + writer: Writer, + shp_reader: CoreReader, + emulator: ShepherdEmulator, +) -> None: emulator.start(wait_blocking=False) fifo_buffer_size = sysfs_interface.get_n_buffers() emulator.wait_for_start(15) @@ -101,11 +102,11 @@ def test_emulation(writer, shp_reader, emulator: ShepherdEmulator) -> None: writer.write_buffer(emu_buf) with pytest.raises(ShepherdIOException): - idx, emu_buf = emulator.get_buffer() + _, _ = emulator.get_buffer() @pytest.mark.hardware -def test_emulate_fn(tmp_path: Path, data_h5: Path, shepherd_up) -> None: +def test_emulate_fn(tmp_path: Path, data_h5: Path, shepherd_up: None) -> None: output = tmp_path / "rec.h5" start_time = round(time.time() + 10) emu_cfg = EmulationTask( @@ -133,44 +134,42 @@ def test_emulate_fn(tmp_path: Path, data_h5: Path, shepherd_up) -> None: @pytest.mark.hardware @pytest.mark.skip(reason="REQUIRES CAPE HARDWARE v2.4") # real cape needed -def test_target_pins(shepherd_up) -> None: - shepherd_io = ShepherdDebug() - shepherd_io.__enter__() - shepherd_io.start() - shepherd_io.select_port_for_power_tracking(TargetPort.A) - - dac_channels = [ - # combination of debug channel number, voltage_index, cal_component, cal_channel - [1, "harvester", "dac_voltage_a", "Harvester VSimBuf"], - [2, "harvester", "dac_voltage_b", "Harvester VMatching"], - [4, "emulator", "dac_voltage_a", "Emulator Rail A"], - [8, "emulator", "dac_voltage_b", "Emulator Rail B"], - ] - - # channels: 5&6 are UART, can only be used when free, 7&8 are SWD - gpio_channels = [0, 1, 2, 3, 4, 7, 8] - # response: corresponding to r31_num (and later 2^num) - pru_responses = [0, 1, 6, 7, 8, 2, 3] - - for channel in [2, 3]: - dac_cfg = dac_channels[channel] - value_raw = shepherd_io.convert_value_to_raw(dac_cfg[1], dac_cfg[2], 2.0) - shepherd_io.dac_write(dac_cfg[0], value_raw) - - shepherd_io.set_io_level_converter(True) - - shepherd_io.select_port_for_io_interface(TargetPort.A) - - for io_index, io_channel in enumerate(gpio_channels): - shepherd_io.set_gpio_one_high(io_channel) - response = int(shepherd_io.gpi_read()) - assert response & (2 ** pru_responses[io_index]) - - shepherd_io.select_port_for_io_interface(TargetPort.B) - - for io_index, io_channel in enumerate(gpio_channels): - shepherd_io.set_gpio_one_high(io_channel) - response = int(shepherd_io.gpi_read()) - assert response & (2 ** pru_responses[io_index]) - +def test_target_pins(shepherd_up: None) -> None: + with ShepherdDebug() as shepherd_io: + shepherd_io.start() + shepherd_io.select_port_for_power_tracking(TargetPort.A) + + dac_channels = [ + # combination of debug channel number, voltage_index, cal_component, cal_channel + [1, "harvester", "dac_voltage_a", "Harvester VSimBuf"], + [2, "harvester", "dac_voltage_b", "Harvester VMatching"], + [4, "emulator", "dac_voltage_a", "Emulator Rail A"], + [8, "emulator", "dac_voltage_b", "Emulator Rail B"], + ] + + # channels: 5&6 are UART, can only be used when free, 7&8 are SWD + gpio_channels = [0, 1, 2, 3, 4, 7, 8] + # response: corresponding to r31_num (and later 2^num) + pru_responses = [0, 1, 6, 7, 8, 2, 3] + + for channel in [2, 3]: + dac_cfg = dac_channels[channel] + value_raw = shepherd_io.convert_value_to_raw(dac_cfg[1], dac_cfg[2], 2.0) + shepherd_io.dac_write(dac_cfg[0], value_raw) + + shepherd_io.set_io_level_converter(True) + + shepherd_io.select_port_for_io_interface(TargetPort.A) + + for io_index, io_channel in enumerate(gpio_channels): + shepherd_io.set_gpio_one_high(io_channel) + response = int(shepherd_io.gpi_read()) + assert response & (2 ** pru_responses[io_index]) + + shepherd_io.select_port_for_io_interface(TargetPort.B) + + for io_index, io_channel in enumerate(gpio_channels): + shepherd_io.set_gpio_one_high(io_channel) + response = int(shepherd_io.gpi_read()) + assert response & (2 ** pru_responses[io_index]) # TODO: could add a loopback for uart, but extra hardware is needed for that diff --git a/software/python-package/tests/test_harvest.py b/software/python-package/tests/test_harvest.py index 46353bc6..8eddf533 100644 --- a/software/python-package/tests/test_harvest.py +++ b/software/python-package/tests/test_harvest.py @@ -1,4 +1,5 @@ import time +from collections.abc import Generator from pathlib import Path import h5py @@ -17,38 +18,37 @@ def mode(request) -> str: @pytest.fixture() -def writer(tmp_path: Path, mode: str): +def writer(tmp_path: Path, mode: str) -> Generator[Writer, None, None]: with Writer( mode=mode, cal_data=CalibrationHarvester(), force_overwrite=True, file_path=tmp_path / "test.h5", - ) as lw: - yield lw + ) as _w: + yield _w @pytest.fixture() -def harvester(request, shepherd_up, mode: str, tmp_path: Path) -> ShepherdHarvester: +def harvester( + shepherd_up: None, + mode: str, + tmp_path: Path, +) -> Generator[ShepherdHarvester, None, None]: cfg = HarvestTask(output_path=tmp_path / "hrv_123.h5") - rec = ShepherdHarvester(cfg=cfg, mode=mode) - request.addfinalizer(rec.__del__) - rec.__enter__() - request.addfinalizer(rec.__exit__) - return rec + with ShepherdHarvester(cfg=cfg, mode=mode) as _h: + yield _h @pytest.mark.hardware -def test_instantiation(shepherd_up, tmp_path: Path) -> None: +def test_instantiation(shepherd_up: None, tmp_path: Path) -> None: cfg = HarvestTask(output_path=tmp_path / "hrv_123.h5") - rec = ShepherdHarvester(cfg) - rec.__enter__() - assert rec is not None - rec.__exit__() - del rec + with ShepherdHarvester(cfg) as _h: + assert _h is not None + del _h @pytest.mark.hardware -def test_harvester(writer, harvester: ShepherdHarvester) -> None: +def test_harvester(writer: Writer, harvester: ShepherdHarvester) -> None: harvester.start(wait_blocking=False) harvester.wait_for_start(15) @@ -60,7 +60,7 @@ def test_harvester(writer, harvester: ShepherdHarvester) -> None: @pytest.mark.hardware # TODO extend with new harvester-options @pytest.mark.timeout(40) -def test_harvester_fn(tmp_path, shepherd_up) -> None: +def test_harvester_fn(tmp_path: Path, shepherd_up: None) -> None: path = tmp_path / "rec.h5" time_start = int(time.time() + 10) cfg = HarvestTask( diff --git a/software/python-package/tests/test_sysfs_interface.py b/software/python-package/tests/test_sysfs_interface.py index 37c2cd0a..3e1c9b5f 100644 --- a/software/python-package/tests/test_sysfs_interface.py +++ b/software/python-package/tests/test_sysfs_interface.py @@ -29,7 +29,7 @@ def hrv_cfg() -> HarvesterPRUConfig: @pytest.fixture() -def shepherd_running(shepherd_up) -> None: +def shepherd_running(shepherd_up: None) -> None: sysfs_interface.set_start() sysfs_interface.wait_for_state("running", 5) @@ -41,20 +41,20 @@ def cal4sysfs() -> dict: @pytest.mark.parametrize("attr", sysfs_interface.attribs) -def test_getters(shepherd_up, attr) -> None: +def test_getters(shepherd_up: None, attr: str) -> None: method_to_call = getattr(sysfs_interface, f"get_{ attr }") assert method_to_call() is not None @pytest.mark.parametrize("attr", sysfs_interface.attribs) -def test_getters_fail(shepherd_down, attr) -> None: +def test_getters_fail(shepherd_down: None, attr: str) -> None: method_to_call = getattr(sysfs_interface, f"get_{ attr }") with pytest.raises(FileNotFoundError): method_to_call() @pytest.mark.hardware -def test_start(shepherd_up) -> None: +def test_start(shepherd_up: None) -> None: sysfs_interface.set_start() time.sleep(5) assert sysfs_interface.get_state() == "running" @@ -63,7 +63,7 @@ def test_start(shepherd_up) -> None: @pytest.mark.hardware -def test_wait_for_state(shepherd_up) -> None: +def test_wait_for_state(shepherd_up: None) -> None: sysfs_interface.set_start() assert sysfs_interface.wait_for_state("running", 3) < 3 sysfs_interface.set_stop() @@ -71,7 +71,7 @@ def test_wait_for_state(shepherd_up) -> None: @pytest.mark.hardware -def test_start_delayed(shepherd_up) -> None: +def test_start_delayed(shepherd_up: None) -> None: start_time = int(time.time() + 5) sysfs_interface.set_start(start_time) @@ -86,29 +86,29 @@ def test_start_delayed(shepherd_up) -> None: @pytest.mark.parametrize("mode", ["harvester", "emulator"]) -def test_set_mode(shepherd_up, mode) -> None: +def test_set_mode(shepherd_up: None, mode: str) -> None: sysfs_interface.write_mode(mode) assert sysfs_interface.get_mode() == mode -def test_initial_mode(shepherd_up) -> None: +def test_initial_mode(shepherd_up: None) -> None: # NOTE: initial config is set in main() of pru0 assert sysfs_interface.get_mode() == "harvester" @pytest.mark.hardware -def test_set_mode_fail_offline(shepherd_running) -> None: +def test_set_mode_fail_offline(shepherd_running: None) -> None: with pytest.raises(sysfs_interface.SysfsInterfaceException): sysfs_interface.write_mode("harvester") -def test_set_mode_fail_invalid(shepherd_up): +def test_set_mode_fail_invalid(shepherd_up: None) -> None: with pytest.raises(sysfs_interface.SysfsInterfaceException): sysfs_interface.write_mode("invalidmode") @pytest.mark.parametrize("value", [0, 0.1, 3.2]) -def test_dac_aux_voltage(shepherd_up, value): +def test_dac_aux_voltage(shepherd_up: None, value: float) -> None: cal_emu = CalibrationEmulator() msb_threshold = cal_emu.dac_V_A.raw_to_si(2) sysfs_interface.write_dac_aux_voltage(value, cal_emu) @@ -116,23 +116,23 @@ def test_dac_aux_voltage(shepherd_up, value): @pytest.mark.parametrize("value", [0, 100, 16000]) -def test_dac_aux_voltage_raw(shepherd_up, value): +def test_dac_aux_voltage_raw(shepherd_up: None, value: int) -> None: sysfs_interface.write_dac_aux_voltage_raw(value) assert sysfs_interface.read_dac_aux_voltage_raw() == value -def test_initial_aux_voltage(shepherd_up): +def test_initial_aux_voltage(shepherd_up: None) -> None: # NOTE: initial config is set in main() of pru0 assert sysfs_interface.read_dac_aux_voltage_raw() == 0 -def test_calibration_settings(shepherd_up, cal4sysfs: dict): +def test_calibration_settings(shepherd_up: None, cal4sysfs: dict) -> None: sysfs_interface.write_calibration_settings(cal4sysfs) assert sysfs_interface.read_calibration_settings() == cal4sysfs @pytest.mark.hardware -def test_initial_calibration_settings(shepherd_up, cal4sysfs): +def test_initial_calibration_settings(shepherd_up: None, cal4sysfs: dict) -> None: # NOTE: initial config is in common_inits.h of kernel-module cal4sysfs["adc_current_gain"] = 255 cal4sysfs["adc_current_offset"] = -1 @@ -144,13 +144,16 @@ def test_initial_calibration_settings(shepherd_up, cal4sysfs): @pytest.mark.hardware -def test_initial_harvester_settings(shepherd_up): +def test_initial_harvester_settings(shepherd_up: None) -> None: hrv_list = [0, *list(range(200, 211))] assert sysfs_interface.read_virtual_harvester_settings() == hrv_list @pytest.mark.hardware # TODO: could also run with fakehardware, but triggers pydantic-error -def test_writing_harvester_settings(shepherd_up, hrv_cfg): +def test_writing_harvester_settings( + shepherd_up: None, + hrv_cfg: HarvesterPRUConfig, +) -> None: sysfs_interface.write_virtual_harvester_settings(hrv_cfg) assert sysfs_interface.read_virtual_harvester_settings() == list( hrv_cfg.model_dump().values(), @@ -158,7 +161,7 @@ def test_writing_harvester_settings(shepherd_up, hrv_cfg): @pytest.mark.hardware -def test_initial_virtsource_settings(shepherd_up): +def test_initial_virtsource_settings(shepherd_up: None) -> None: # NOTE: initial config is set in main() of pru0 vsource_settings = [ list(range(100, 124)), @@ -169,7 +172,10 @@ def test_initial_virtsource_settings(shepherd_up): assert sysfs_interface.read_virtual_converter_settings() == values_1d -def test_writing_virtsource_settings(shepherd_up, cnv_cfg): +def test_writing_virtsource_settings( + shepherd_up: None, + cnv_cfg: ConverterPRUConfig, +) -> None: sysfs_interface.write_virtual_converter_settings(cnv_cfg) values_1d = flatten_list(list(cnv_cfg.model_dump().values())) assert sysfs_interface.read_virtual_converter_settings() == values_1d diff --git a/software/python-package/tests/test_virtual_source.py b/software/python-package/tests/test_virtual_source.py index 7f183680..cdeab281 100644 --- a/software/python-package/tests/test_virtual_source.py +++ b/software/python-package/tests/test_virtual_source.py @@ -1,3 +1,4 @@ +from collections.abc import Generator from pathlib import Path import pytest @@ -33,25 +34,21 @@ def cal_cape() -> CalibrationCape: @pytest.fixture() def pru_vsource( - request, - shepherd_up, + shepherd_up: None, src_cfg: VirtualSourceConfig, cal_cape: CalibrationCape, dtype_in: EnergyDType = EnergyDType.ivsample, window_size: int = 0, -) -> ShepherdDebug: - pru = ShepherdDebug() - request.addfinalizer(pru.__del__) - pru.__enter__() - request.addfinalizer(pru.__exit__) - pru.vsource_init( - src_cfg=src_cfg, - cal_emu=cal_cape.emulator, - log_intermediate=False, - dtype_in=dtype_in, - window_size=window_size, - ) # TODO: extend to be real vsource - return pru +) -> Generator[ShepherdDebug, None, None]: + with ShepherdDebug() as _d: + _d.vsource_init( + src_cfg=src_cfg, + cal_emu=cal_cape.emulator, + log_intermediate=False, + dtype_in=dtype_in, + window_size=window_size, + ) # TODO: extend to be real vsource + yield _d @pytest.fixture diff --git a/software/python-package/tests_manual/testbench_gen_fake.py b/software/python-package/tests_manual/testbench_gen_fake.py index a278fb77..892295d3 100644 --- a/software/python-package/tests_manual/testbench_gen_fake.py +++ b/software/python-package/tests_manual/testbench_gen_fake.py @@ -10,7 +10,7 @@ store_path = tmp_path / "harvest_example.h5" -def random_data(length): +def random_data(length: int) -> np.ndarray: rng = np.random.default_rng() return rng.integers(low=0, high=2**18, size=length, dtype="u4") diff --git a/software/shepherd-calibration/shepherd_cal/calibrator.py b/software/shepherd-calibration/shepherd_cal/calibrator.py index 74580dc5..a3299b53 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator.py @@ -51,7 +51,7 @@ def __init__( smu_ip: str | None = None, mode_4wire: bool = True, pwrline_cycles: float = 16, - ): + ) -> None: fabric_args: dict[str, str] = {} if password is not None: fabric_args["password"] = password @@ -77,7 +77,7 @@ def __init__( if self.kth is not None: self.kth.reset() - def __del__(self): + def __del__(self) -> None: # ... overcautious self._cnx.sudo("systemctl stop shepherd-rpc", hide=True, warn=True) self._cnx.close() @@ -123,7 +123,7 @@ def set_smu_to_isource( return value_i @staticmethod - def reject_outliers(data: np.ndarray, m: float = 2.0): + def reject_outliers(data: np.ndarray, m: float = 2.0) -> np.ndarray: d = np.abs(data - np.median(data)) mdev = np.median(d) s = d / mdev if mdev else 0.0 @@ -381,7 +381,7 @@ def measure_emulator(self) -> CalMeasurementEmulator: def write( self, cal_file: str | Path, - ): + ) -> None: temp_file = "/tmp/calib.yaml" # noqa: S108 if isinstance(cal_file, str): cal_file = Path(cal_file) @@ -400,7 +400,7 @@ def write( logger.info(result.stderr) logger.info("---------------------------------") - def read(self): + def read(self) -> None: logger.info("----------EEPROM READ------------") result = self._cnx.sudo( "shepherd-sheep --verbose eeprom read", @@ -411,7 +411,7 @@ def read(self): logger.info(result.stderr) logger.info("---------------------------------") - def retrieve(self, cal_file: Path): + def retrieve(self, cal_file: Path) -> None: temp_file = "/tmp/calib.yaml" # noqa: S108 result = self._cnx.sudo( f"shepherd-sheep --verbose eeprom read -c {temp_file}", diff --git a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py index 9294eb04..20b78037 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator_cli.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator_cli.py @@ -63,7 +63,7 @@ def measure( write: bool = write_opt_t, version: str | None = version_opt_t, verbose: bool = verbose_opt_t, -): +) -> None: """Measure calibration-data for shepherd cape""" cli_setup_callback(verbose) smu_4wire = not smu_2wire @@ -135,7 +135,7 @@ def write( cal_file: Path | None = ifile_opt_t, measurement_file: Path | None = ifile_opt_t, verbose: bool = verbose_opt_t, -): +) -> None: """Write calibration-data to shepherd cape eeprom (choose cal- or measurement-file)""" cli_setup_callback(verbose) if not any([cal_file, measurement_file]): @@ -163,7 +163,7 @@ def read( password: str | None = pass_opt_t, cal_file: Path | None = ofile_opt_t, verbose: bool = verbose_opt_t, -): +) -> None: """Read calibration-data from shepherd cape""" cli_setup_callback(verbose) shpcal = Calibrator(host, user, password) diff --git a/software/shepherd-calibration/shepherd_cal/cli_helper.py b/software/shepherd-calibration/shepherd_cal/cli_helper.py index 1c5fadf1..458ad826 100644 --- a/software/shepherd-calibration/shepherd_cal/cli_helper.py +++ b/software/shepherd-calibration/shepherd_cal/cli_helper.py @@ -10,7 +10,7 @@ from .logger import set_verbosity -def exit_gracefully(*_args): # type: ignore +def exit_gracefully(*_args) -> None: logger.warning("Aborted!") sys.exit(0) diff --git a/software/shepherd-calibration/shepherd_cal/profile.py b/software/shepherd-calibration/shepherd_cal/profile.py index cb3f1e4b..2cfcc585 100644 --- a/software/shepherd-calibration/shepherd_cal/profile.py +++ b/software/shepherd-calibration/shepherd_cal/profile.py @@ -37,7 +37,7 @@ class Profile: - def __init__(self, file: Path): + def __init__(self, file: Path) -> None: if not isinstance(file, Path): file = Path(file) if file.suffix != ".npz": @@ -156,7 +156,7 @@ def _prepare_results(self, component: str, data: pd.DataFrame) -> None: ) self.results[component] = result - def _prepare_filters(self, component: str): + def _prepare_filters(self, component: str) -> None: data = self.data[component] filter_c = (data["c_ref_A"] >= 3e-6) & (data["c_ref_A"] <= 40e-3) filter_v = (data.v_shp_V >= 1.0) & (data.v_shp_V <= 3.9) @@ -194,7 +194,7 @@ def _prepare_stats(self, component: str, data: pd.DataFrame) -> None: def get_stats(self) -> pd.DataFrame: return pd.concat(self.stats, axis=0, ignore_index=True) - def scatter_setpoints_stddev(self, component: str, filtered: bool = False): + def scatter_setpoints_stddev(self, component: str, filtered: bool = False) -> None: data = self.results[component] if filtered: data = data[self.res_filters[component]] @@ -237,7 +237,7 @@ def scatter_setpoints_stddev(self, component: str, filtered: bool = False): plt.close(fig) plt.clf() - def scatter_setpoints_dynamic(self, component: str, filtered: bool = False): + def scatter_setpoints_dynamic(self, component: str, filtered: bool = False) -> None: data = self.results[component] if filtered: data = data[self.res_filters[component]] @@ -290,7 +290,7 @@ def scatter_setpoints_dynamic(self, component: str, filtered: bool = False): plt.close(fig) plt.clf() - def quiver_setpoints_offset(self, component: str, filtered: bool = False): + def quiver_setpoints_offset(self, component: str, filtered: bool = False) -> None: data = self.results[component] if filtered: data = data[self.res_filters[component]] diff --git a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py index a76eb602..c779508b 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py +++ b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py @@ -9,7 +9,7 @@ def analyze_directory( folder_path: Path, stats_path: Path | None = None, - *, + *, do_plots: bool = False, ) -> None: stats_list = [] diff --git a/software/shepherd-calibration/shepherd_cal/profile_calibration.py b/software/shepherd-calibration/shepherd_cal/profile_calibration.py index af5e485a..1ad607e7 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_calibration.py +++ b/software/shepherd-calibration/shepherd_cal/profile_calibration.py @@ -1,3 +1,4 @@ +from typing import Self from typing import TypeVar import numpy as np @@ -14,7 +15,7 @@ class ProfileCalibration(CalibrationSeries): @classmethod - def from_measurement(cls, result: pd.DataFrame): + def from_measurement(cls, result: pd.DataFrame) -> Self: values: dict[str, CalibrationPair] = {} cal_c = cls._determine_current_cal(result) cal_v = cls._determine_voltage_cal(result) diff --git a/software/shepherd-calibration/shepherd_cal/profiler.py b/software/shepherd-calibration/shepherd_cal/profiler.py index 3ec939ec..7ea4539a 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler.py +++ b/software/shepherd-calibration/shepherd_cal/profiler.py @@ -38,7 +38,7 @@ class Profiler: - def __init__(self, calibrator: Calibrator, *, short: bool = False): + def __init__(self, calibrator: Calibrator, *, short: bool = False) -> None: self._cal: Calibrator = calibrator if short: diff --git a/software/shepherd-calibration/shepherd_cal/profiler_cli.py b/software/shepherd-calibration/shepherd_cal/profiler_cli.py index 0c9a58bb..f07eebda 100644 --- a/software/shepherd-calibration/shepherd_cal/profiler_cli.py +++ b/software/shepherd-calibration/shepherd_cal/profiler_cli.py @@ -66,7 +66,7 @@ def measure( cape_serial: str = serial_opt_t, quiet: bool = quiet_opt_t, verbose: bool = verbose_opt_t, -): +) -> None: """Measure profile-data for shepherd cape""" cli_setup_callback(verbose) if not any([harvester, emulator]): @@ -141,7 +141,7 @@ def analyze( outfile: Path | None = out_file_opt_t, plot: bool = plot_opt_t, verbose: bool = verbose_opt_t, -): +) -> None: """Analyze profile-data""" cli_setup_callback(verbose) analyze_directory(infiles, outfile, do_plots=plot) diff --git a/software/shepherd-devicetest/src/__init__.py b/software/shepherd-devicetest/src/__init__.py index 63bd5109..61d252bc 100644 --- a/software/shepherd-devicetest/src/__init__.py +++ b/software/shepherd-devicetest/src/__init__.py @@ -16,7 +16,7 @@ # - id= is now tag= -def assemble_window(): +def assemble_window() -> None: with dpg.window( tag="main", label="Shepherd Testing and Debug Tool", diff --git a/software/shepherd-devicetest/src/shepherd_callbacks.py b/software/shepherd-devicetest/src/shepherd_callbacks.py index f71aa463..fb2ff9ac 100644 --- a/software/shepherd-devicetest/src/shepherd_callbacks.py +++ b/software/shepherd-devicetest/src/shepherd_callbacks.py @@ -161,7 +161,7 @@ def shepherd_power_callback(sender, element_data, user_data) -> None: shepherd_io.set_shepherd_pcb_power(able_dict[element_data]) -def update_power_state_shepherd(): +def update_power_state_shepherd() -> None: if shepherd_io is not None: value = int(shepherd_io.get_power_state_shepherd()) dpg.set_value("shepherd_pwr", list(able_dict.keys())[value]) @@ -174,7 +174,7 @@ def shepherd_state_callback(sender, element_data, user_data) -> None: update_gui_elements() -def update_shepherd_state(): +def update_shepherd_state() -> None: dpg.set_value("shepherd_state", stateTrans_dict[shepherd_io.get_shepherd_state()]) @@ -183,7 +183,7 @@ def target_power_callback(sender, element_data, user_data) -> None: shepherd_io.select_port_for_power_tracking(port) -def update_target_power(): +def update_target_power() -> None: value = int(not shepherd_io.get_main_target_for_power()) dpg.set_value("target_pwr", list(tgt_dict)[value]) # TODO: twisted @@ -193,7 +193,7 @@ def target_io_callback(sender, element_data, user_data) -> None: shepherd_io.select_port_for_io_interface(port) -def update_target_io(): +def update_target_io() -> None: value = int(not shepherd_io.get_main_target_for_io()) dpg.set_value("target_io", list(tgt_dict)[value]) # TODO: twisted @@ -203,7 +203,7 @@ def io_level_converter_callback(sender, element_data, user_data) -> None: shepherd_io.set_io_level_converter(state) -def update_io_level_state(): +def update_io_level_state() -> None: value = int(shepherd_io.get_target_io_level_conv()) dpg.set_value("io_lvl_converter", list(able_dict.keys())[value]) diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 17cc78b2..25fcd2a1 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -8,7 +8,10 @@ from datetime import timedelta from io import StringIO from pathlib import Path +from types import TracebackType +from typing import Any from typing import ClassVar +from typing import Self import yaml from fabric import Connection @@ -47,7 +50,7 @@ def __init__( limit: str | None = None, user: str | None = None, key_filepath: Path | None = None, - ): + ) -> None: limits_list: list[str] | None = None if isinstance(limit, str): limits_list = limit.split(",") @@ -130,7 +133,7 @@ def __init__( logger.info("Herd consists of %d sheep", len(self.group)) - def __del__(self): + def __del__(self) -> None: # ... overcautious closing of connections if not hasattr(self, "group") or not isinstance(self.group, Group): return @@ -139,25 +142,31 @@ def __del__(self): cnx.close() del cnx - def __enter__(self): + def __enter__(self) -> Self: self._open() if len(self.group) < 1: raise ValueError("No remote sheep in current herd!") return self - def __exit__(self, *args): # type: ignore + def __exit__( + self, + typ: type[BaseException] | None = None, + exc: BaseException | None = None, + tb: TracebackType | None = None, + extra_arg: int = 0, + ) -> None: if not hasattr(self, "group") or not isinstance(self.group, Group): return with contextlib.suppress(TypeError): for cnx in self.group: cnx.close() - def __getitem__(self, key: str): + def __getitem__(self, key: str) -> Any: if key in self.hostnames: return self.hostnames[key] raise KeyError - def __repr__(self): + def __repr__(self) -> dict: return self.hostnames @staticmethod @@ -230,7 +239,12 @@ def run_cmd(self, cmd: str, *, sudo: bool = False) -> dict[int, Result]: raise RuntimeError("ZERO nodes answered - check your config") return results - def print_output(self, replies: dict[int, Result], *, verbose: bool = False) -> None: + def print_output( + self, + replies: dict[int, Result], + *, + verbose: bool = False, + ) -> None: """Logs output-results of shell commands""" for i, hostname in enumerate(self.hostnames.values()): if not isinstance(replies.get(i), Result): @@ -251,7 +265,7 @@ def _thread_put( src: Path | StringIO, dst: Path, force_overwrite: bool, # noqa: FBT001 - ): + ) -> None: if isinstance(src, StringIO): filename = dst.name else: @@ -281,7 +295,8 @@ def _thread_put( def put_file( self, src: StringIO | Path | str, - dst: Path | str, *, + dst: Path | str, + *, force_overwrite: bool = False, ) -> None: if isinstance(src, StringIO): @@ -319,7 +334,7 @@ def put_file( del thread # ... overcautious @staticmethod - def _thread_get(cnx: Connection, src: Path, dst: Path): + def _thread_get(cnx: Connection, src: Path, dst: Path) -> None: if not cnx.is_connected: return try: @@ -336,7 +351,8 @@ def _thread_get(cnx: Connection, src: Path, dst: Path): def get_file( self, src: Path | str, - dst_dir: Path | str, *, + dst_dir: Path | str, + *, timestamp: bool = False, separate: bool = False, delete_src: bool = False, @@ -613,7 +629,8 @@ def run_task(self, config: Path | ShpModel, *, attach: bool = False) -> int: def get_task_files( self, config: Path | ShpModel, - dst_dir: Path | str, *, + dst_dir: Path | str, + *, separate: bool = False, delete_src: bool = False, ) -> bool: @@ -629,7 +646,12 @@ def get_task_files( for task in tasks: if hasattr(task, "output_path"): logger.info("General remote path is: %s", task.output_path) - failed |= self.get_file(task.output_path, dst_dir, separate=separate, delete_src=delete_src) + failed |= self.get_file( + task.output_path, + dst_dir, + separate=separate, + delete_src=delete_src, + ) if hasattr(task, "get_output_paths"): for host, path in task.get_output_paths().items(): logger.info("Remote path of '%s' is: %s, WON'T COPY", host, path) diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index bfad063e..327143a8 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -2,6 +2,8 @@ import sys from datetime import datetime from pathlib import Path +from typing import TypedDict +from typing import Unpack import click import shepherd_core @@ -25,7 +27,7 @@ # - arguments can be configured in a dict and standardized across tools -def exit_gracefully(*_args) -> None: # type: ignore +def exit_gracefully(*_args) -> None: log.warning("Aborted!") sys.exit(0) @@ -74,7 +76,7 @@ def cli( key_filepath: Path | None, verbose: bool, version: bool, -): +) -> None: """A primary set of options to configure how to interface the herd""" signal.signal(signal.SIGTERM, exit_gracefully) signal.signal(signal.SIGINT, exit_gracefully) @@ -102,7 +104,7 @@ def cli( @cli.command(short_help="Power off shepherd nodes") @click.option("--restart", "-r", is_flag=True, help="Reboot") @click.pass_context -def poweroff(ctx: click.Context, restart: bool): +def poweroff(ctx: click.Context, restart: bool) -> None: with ctx.obj["herd"] as herd: exit_code = herd.poweroff(restart=restart) sys.exit(exit_code) @@ -112,7 +114,7 @@ def poweroff(ctx: click.Context, restart: bool): @click.pass_context @click.argument("command", type=click.STRING) @click.option("--sudo", "-s", is_flag=True, help="Run command with sudo") -def shell_cmd(ctx: click.Context, command: str, sudo: bool): +def shell_cmd(ctx: click.Context, command: str, sudo: bool) -> None: with ctx.obj["herd"] as herd: replies = herd.run_cmd(sudo=sudo, cmd=command) herd.print_output(replies, verbose=True) @@ -152,7 +154,7 @@ def inventorize(ctx: click.Context, output_path: Path) -> None: ) @click.option("--attach", "-a", is_flag=True, help="Wait and receive output") @click.pass_context -def run(ctx: click.Context, config: Path, attach: bool): +def run(ctx: click.Context, config: Path, attach: bool) -> None: with ctx.obj["herd"] as herd: exit_code = herd.run_task(config, attach=attach) sys.exit(exit_code) @@ -197,8 +199,8 @@ def run(ctx: click.Context, config: Path, attach: bool): def harvest( ctx: click.Context, no_start: bool, - **kwargs, -): + **kwargs: Unpack[TypedDict], +) -> None: with ctx.obj["herd"] as herd: for path in ["output_path"]: file_path = Path(kwargs[path]) @@ -301,8 +303,8 @@ def harvest( def emulate( ctx: click.Context, no_start: bool, - **kwargs, -): + **kwargs: Unpack[TypedDict], +) -> None: with ctx.obj["herd"] as herd: for path in ["input_path", "output_path"]: file_path = Path(kwargs[path]) @@ -409,7 +411,7 @@ def distribute( filename: Path, remote_path: Path, force_overwrite: bool, -): +) -> None: with ctx.obj["herd"] as herd: herd.put_file(filename, remote_path, force_overwrite=force_overwrite) @@ -474,7 +476,13 @@ def retrieve( if herd.await_stop(timeout=30): raise Exception("shepherd still active after timeout") - failed = herd.get_file(filename, outdir, timestamp=timestamp, separate=separate, delete_src=delete) + failed = herd.get_file( + filename, + outdir, + timestamp=timestamp, + separate=separate, + delete_src=delete, + ) sys.exit(failed) @@ -531,7 +539,7 @@ def retrieve( help="dry-run the programmer - no data gets written", ) @click.pass_context -def program(ctx: click.Context, **kwargs): +def program(ctx: click.Context, **kwargs: Unpack[TypedDict]) -> None: tmp_file = "/tmp/target_image.hex" # noqa: S108 cfg_path = Path("/etc/shepherd/config_for_herd.yaml") diff --git a/software/shepherd-herd/tests/conftest.py b/software/shepherd-herd/tests/conftest.py index 03529a3e..82216784 100644 --- a/software/shepherd-herd/tests/conftest.py +++ b/software/shepherd-herd/tests/conftest.py @@ -83,7 +83,7 @@ def local_herd(tmp_path: Path) -> Path: @pytest.fixture -def stopped_herd(cli_runner: CliRunner): +def stopped_herd(cli_runner: CliRunner) -> None: cli_runner.invoke(cli, ["-v", "stop"]) wait_for_end(cli_runner) # make sure kernel module is active diff --git a/software/shepherd-herd/tests/test_cli_emu.py b/software/shepherd-herd/tests/test_cli_emu.py index 9cf93293..94d29ed6 100644 --- a/software/shepherd-herd/tests/test_cli_emu.py +++ b/software/shepherd-herd/tests/test_cli_emu.py @@ -1,4 +1,5 @@ import time +from pathlib import Path import pytest from click.testing import CliRunner @@ -9,7 +10,7 @@ @pytest.mark.timeout(60) -def test_emu_prepare(cli_runner: CliRunner, stopped_herd, tmp_path) -> None: +def test_emu_prepare(cli_runner: CliRunner, stopped_herd: None, tmp_path: Path) -> None: # distribute file and emulate from it in following tests test_file = generate_h5_file(tmp_path, "pytest_src.h5") res = cli_runner.invoke( @@ -26,7 +27,7 @@ def test_emu_prepare(cli_runner: CliRunner, stopped_herd, tmp_path) -> None: @pytest.mark.timeout(150) -def test_emu_example(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_example(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -44,7 +45,7 @@ def test_emu_example(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(60) -def test_emu_example_fail(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_example_fail(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -62,7 +63,7 @@ def test_emu_example_fail(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_emu_minimal(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_minimal(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -75,7 +76,7 @@ def test_emu_minimal(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_emu_all_args_long(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_all_args_long(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -104,7 +105,7 @@ def test_emu_all_args_long(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_emu_all_args_short(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_all_args_short(cli_runner: CliRunner, stopped_herd: None) -> None: # short arg or opposite bool val res = cli_runner.invoke( cli, @@ -134,7 +135,7 @@ def test_emu_all_args_short(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_emu_no_start(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_no_start(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -160,7 +161,7 @@ def test_emu_no_start(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(60) -def test_emu_force_stop(cli_runner: CliRunner, stopped_herd) -> None: +def test_emu_force_stop(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ diff --git a/software/shepherd-herd/tests/test_cli_hrv.py b/software/shepherd-herd/tests/test_cli_hrv.py index 059105a2..68e661a7 100644 --- a/software/shepherd-herd/tests/test_cli_hrv.py +++ b/software/shepherd-herd/tests/test_cli_hrv.py @@ -8,7 +8,7 @@ @pytest.mark.timeout(150) -def test_hrv_example(cli_runner: CliRunner, stopped_herd) -> None: +def test_hrv_example(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -27,7 +27,7 @@ def test_hrv_example(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(60) -def test_hrv_example_fail(cli_runner: CliRunner, stopped_herd) -> None: +def test_hrv_example_fail(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -46,7 +46,7 @@ def test_hrv_example_fail(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(60) -def test_hrv_minimal(cli_runner: CliRunner, stopped_herd) -> None: +def test_hrv_minimal(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, ["harvest"], @@ -63,7 +63,7 @@ def test_hrv_minimal(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_hrv_all_args_long(cli_runner: CliRunner, stopped_herd) -> None: +def test_hrv_all_args_long(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -84,7 +84,7 @@ def test_hrv_all_args_long(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_hrv_all_args_short(cli_runner: CliRunner, stopped_herd) -> None: +def test_hrv_all_args_short(cli_runner: CliRunner, stopped_herd: None) -> None: res = cli_runner.invoke( cli, [ @@ -105,7 +105,7 @@ def test_hrv_all_args_short(cli_runner: CliRunner, stopped_herd) -> None: @pytest.mark.timeout(150) -def test_hrv_no_start(cli_runner: CliRunner, stopped_herd) -> None: +def test_hrv_no_start(cli_runner: CliRunner, stopped_herd: None) -> None: # Note: short timeout is the catch res = cli_runner.invoke( cli, From d15640e2241f3226ed1052645f9237c75c960f8f Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 19:52:39 +0200 Subject: [PATCH 17/35] try to properly exit infinite loops when needed --- .../python-package/shepherd_sheep/__init__.py | 4 +- software/python-package/shepherd_sheep/cli.py | 8 ++- .../python-package/shepherd_sheep/eeprom.py | 2 +- .../shepherd_sheep/h5_writer.py | 2 +- .../python-package/shepherd_sheep/launcher.py | 49 ++++++++------- .../shepherd_sheep/monitor_kernel.py | 4 ++ .../shepherd_sheep/monitor_ptp.py | 4 ++ .../shepherd_sheep/monitor_sheep.py | 19 ++++-- .../shepherd_sheep/monitor_uart.py | 2 + .../shepherd_sheep/shared_memory.py | 36 +++++------ .../shepherd_sheep/shepherd_debug.py | 34 ++++++----- .../shepherd_sheep/shepherd_emulator.py | 23 +++++-- .../shepherd_sheep/shepherd_harvester.py | 14 +++-- .../shepherd_sheep/shepherd_io.py | 36 +++++------ .../shepherd_sheep/sysfs_interface.py | 61 ++++++++++--------- .../tests/test_cli_programmer.py | 16 +++-- .../python-package/tests/test_cli_run_task.py | 4 +- software/python-package/tests/test_eeprom.py | 3 +- .../python-package/tests/test_emulation.py | 4 +- .../tests/test_sysfs_interface.py | 10 +-- .../shepherd_cal/profile_calibration.py | 2 +- software/shepherd-herd/shepherd_herd/herd.py | 2 +- .../shepherd-herd/shepherd_herd/herd_cli.py | 2 +- 23 files changed, 198 insertions(+), 143 deletions(-) diff --git a/software/python-package/shepherd_sheep/__init__.py b/software/python-package/shepherd_sheep/__init__.py index 79cf8574..0a9a25af 100644 --- a/software/python-package/shepherd_sheep/__init__.py +++ b/software/python-package/shepherd_sheep/__init__.py @@ -38,7 +38,7 @@ from .shepherd_debug import ShepherdDebug from .shepherd_emulator import ShepherdEmulator from .shepherd_harvester import ShepherdHarvester -from .shepherd_io import ShepherdIOException +from .shepherd_io import ShepherdIOError from .sysfs_interface import check_sys_access from .sysfs_interface import flatten_list from .target_io import TargetIO @@ -58,7 +58,7 @@ "run_programmer", "run_firmware_mod", "run_task", - "ShepherdIOException", + "ShepherdIOError", "log", "flatten_list", ] diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index b4b62625..952be15e 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -13,7 +13,7 @@ import time from pathlib import Path from typing import TypedDict -from typing import Unpack +from typing_extensions import Unpack import click import gevent @@ -23,12 +23,12 @@ from shepherd_core.data_models.testbed import ProgrammerProtocol from shepherd_core.inventory import Inventory +from . import Launcher from . import __version__ from . import run_programmer from . import run_task from . import sysfs_interface from .eeprom import EEPROM -from .launcher import Launcher from .logger import log from .logger import set_verbosity from .shepherd_debug import ShepherdDebug @@ -234,7 +234,9 @@ def stop_server() -> None: gevent.signal_handler(signal.SIGTERM, stop_server) gevent.signal_handler(signal.SIGINT, stop_server) - shepherd_io.start() + success = shepherd_io.start() + if not success: + return log.info("Shepherd RPC Interface: Started") server.run() diff --git a/software/python-package/shepherd_sheep/eeprom.py b/software/python-package/shepherd_sheep/eeprom.py index 60aaa54d..bd8a8698 100644 --- a/software/python-package/shepherd_sheep/eeprom.py +++ b/software/python-package/shepherd_sheep/eeprom.py @@ -14,7 +14,7 @@ import struct from contextlib import suppress from types import TracebackType -from typing import Self +from typing_extensions import Self from shepherd_core.data_models.base.calibration import CalibrationCape from shepherd_core.data_models.base.calibration import CapeData diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index d40b6074..fdae428e 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -12,7 +12,7 @@ from types import TracebackType from typing import TYPE_CHECKING from typing import ClassVar -from typing import Self +from typing_extensions import Self if TYPE_CHECKING: import h5py diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index c59a81a6..9be2297c 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -14,6 +14,7 @@ from threading import Event from threading import Thread from types import TracebackType +from typing_extensions import Self from .logger import log @@ -58,7 +59,7 @@ def __init__( self.pin_ack_watchdog = pin_ack_watchdog self.service_name = service_name - def __enter__(self) -> None: + def __enter__(self) -> Self: self.gpio_led = GPIO(self.pin_led, "out") self.gpio_button = GPIO(self.pin_button, "in") self.gpio_ack_watchdog = GPIO(self.pin_ack_watchdog, "out") @@ -79,7 +80,6 @@ def __enter__(self) -> None: str(shepherd_object), ) log.debug("configured dbus for systemd") - return self def __exit__( @@ -99,27 +99,30 @@ def run(self) -> None: edge, shepherd service is either started or stopped. Double button press while idle causes system shutdown. """ - while True: - log.info("waiting for falling edge..") - self.gpio_led.write(True) - if not self.gpio_button.poll(): - # NOTE poll is suspected to exit after ~ 1-2 weeks running - # -> fills mmc with random measurement - # TODO observe behavior, hopefully this change fixes the bug - continue - self.gpio_led.write(False) - log.debug("edge detected") - if not self.get_state(): - time.sleep(0.25) - if self.gpio_button.poll(timeout=5): - log.debug("falling edge detected") - log.info("shutdown requested") - self.initiate_shutdown() - self.gpio_led.write(False) - time.sleep(3) + try: + while True: + log.info("waiting for falling edge..") + self.gpio_led.write(True) + if not self.gpio_button.poll(): + # NOTE poll is suspected to exit after ~ 1-2 weeks running + # -> fills mmc with random measurement + # TODO observe behavior, hopefully this change fixes the bug continue - self.set_service(not self.get_state()) - time.sleep(10) + self.gpio_led.write(False) + log.debug("edge detected") + if not self.get_state(): + time.sleep(0.25) + if self.gpio_button.poll(timeout=5): + log.debug("falling edge detected") + log.info("shutdown requested") + self.initiate_shutdown() + self.gpio_led.write(False) + time.sleep(3) + continue + self.set_service(not self.get_state()) + time.sleep(10) + except SystemExit: + return def get_state(self, timeout: float = 10) -> bool: """Queries systemd for state of shepherd service. @@ -153,7 +156,7 @@ def get_state(self, timeout: float = 10) -> bool: return False raise Exception(f"Unknown state { systemd_state }") - def set_service(self, requested_state: bool) -> bool: + def set_service(self, requested_state: bool) -> bool | None: """Changes state of shepherd service. Args: diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index 11c1003f..8bc5df97 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -78,6 +78,10 @@ def thread_fn(self) -> None: data_length += self.increment self.data["time"].resize((data_length,)) self.data["message"].resize((data_length,)) + except RuntimeError: + log.error("[%s] HDF5-File unavailable - will stop", type(self).__name__) + break + try: self.data["time"][self.position] = int(time.time() * 1e9) self.data["message"][self.position] = line self.position += 1 diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index 6ec5cb82..f5dc0edf 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -90,6 +90,10 @@ def thread_fn(self) -> None: data_length += self.increment self.data["time"].resize((data_length,)) self.data["values"].resize((data_length, 3)) + except RuntimeError: + log.error("[%s] HDF5-File unavailable - will stop", type(self).__name__) + break + try: self.data["time"][self.position] = int(time.time() * 1e9) self.data["values"][self.position, :] = values[0:3] self.position += 1 diff --git a/software/python-package/shepherd_sheep/monitor_sheep.py b/software/python-package/shepherd_sheep/monitor_sheep.py index 6eab029e..a30cbcd6 100644 --- a/software/python-package/shepherd_sheep/monitor_sheep.py +++ b/software/python-package/shepherd_sheep/monitor_sheep.py @@ -58,12 +58,19 @@ def thread_fn(self) -> None: while not self.event.is_set(): if self.queue.qsize() > 0: rec = self.queue.get() - data_length = self.data["time"].shape[0] - if self.position >= data_length: - data_length += self.increment - self.data["time"].resize((data_length,)) - self.data["message"].resize((data_length,)) - self.data["level"].resize((data_length,)) + try: + data_length = self.data["time"].shape[0] + if self.position >= data_length: + data_length += self.increment + self.data["time"].resize((data_length,)) + self.data["message"].resize((data_length,)) + self.data["level"].resize((data_length,)) + except RuntimeError: + log.error( + "[%s] HDF5-File unavailable - will stop", + type(self).__name__, + ) + break self.data["time"][self.position] = int(rec.created * 1e9) self.data["message"][self.position] = rec.message self.data["level"][self.position] = rec.levelno diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index 47a3aa7d..f04f1c3c 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -93,6 +93,8 @@ def thread_fn(self) -> None: self.data["message"][self.position] = output self.position += 1 self.event.wait(self.poll_intervall) # rate limiter + except RuntimeError: + log.error("[%s] HDF5-File unavailable - will stop", type(self).__name__) except ValueError as e: log.error( # noqa: G200 "[%s] PySerial ValueError '%s' - " diff --git a/software/python-package/shepherd_sheep/shared_memory.py b/software/python-package/shepherd_sheep/shared_memory.py index 1ffd4cd6..4d0098d4 100644 --- a/software/python-package/shepherd_sheep/shared_memory.py +++ b/software/python-package/shepherd_sheep/shared_memory.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from datetime import timedelta from types import TracebackType -from typing import Self +from typing_extensions import Self import numpy as np from shepherd_core.data_models import GpioTracing @@ -301,11 +301,10 @@ def read_buffer(self, index: int, verbose: bool = False) -> DataBuffer: if not (0 <= n_gpio_events <= commons.MAX_GPIO_EVT_PER_BUFFER): log.error( - "Size of gpio_events out of range with %d entries", + "Size of gpio_events out of range with %d entries (max=%d)", n_gpio_events, + commons.MAX_GPIO_EVT_PER_BUFFER, ) - # TODO: should be exception, also - # put into Writer.write_exception() with ShepherdIOException n_gpio_events = commons.MAX_GPIO_EVT_PER_BUFFER if self.ts_start_gp <= buffer_timestamp <= self.ts_stop_gp: @@ -335,22 +334,19 @@ def read_buffer(self, index: int, verbose: bool = False) -> DataBuffer: pru0_util_mean = round(100 * pru0_sum_ticks / n_samples / 2000, 1) if pru0_util_mean > pru0_util_max: pru0_util_mean = 0.1 - if verbose: - if (pru0_util_mean > 95) or (pru0_util_max > 100): - log.warning( - "Pru0 Loop-Util: mean = %d %%, max = %d %% " - "-> WARNING: broken real-time-condition", - pru0_util_mean, - pru0_util_max, - ) - # TODO: raise ShepherdIOException or add this info into output-file? - # WRONG PLACE HERE - else: - log.info( - "Pru0 Loop-Util: mean = %d %%, max = %d %%", - pru0_util_mean, - pru0_util_max, - ) + if (pru0_util_mean > 98) or (pru0_util_max > 100): + log.warning( + "Pru0 Loop-Util: mean = %d %%, max = %d %% " + "-> WARNING: probably broken real-time-condition", + pru0_util_mean, + pru0_util_max, + ) + elif verbose: + log.info( + "Pru0 Loop-Util: mean = %d %%, max = %d %%", + pru0_util_mean, + pru0_util_max, + ) return DataBuffer( voltage, diff --git a/software/python-package/shepherd_sheep/shepherd_debug.py b/software/python-package/shepherd_sheep/shepherd_debug.py index edee4886..e50436cd 100644 --- a/software/python-package/shepherd_sheep/shepherd_debug.py +++ b/software/python-package/shepherd_sheep/shepherd_debug.py @@ -1,7 +1,7 @@ import contextlib import time from typing import NoReturn -from typing import Self +from typing_extensions import Self import msgpack import msgpack_numpy @@ -21,7 +21,7 @@ from .eeprom import EEPROM from .logger import log from .shepherd_io import ShepherdIO -from .shepherd_io import ShepherdIOException +from .shepherd_io import ShepherdIOError from .target_io import TargetIO @@ -93,7 +93,7 @@ def adc_read(self, channel: str) -> int: msg_type, values = self._get_msg(30) if msg_type != commons.MSG_DBG_ADC: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_ADC) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -108,7 +108,7 @@ def gpi_read(self) -> int: super()._send_msg(commons.MSG_DBG_GPI, 0) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_GPI: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_GPI) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -144,7 +144,7 @@ def dbg_fn_test(self, factor: int, mode: int) -> int: super()._send_msg(commons.MSG_DBG_FN_TESTS, [factor, mode]) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_FN_TESTS: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_FN_TESTS) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -160,7 +160,8 @@ def vsource_init( ) -> None: super().send_calibration_settings(cal_emu) src_pru = ConverterPRUConfig.from_vsrc( - src_cfg, log_intermediate_node=log_intermediate + src_cfg, + log_intermediate_node=log_intermediate, ) super().send_virtual_converter_settings(src_pru) @@ -177,7 +178,7 @@ def vsource_init( super()._send_msg(commons.MSG_DBG_VSRC_INIT, 0) msg_type, values = super()._get_msg() # no data, just a confirmation if msg_type != commons.MSG_DBG_VSRC_INIT: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_INIT) }, " f"but got type={ hex(msg_type) } val={ values }, " " is ENABLE_DBG_VSOURCE defined in pru0/main.c??", @@ -201,7 +202,7 @@ def cnv_calc_inp_power( ) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_P_INP: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_P_INP) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -218,7 +219,7 @@ def cnv_charge( ) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_CHARGE: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_CHARGE) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -228,7 +229,7 @@ def cnv_calc_out_power(self, current_adc_raw: int) -> int: self._send_msg(commons.MSG_DBG_VSRC_P_OUT, int(current_adc_raw)) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_P_OUT: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_P_OUT) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -238,7 +239,7 @@ def cnv_drain(self, current_adc_raw: int) -> tuple[int, int]: self._send_msg(commons.MSG_DBG_VSRC_DRAIN, int(current_adc_raw)) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_DRAIN: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_DRAIN) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -248,7 +249,7 @@ def cnv_update_cap_storage(self) -> int: self._send_msg(commons.MSG_DBG_VSRC_V_CAP, 0) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_V_CAP: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_V_CAP) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -258,7 +259,7 @@ def cnv_update_states_and_output(self) -> int: self._send_msg(commons.MSG_DBG_VSRC_V_OUT, 0) msg_type, values = self._get_msg() if msg_type != commons.MSG_DBG_VSRC_V_OUT: - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { hex(commons.MSG_DBG_VSRC_V_OUT) }, " f"but got type={ hex(msg_type) } val={ values }", ) @@ -266,7 +267,10 @@ def cnv_update_states_and_output(self) -> int: # TEST-SIMPLIFICATION - code below is also part py-vsource with same interface def iterate_sampling( - self, V_inp_uV: int = 0, A_inp_nA: int = 0, A_out_nA: int = 0 + self, + V_inp_uV: int = 0, + A_inp_nA: int = 0, + A_out_nA: int = 0, ) -> int: # NOTE: this includes the harvester P_inp_fW = self.cnv_calc_inp_power(V_inp_uV, A_inp_nA, include_hrv=True) @@ -421,7 +425,7 @@ def sample_from_pru(self, length_n_buffers: int = 10) -> bytes | None: def process_programming_messages(self) -> None: """Prints messages to console until timeout occurs""" - with contextlib.suppress(ShepherdIOException): + with contextlib.suppress(ShepherdIOError): while True: msg_type, values = self._get_msg(5) if msg_type != commons.MSG_PGM_ERROR_WRITE: diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index d307f4b6..6ab8256c 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -4,7 +4,7 @@ from contextlib import ExitStack from datetime import datetime from types import TracebackType -from typing import Self +from typing_extensions import Self from shepherd_core import CalibrationPair from shepherd_core import CalibrationSeries @@ -23,7 +23,7 @@ from .logger import log from .shared_memory import DataBuffer from .shepherd_io import ShepherdIO -from .shepherd_io import ShepherdIOException +from .shepherd_io import ShepherdIOError from .target_io import TargetIO from .target_io import target_pins @@ -189,7 +189,10 @@ def __exit__( super().__exit__() def return_buffer( - self, index: int, buffer: DataBuffer, verbose: bool = False + self, + index: int, + buffer: DataBuffer, + verbose: bool = False, ) -> None: ts_start = time.time() if verbose else 0 @@ -207,7 +210,9 @@ def return_buffer( ) def run(self) -> None: - self.start(self.start_time, wait_blocking=False) + success = self.start(self.start_time, wait_blocking=False) + if not success: + return log.info("waiting %.2f s until start", self.start_time - time.time()) self.wait_for_start(self.start_time - time.time() + 15) log.info("shepherd started!") @@ -226,7 +231,7 @@ def run(self) -> None: ): try: idx, emu_buf = self.get_buffer(verbose=self.verbose_extra) - except ShepherdIOException as e: + except ShepherdIOError as e: if self.cfg.abort_on_error: raise RuntimeError( "Caught unforgivable ShepherdIO-Exception", @@ -251,11 +256,17 @@ def run(self) -> None: break if self.writer is not None: self.writer.write_buffer(emu_buf) - except ShepherdIOException as e: + except ShepherdIOError as e: # We're done when the PRU has processed all emulation data buffers if e.id_num == commons.MSG_DEP_ERR_NOFREEBUF: + log.debug("Collected all Buffers from PRU -> begin to exit now") + break + if e.id_num == ShepherdIOError.ID_TIMEOUT: + log.error("Reception from PRU had a timeout -> begin to exit now") break + if self.cfg.abort_on_error: raise RuntimeError( "Caught unforgivable ShepherdIO-Exception", ) from e + log.warning("Caught an Exception", exc_info=e) diff --git a/software/python-package/shepherd_sheep/shepherd_harvester.py b/software/python-package/shepherd_sheep/shepherd_harvester.py index 7de11bb9..eb5a37ec 100644 --- a/software/python-package/shepherd_sheep/shepherd_harvester.py +++ b/software/python-package/shepherd_sheep/shepherd_harvester.py @@ -4,7 +4,7 @@ import time from contextlib import ExitStack from types import TracebackType -from typing import Self +from typing_extensions import Self from shepherd_core import local_tz from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig @@ -16,7 +16,7 @@ from .logger import get_verbosity from .logger import log from .shepherd_io import ShepherdIO -from .shepherd_io import ShepherdIOException +from .shepherd_io import ShepherdIOError class ShepherdHarvester(ShepherdIO): @@ -135,8 +135,9 @@ def return_buffer(self, index: int, verbose: bool = False) -> None: log.debug("Sent empty buffer #%s to PRU", index) def run(self) -> None: - self.start(self.start_time, wait_blocking=False) - + success = self.start(self.start_time, wait_blocking=False) + if not success: + return log.info("waiting %.2f s until start", self.start_time - time.time()) self.wait_for_start(self.start_time - time.time() + 15) log.info("shepherd started!") @@ -151,7 +152,10 @@ def run(self) -> None: while True: try: idx, hrv_buf = self.get_buffer(verbose=self.verbose_extra) - except ShepherdIOException as e: + except ShepherdIOError as e: + if e.id_num == ShepherdIOError.ID_TIMEOUT: + log.error("Reception from PRU had a timeout -> begin to exit now") + break if self.cfg.abort_on_error: raise RuntimeError( "Caught unforgivable ShepherdIO-Exception", diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 5668971c..32b9dd22 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -10,7 +10,7 @@ import time from contextlib import suppress from types import TracebackType -from typing import Self +from typing_extensions import Self from pydantic import validate_call from shepherd_core import CalibrationEmulator @@ -32,9 +32,6 @@ with suppress(ModuleNotFoundError): from periphery import GPIO - -ID_ERR_TIMEOUT = 100 - gpio_pin_nums = { "target_pwr_sel": 31, "target_io_en": 60, @@ -45,7 +42,9 @@ } -class ShepherdIOException(Exception): +class ShepherdIOError(Exception): + ID_TIMEOUT = 100 + def __init__(self, message: str, id_num: int = 0, value: int = 0) -> None: super().__init__(message + f" [id=0x{id_num:x}, val=0x{value:x}]") self.id_num = id_num @@ -178,10 +177,10 @@ def _get_msg(self, timeout_n: int = 5) -> tuple[int, list[int]]: for _ in range(timeout_n): try: return sfs.read_pru_msg() - except sfs.SysfsInterfaceException: # noqa: PERF203 + except sfs.SysfsInterfaceError: # noqa: PERF203 time.sleep(self._buffer_period) continue - raise ShepherdIOException("Timeout waiting for message", ID_ERR_TIMEOUT) + raise ShepherdIOError("Timeout waiting for message", ShepherdIOError.ID_TIMEOUT) @staticmethod def _flush_msgs() -> None: @@ -189,14 +188,14 @@ def _flush_msgs() -> None: while True: try: sfs.read_pru_msg() - except sfs.SysfsInterfaceException: # noqa: PERF203 + except sfs.SysfsInterfaceError: # noqa: PERF203 break def start( self, start_time: float | None = None, wait_blocking: bool = True, - ) -> None: + ) -> bool: """Starts sampling either now or at later point in time. Args: @@ -205,9 +204,10 @@ def start( """ if isinstance(start_time, float | int): log.debug("asking kernel module for start at %.2f", start_time) - sfs.set_start(start_time) + success = sfs.set_start(start_time) if wait_blocking: self.wait_for_start(3_000_000) + return success @staticmethod def wait_for_start(timeout: float) -> None: @@ -266,14 +266,14 @@ def _cleanup(self) -> None: while count < 6 and sfs.get_state() != "idle": try: sfs.set_stop(force=True) - except sfs.SysfsInterfaceException: + except sfs.SysfsInterfaceError: log.exception( "CleanupRoutine caused an exception while trying to stop PRU (n=%d)", count, ) try: sfs.wait_for_state("idle", 3.0) - except sfs.SysfsInterfaceException: + except sfs.SysfsInterfaceError: log.warning( "CleanupRoutine caused an exception while waiting for PRU to go to idle (n=%d)", count, @@ -497,7 +497,9 @@ def _return_buffer(self, index: int) -> None: self._send_msg(commons.MSG_BUF_FROM_HOST, index) def get_buffer( - self, timeout_n: int = 10, verbose: bool = False + self, + timeout_n: int = 10, + verbose: bool = False, ) -> tuple[int, DataBuffer]: """Reads a data buffer from shared memory. @@ -536,26 +538,26 @@ def get_buffer( continue if msg_type == commons.MSG_DEP_ERR_INCMPLT: - raise ShepherdIOException( + raise ShepherdIOError( "Got incomplete buffer", commons.MSG_DEP_ERR_INCMPLT, value, ) if msg_type == commons.MSG_DEP_ERR_INVLDCMD: - raise ShepherdIOException( + raise ShepherdIOError( "PRU received invalid command", commons.MSG_DEP_ERR_INVLDCMD, value, ) if msg_type == commons.MSG_DEP_ERR_NOFREEBUF: - raise ShepherdIOException( + raise ShepherdIOError( "PRU ran out of buffers", commons.MSG_DEP_ERR_NOFREEBUF, value, ) - raise ShepherdIOException( + raise ShepherdIOError( f"Expected msg type { commons.MSG_BUF_FROM_PRU } " f"got { msg_type }[{ value }]", ) diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index 65682f4a..b6270b0d 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -23,7 +23,7 @@ sysfs_path = Path("/sys/shepherd") -class SysfsInterfaceException(Exception): +class SysfsInterfaceError(Exception): pass @@ -147,7 +147,7 @@ def wait_for_state(wanted_state: str, timeout: float) -> float: return time.time() - ts_start if time.time() - ts_start > timeout: - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"timed out waiting for state { wanted_state } - " f"state is { current_state }", ) @@ -155,7 +155,7 @@ def wait_for_state(wanted_state: str, timeout: float) -> float: time.sleep(0.1) -def set_start(start_time: float | int | None = None) -> None: # noqa: PYI041 +def set_start(start_time: float | int | None = None) -> True: # noqa: PYI041 """Starts shepherd. Writes 'start' to the 'state' sysfs attribute in order to transition from @@ -168,17 +168,22 @@ def set_start(start_time: float | int | None = None) -> None: # noqa: PYI041 current_state = get_state() log.debug("current state of shepherd kernel module: %s", current_state) if current_state != "idle": - raise SysfsInterfaceException(f"Cannot start from state { current_state }") - - with (sysfs_path / "state").open("w", encoding="utf-8") as f: - if isinstance(start_time, float): - start_time = int(start_time) - if isinstance(start_time, int): - log.debug("writing start-time = %d to sysfs", start_time) - f.write(f"{start_time}") - else: # unknown type - log.debug("writing 'start' to sysfs") - f.write("start") + raise SysfsInterfaceError(f"Cannot start from state { current_state }") + + try: + with (sysfs_path / "state").open("w", encoding="utf-8") as f: + if isinstance(start_time, float): + start_time = int(start_time) + if isinstance(start_time, int): + log.debug("writing start-time = %d to sysfs", start_time) + f.write(f"{start_time}") + else: # unknown type + log.debug("writing 'start' to sysfs") + f.write("start") + except OSError: + log.error("Failed to write 'Start' to sysfs") + return False + return True def set_stop(force: bool = False) -> None: @@ -190,7 +195,7 @@ def set_stop(force: bool = False) -> None: if not force: current_state = get_state() if current_state != "running": - raise SysfsInterfaceException(f"Cannot stop from state { current_state }") + raise SysfsInterfaceError(f"Cannot stop from state { current_state }") with (sysfs_path / "state").open("w", encoding="utf-8") as f: f.write("stop") @@ -206,12 +211,12 @@ def write_mode(mode: str, force: bool = False) -> None: :param force: """ if mode not in shepherd_modes: - raise SysfsInterfaceException("invalid value for mode") + raise SysfsInterfaceError("invalid value for mode") if force: set_stop(force=True) wait_for_state("idle", 5) elif get_state() != "idle": - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"Cannot set mode when shepherd is { get_state() }", ) @@ -248,9 +253,9 @@ def write_dac_aux_voltage( return if voltage < 0.0: - raise SysfsInterfaceException(f"sending voltage with negative value: {voltage}") + raise SysfsInterfaceError(f"sending voltage with negative value: {voltage}") if voltage > 5.0: - raise SysfsInterfaceException(f"sending voltage above limit of 5V: {voltage}") + raise SysfsInterfaceError(f"sending voltage above limit of 5V: {voltage}") if not cal_emu: cal_emu = CalibrationEmulator() output = int(cal_emu.dac_V_A.si_to_raw(voltage)) @@ -326,15 +331,15 @@ def write_calibration_settings( """ if cal_pru["adc_current_gain"] < 0: - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"sending calibration with negative ADC-C-gain: {cal_pru['adc_current_gain']}", ) if cal_pru["adc_voltage_gain"] < 0: - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"sending calibration with negative ADC-V-gain: {cal_pru['adc_voltage_gain']}", ) if cal_pru["dac_voltage_gain"] < 0: - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"sending calibration with negative DAC-gain: {cal_pru['dac_voltage_gain']}", ) wait_for_state("idle", 3.0) @@ -393,7 +398,7 @@ def write_virtual_converter_settings(settings: ConverterPRUConfig) -> None: _set = [str(i) for i in _set] output += " ".join(_set) + " \n" else: - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"virtual-converter value {setting} has wrong type ({type(setting)})", ) @@ -433,7 +438,7 @@ def write_virtual_harvester_settings(settings: HarvesterPRUConfig) -> None: if isinstance(setting, int): output += f"{setting} \n" else: - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"virtual harvester value {setting} has wrong type ({type(setting)})", ) @@ -462,7 +467,7 @@ def write_pru_msg(msg_type: int, values: list | float | int) -> None: # noqa: P :param values: """ if (not isinstance(msg_type, int)) or (msg_type < 0) or (msg_type > 255): - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"pru_msg-type has invalid type, " f"expected u8 for type (={type(msg_type)}) " f"and content (={msg_type})", @@ -476,7 +481,7 @@ def write_pru_msg(msg_type: int, values: list | float | int) -> None: # noqa: P for value in values: if (not isinstance(value, int)) or (value < 0) or (value >= 2**32): - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"pru_msg-value has invalid type, " f"expected u32 for type (={type(value)}) and content (={value})", ) @@ -493,7 +498,7 @@ def read_pru_msg() -> tuple[int, list[int]]: message = f.read().rstrip() msg_parts = [int(x) for x in message.split()] if len(msg_parts) < 2: - raise SysfsInterfaceException("pru_msg was too short") + raise SysfsInterfaceError("pru_msg was too short") return msg_parts[0], msg_parts[1:] @@ -540,7 +545,7 @@ def write_programmer_ctrl( if value is None: continue if num > 0 and ((value < 0) or (value >= 2**32)): - raise SysfsInterfaceException( + raise SysfsInterfaceError( f"at least one parameter out of u32-bounds, value={value}", ) with (sysfs_path / "programmer" / attribute).open( diff --git a/software/python-package/tests/test_cli_programmer.py b/software/python-package/tests/test_cli_programmer.py index 963a25f0..f680f1dc 100644 --- a/software/python-package/tests/test_cli_programmer.py +++ b/software/python-package/tests/test_cli_programmer.py @@ -139,7 +139,9 @@ def test_cli_program_sbw_explicit( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_file_defective_a( - shepherd_up: None, cli_runner: CliRunner, fw_empty: Path + shepherd_up: None, + cli_runner: CliRunner, + fw_empty: Path, ) -> None: res = cli_runner.invoke( cli, @@ -156,7 +158,9 @@ def test_cli_program_file_defective_a( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_file_defective_b( - shepherd_up: None, cli_runner: CliRunner, tmp_path: Path + shepherd_up: None, + cli_runner: CliRunner, + tmp_path: Path, ) -> None: res = cli_runner.invoke( cli, @@ -173,7 +177,9 @@ def test_cli_program_file_defective_b( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_file_defective_c( - shepherd_up: None, cli_runner: CliRunner, tmp_path: Path + shepherd_up: None, + cli_runner: CliRunner, + tmp_path: Path, ) -> None: res = cli_runner.invoke( cli, @@ -232,7 +238,9 @@ def test_cli_program_datarate_invalid_b( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_program_target_invalid( - shepherd_up: None, cli_runner: CliRunner, fw_nrf: Path + shepherd_up: None, + cli_runner: CliRunner, + fw_nrf: Path, ) -> None: res = cli_runner.invoke( cli, diff --git a/software/python-package/tests/test_cli_run_task.py b/software/python-package/tests/test_cli_run_task.py index 40dda59b..3397dccf 100644 --- a/software/python-package/tests/test_cli_run_task.py +++ b/software/python-package/tests/test_cli_run_task.py @@ -338,7 +338,9 @@ def test_cli_emulate_parameters_minimal( @pytest.mark.hardware @pytest.mark.timeout(60) def test_cli_emulate_preconfigured( - shepherd_up: None, cli_runner: CliRunner, path_here: Path + shepherd_up: None, + cli_runner: CliRunner, + path_here: Path, ) -> None: file_path = path_here / "_test_config_emulation.yaml" res = cli_runner.invoke(cli, ["run", file_path.as_posix()]) diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index 5d0cbcf3..203d56a6 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -46,7 +46,8 @@ def eeprom_retained(eeprom_open: EEPROM) -> Generator[EEPROM, None, None]: @pytest.fixture() def eeprom_with_data( - eeprom_retained: EEPROM, cape_data: CapeData + eeprom_retained: EEPROM, + cape_data: CapeData, ) -> Generator[EEPROM, None, None]: eeprom_retained._write_cape_data(cape_data) yield eeprom_retained diff --git a/software/python-package/tests/test_emulation.py b/software/python-package/tests/test_emulation.py index ef5f6d25..0d9f137e 100644 --- a/software/python-package/tests/test_emulation.py +++ b/software/python-package/tests/test_emulation.py @@ -13,7 +13,7 @@ from shepherd_core.data_models.testbed import TargetPort from shepherd_sheep import ShepherdDebug from shepherd_sheep import ShepherdEmulator -from shepherd_sheep import ShepherdIOException +from shepherd_sheep import ShepherdIOError from shepherd_sheep import Writer from shepherd_sheep import run_emulator from shepherd_sheep import sysfs_interface @@ -101,7 +101,7 @@ def test_emulation( idx, emu_buf = emulator.get_buffer() writer.write_buffer(emu_buf) - with pytest.raises(ShepherdIOException): + with pytest.raises(ShepherdIOError): _, _ = emulator.get_buffer() diff --git a/software/python-package/tests/test_sysfs_interface.py b/software/python-package/tests/test_sysfs_interface.py index 3e1c9b5f..2aa6f836 100644 --- a/software/python-package/tests/test_sysfs_interface.py +++ b/software/python-package/tests/test_sysfs_interface.py @@ -58,7 +58,7 @@ def test_start(shepherd_up: None) -> None: sysfs_interface.set_start() time.sleep(5) assert sysfs_interface.get_state() == "running" - with pytest.raises(sysfs_interface.SysfsInterfaceException): + with pytest.raises(sysfs_interface.SysfsInterfaceError): sysfs_interface.set_start() @@ -76,12 +76,12 @@ def test_start_delayed(shepherd_up: None) -> None: sysfs_interface.set_start(start_time) sysfs_interface.wait_for_state("armed", 1) - with pytest.raises(sysfs_interface.SysfsInterfaceException): + with pytest.raises(sysfs_interface.SysfsInterfaceError): sysfs_interface.wait_for_state("running", 3) sysfs_interface.wait_for_state("running", 3) - with pytest.raises(sysfs_interface.SysfsInterfaceException): + with pytest.raises(sysfs_interface.SysfsInterfaceError): sysfs_interface.set_start() @@ -98,12 +98,12 @@ def test_initial_mode(shepherd_up: None) -> None: @pytest.mark.hardware def test_set_mode_fail_offline(shepherd_running: None) -> None: - with pytest.raises(sysfs_interface.SysfsInterfaceException): + with pytest.raises(sysfs_interface.SysfsInterfaceError): sysfs_interface.write_mode("harvester") def test_set_mode_fail_invalid(shepherd_up: None) -> None: - with pytest.raises(sysfs_interface.SysfsInterfaceException): + with pytest.raises(sysfs_interface.SysfsInterfaceError): sysfs_interface.write_mode("invalidmode") diff --git a/software/shepherd-calibration/shepherd_cal/profile_calibration.py b/software/shepherd-calibration/shepherd_cal/profile_calibration.py index 1ad607e7..8c53e2b4 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_calibration.py +++ b/software/shepherd-calibration/shepherd_cal/profile_calibration.py @@ -1,4 +1,3 @@ -from typing import Self from typing import TypeVar import numpy as np @@ -7,6 +6,7 @@ from scipy import stats from shepherd_core import CalibrationPair from shepherd_core import CalibrationSeries +from typing_extensions import Self from .logger import logger diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 25fcd2a1..cc947baa 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -11,7 +11,7 @@ from types import TracebackType from typing import Any from typing import ClassVar -from typing import Self +from typing_extensions import Self import yaml from fabric import Connection diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index 327143a8..262e47ff 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -3,7 +3,7 @@ from datetime import datetime from pathlib import Path from typing import TypedDict -from typing import Unpack +from typing_extensions import Unpack import click import shepherd_core From 32bba75b5c270537f48fa10b44360e6b1b4bbd16 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Fri, 20 Oct 2023 20:45:42 +0200 Subject: [PATCH 18/35] fix unittests --- pyproject.toml | 6 +- .../shepherd_sheep/h5_writer.py | 6 +- .../python-package/shepherd_sheep/launcher.py | 4 +- .../shepherd_sheep/sysfs_interface.py | 64 +++++++++---------- software/python-package/tests/conftest.py | 19 +++--- software/python-package/tests/test_eeprom.py | 2 +- .../shepherd_cal/calibration_plot.py | 2 +- 7 files changed, 54 insertions(+), 49 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 55fb7c8c..2d1d421c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,7 @@ statistics = true line-length = 100 select = [ "A", # flake8-builtins -# "ANN", # flake8-annotations, TODO: turn on + "ANN", # flake8-annotations "ARG", # flake8-unused-arguments "B", # Bugbear "C", @@ -96,7 +96,7 @@ select = [ "ERA", # eradicate commented out code "F", # pyflakes # "FA", # flake8-future-annotations - "FBT", # boolean traps +# "FBT", # boolean traps "FLY", # flynt "FURB", # refurb "G", # flake8-logging-format @@ -149,7 +149,7 @@ exclude= [ # external projects "*/tests/**" = ["ARG", "S", "D", "SLF001"] "*/examples/**" = ["INP001"] # no namespace "software/shepherd-calibration/**" = ["ERA001"] # comments -"software/shepherd-devicetest/**" = ["ERA001", "ARG001", "PLW0603", "F405", "F403"] +"software/shepherd-devicetest/**" = ["ERA001", "ARG001", "PLW0603", "F405", "F403", "ANN001"] "docs/**" = ["INP001"] [tool.ruff.mccabe] diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index fdae428e..5698a6da 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -88,9 +88,9 @@ def __init__( window_samples, cal_data, compression, - modify_existing, - force_overwrite, - verbose, + modify_existing=modify_existing, + force_overwrite=force_overwrite, + verbose=verbose, ) self.buffer_timeseries = self.sample_interval_ns * np.arange( diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index 9be2297c..45493a05 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -14,6 +14,8 @@ from threading import Event from threading import Thread from types import TracebackType +from typing import Callable + from typing_extensions import Self from .logger import log @@ -24,7 +26,7 @@ from periphery import GPIO -def call_repeatedly(interval: float, func, *args): +def call_repeatedly(interval: float, func: Callable, *args) -> Callable: stopped = Event() def loop() -> None: diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index b6270b0d..cb075de2 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -20,8 +20,6 @@ from .logger import log -sysfs_path = Path("/sys/shepherd") - class SysfsInterfaceError(Exception): pass @@ -30,13 +28,13 @@ class SysfsInterfaceError(Exception): # dedicated sampling modes # - _adc_read - modes are used per rpc (currently to calibrate the hardware) # TODO: what is with "None"? -shepherd_modes = [ +shepherd_modes = { "harvester", "hrv_adc_read", "emulator", "emu_adc_read", "debug", -] +} def flatten_list(dl: list) -> list: @@ -171,7 +169,7 @@ def set_start(start_time: float | int | None = None) -> True: # noqa: PYI041 raise SysfsInterfaceError(f"Cannot start from state { current_state }") try: - with (sysfs_path / "state").open("w", encoding="utf-8") as f: + with Path("/sys/shepherd/state").open("w", encoding="utf-8") as f: if isinstance(start_time, float): start_time = int(start_time) if isinstance(start_time, int): @@ -197,7 +195,7 @@ def set_stop(force: bool = False) -> None: if current_state != "running": raise SysfsInterfaceError(f"Cannot stop from state { current_state }") - with (sysfs_path / "state").open("w", encoding="utf-8") as f: + with Path("/sys/shepherd/state").open("w", encoding="utf-8") as f: f.write("stop") @@ -221,7 +219,7 @@ def write_mode(mode: str, force: bool = False) -> None: ) log.debug("sysfs/mode: '%s'", mode) - with (sysfs_path / "mode").open("w", encoding="utf-8") as f: + with Path("/sys/shepherd/mode").open("w", encoding="utf-8") as f: f.write(mode) @@ -288,7 +286,7 @@ def write_dac_aux_voltage_raw( voltage_raw = min(voltage_raw, 2**16 - 1) voltage_raw |= int(ch_link) << 20 voltage_raw |= int(cap_out) << 21 - with (sysfs_path / "dac_auxiliary_voltage_raw").open("w", encoding="utf-8") as f: + with Path("/sys/shepherd/dac_auxiliary_voltage_raw").open("w", encoding="utf-8") as f: log.debug("Sending raw auxiliary voltage (dac channel B): %d", voltage_raw) f.write(str(voltage_raw)) @@ -315,7 +313,7 @@ def read_dac_aux_voltage_raw() -> int: Returns: voltage as dac_raw """ - with (sysfs_path / "dac_auxiliary_voltage_raw").open(encoding="utf-8") as f: + with Path("/sys/shepherd/dac_auxiliary_voltage_raw").open(encoding="utf-8") as f: settings = f.read().rstrip() int_settings = [int(x) for x in settings.split()] @@ -344,7 +342,7 @@ def write_calibration_settings( ) wait_for_state("idle", 3.0) - with (sysfs_path / "calibration_settings").open("w", encoding="utf-8") as f: + with Path("/sys/shepherd/calibration_settings").open("w", encoding="utf-8") as f: output = ( f"{int(cal_pru['adc_current_gain'])} {int(cal_pru['adc_current_offset'])} \n" f"{int(cal_pru['adc_voltage_gain'])} {int(cal_pru['adc_voltage_offset'])} \n" @@ -362,7 +360,7 @@ def read_calibration_settings() -> ( The virtual-source algorithms use adc measurements and dac-output """ - with (sysfs_path / "calibration_settings").open(encoding="utf-8") as f: + with Path("/sys/shepherd/calibration_settings").open(encoding="utf-8") as f: settings = f.read().rstrip() int_settings = [int(x) for x in settings.split()] @@ -403,7 +401,7 @@ def write_virtual_converter_settings(settings: ConverterPRUConfig) -> None: ) wait_for_state("idle", 3.0) - with (sysfs_path / "virtual_converter_settings").open( + with Path("/sys/shepherd/virtual_converter_settings").open( "w", encoding="utf-8", ) as file: @@ -416,7 +414,7 @@ def read_virtual_converter_settings() -> list: The pru-algorithm uses these settings to configure emulator. """ - with (sysfs_path / "virtual_converter_settings").open(encoding="utf-8") as f: + with Path("/sys/shepherd/virtual_converter_settings").open(encoding="utf-8") as f: settings = f.read().rstrip() return [int(x) for x in settings.split()] @@ -443,7 +441,7 @@ def write_virtual_harvester_settings(settings: HarvesterPRUConfig) -> None: ) wait_for_state("idle", 3.0) - with (sysfs_path / "virtual_harvester_settings").open( + with Path("/sys/shepherd/virtual_harvester_settings").open( "w", encoding="utf-8", ) as file: @@ -456,7 +454,7 @@ def read_virtual_harvester_settings() -> list: The pru-algorithm uses these settings to configure emulator. """ - with (sysfs_path / "virtual_harvester_settings").open(encoding="utf-8") as f: + with Path("/sys/shepherd/virtual_harvester_settings").open(encoding="utf-8") as f: settings = f.read().rstrip() return [int(x) for x in settings.split()] @@ -486,7 +484,7 @@ def write_pru_msg(msg_type: int, values: list | float | int) -> None: # noqa: P f"expected u32 for type (={type(value)}) and content (={value})", ) - with (sysfs_path / "pru_msg_box").open("w", encoding="utf-8") as file: + with Path("/sys/shepherd/pru_msg_box").open("w", encoding="utf-8") as file: file.write(f"{msg_type} {values[0]} {values[1]}") @@ -494,7 +492,7 @@ def read_pru_msg() -> tuple[int, list[int]]: """ Returns: """ - with (sysfs_path / "pru_msg_box").open(encoding="utf-8") as f: + with Path("/sys/shepherd/pru_msg_box").open(encoding="utf-8") as f: message = f.read().rstrip() msg_parts = [int(x) for x in message.split()] if len(msg_parts) < 2: @@ -540,6 +538,7 @@ def write_programmer_ctrl( # processing args = locals() log.debug("set programmerCTRL") + prog_path = Path("/sys/shepherd/programmer") for num, attribute in enumerate(prog_attribs): value = args[attribute] if value is None: @@ -548,7 +547,7 @@ def write_programmer_ctrl( raise SysfsInterfaceError( f"at least one parameter out of u32-bounds, value={value}", ) - with (sysfs_path / "programmer" / attribute).open( + with (prog_path / attribute).open( "w", encoding="utf-8", ) as file: @@ -558,26 +557,27 @@ def write_programmer_ctrl( def read_programmer_ctrl() -> list: parameters = [] + prog_path = Path("/sys/shepherd/programmer") for attribute in prog_attribs: - with (sysfs_path / "programmer" / attribute).open(encoding="utf-8") as file: + with (prog_path / attribute).open(encoding="utf-8") as file: parameters.append(file.read().rstrip()) return parameters def write_programmer_datasize(value: int) -> None: - with (sysfs_path / "programmer/datasize").open("w", encoding="utf-8") as file: + with Path("/sys/shepherd/programmer/datasize").open("w", encoding="utf-8") as file: file.write(str(value)) def start_programmer() -> None: - with (sysfs_path / "programmer/state").open("w", encoding="utf-8") as file: + with Path("/sys/shepherd/programmer/state").open("w", encoding="utf-8") as file: file.write("start") # force a pru-reset to jump into programming routine set_stop(force=True) def check_programmer() -> str: - with (sysfs_path / "programmer/state").open(encoding="utf-8") as file: + with Path("/sys/shepherd/programmer/state").open(encoding="utf-8") as file: return file.read().rstrip() @@ -604,10 +604,10 @@ def load_pru0_firmware(value: str = "shepherd") -> None: _count = 1 while _count < 6: try: - with (sysfs_path / "pru0_firmware").open("w", encoding="utf-8") as file: + with Path("/sys/shepherd/pru0_firmware").open("w", encoding="utf-8") as file: file.write(request) time.sleep(2) - with (sysfs_path / "pru0_firmware").open(encoding="utf-8") as file: + with Path("/sys/shepherd/pru0_firmware").open(encoding="utf-8") as file: result = file.read().rstrip() if result == request: return @@ -634,7 +634,7 @@ def pru0_firmware_is_default() -> bool: _count = 1 while _count < 6: try: - with (sysfs_path / "pru0_firmware").open(encoding="utf-8") as file: + with Path("/sys/shepherd/pru0_firmware").open(encoding="utf-8") as file: return file.read().rstrip() in pru0_firmwares[0] except OSError: # noqa: PERF203 log.warning( @@ -662,35 +662,35 @@ def pru0_firmware_is_default() -> bool: def get_mode() -> str: - with (sysfs_path / "mode").open(encoding="utf-8") as f: + with Path("/sys/shepherd/mode").open(encoding="utf-8") as f: return str(f.read().rstrip()) def get_state() -> str: - with (sysfs_path / "state").open(encoding="utf-8") as f: + with Path("/sys/shepherd/state").open(encoding="utf-8") as f: return str(f.read().rstrip()) def get_n_buffers() -> int: - with (sysfs_path / "n_buffers").open(encoding="utf-8") as f: + with Path("/sys/shepherd/n_buffers").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_buffer_period_ns() -> int: - with (sysfs_path / "buffer_period_ns").open(encoding="utf-8") as f: + with Path("/sys/shepherd/buffer_period_ns").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_samples_per_buffer() -> int: - with (sysfs_path / "samples_per_buffer").open(encoding="utf-8") as f: + with Path("/sys/shepherd/samples_per_buffer").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_mem_address() -> int: - with (sysfs_path / "memory/address").open(encoding="utf-8") as f: + with Path("/sys/shepherd/memory/address").open(encoding="utf-8") as f: return int(f.read().rstrip()) def get_mem_size() -> int: - with (sysfs_path / "memory/size").open(encoding="utf-8") as f: + with Path("/sys/shepherd/memory/size").open(encoding="utf-8") as f: return int(f.read().rstrip()) diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 9961caa9..247f7215 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -6,9 +6,11 @@ import pytest from click.testing import CliRunner +from pyfakefs.fake_filesystem_unittest import patchfs, Patcher + from shepherd_sheep.sysfs_interface import load_kernel_module from shepherd_sheep.sysfs_interface import remove_kernel_module - +from shepherd_sheep import sysfs_interface def check_beagleboard() -> bool: with suppress(Exception), Path("/proc/cpuinfo").open(encoding="utf-8-sig") as info: @@ -84,13 +86,14 @@ def shepherd_up(fake_hardware, shepherd_down: None) -> Generator[None, None, Non # TODO: design tests for programmer, also check if all hardware-tests need real hw # -> there should be more tests that don't require a pru ] - for file_, content in files: - fake_hardware.create_file(file_, contents=content) - here = Path(__file__).resolve().parent - fake_hardware.add_real_file(here / "_test_config_emulation.yaml") - fake_hardware.add_real_file(here / "_test_config_harvest.yaml") - fake_hardware.add_real_file(here / "_test_config_virtsource.yaml") - yield + with Patcher(modules_to_reload=[sysfs_interface]) as patcher: + for file_, content in files: + patcher.fs.create_file(file_, contents=content) + here = Path(__file__).resolve().parent + patcher.fs.add_real_file(here / "_test_config_emulation.yaml") + patcher.fs.add_real_file(here / "_test_config_harvest.yaml") + patcher.fs.add_real_file(here / "_test_config_virtsource.yaml") + yield else: load_kernel_module() yield diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index 203d56a6..4bb1d851 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -109,7 +109,7 @@ def test_write_capedata(eeprom_retained: EEPROM, cape_data: CapeData) -> None: @pytest.mark.hardware def test_read_capedata(eeprom_with_data: EEPROM, cape_data: CapeData) -> None: cape_data = eeprom_with_data._read_cape_data() - for key in cape_data.keys(): + for key in cape_data: assert cape_data[key] == cape_data[key] diff --git a/software/shepherd-calibration/shepherd_cal/calibration_plot.py b/software/shepherd-calibration/shepherd_cal/calibration_plot.py index fff7f275..3314cebd 100644 --- a/software/shepherd-calibration/shepherd_cal/calibration_plot.py +++ b/software/shepherd-calibration/shepherd_cal/calibration_plot.py @@ -21,7 +21,7 @@ def plot_calibration( component, ) continue - for channel in msr_component.keys(): + for channel in msr_component: try: sample_points = msr_component[channel] xp = np.empty(len(sample_points)) From 64905fa9aae09a8bdce182dda8344e49f326d7d5 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 10:30:56 +0200 Subject: [PATCH 19/35] fix unittests --- software/python-package/shepherd_sheep/cli.py | 2 +- software/python-package/shepherd_sheep/eeprom.py | 2 +- software/python-package/shepherd_sheep/h5_writer.py | 1 + software/python-package/shepherd_sheep/shared_memory.py | 2 +- software/python-package/shepherd_sheep/shepherd_debug.py | 2 +- .../python-package/shepherd_sheep/shepherd_emulator.py | 2 +- .../python-package/shepherd_sheep/shepherd_harvester.py | 2 +- software/python-package/shepherd_sheep/shepherd_io.py | 2 +- software/python-package/shepherd_sheep/sysfs_interface.py | 8 ++++++-- software/python-package/tests/conftest.py | 6 +++--- software/shepherd-herd/shepherd_herd/herd.py | 2 +- software/shepherd-herd/shepherd_herd/herd_cli.py | 2 +- 12 files changed, 19 insertions(+), 14 deletions(-) diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index 952be15e..0775ee1a 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -13,7 +13,6 @@ import time from pathlib import Path from typing import TypedDict -from typing_extensions import Unpack import click import gevent @@ -22,6 +21,7 @@ from shepherd_core.data_models.task import ProgrammingTask from shepherd_core.data_models.testbed import ProgrammerProtocol from shepherd_core.inventory import Inventory +from typing_extensions import Unpack from . import Launcher from . import __version__ diff --git a/software/python-package/shepherd_sheep/eeprom.py b/software/python-package/shepherd_sheep/eeprom.py index bd8a8698..1fe6c484 100644 --- a/software/python-package/shepherd_sheep/eeprom.py +++ b/software/python-package/shepherd_sheep/eeprom.py @@ -14,10 +14,10 @@ import struct from contextlib import suppress from types import TracebackType -from typing_extensions import Self from shepherd_core.data_models.base.calibration import CalibrationCape from shepherd_core.data_models.base.calibration import CapeData +from typing_extensions import Self from .logger import log diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index 5698a6da..18b496d4 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -12,6 +12,7 @@ from types import TracebackType from typing import TYPE_CHECKING from typing import ClassVar + from typing_extensions import Self if TYPE_CHECKING: diff --git a/software/python-package/shepherd_sheep/shared_memory.py b/software/python-package/shepherd_sheep/shared_memory.py index 4d0098d4..7a71995b 100644 --- a/software/python-package/shepherd_sheep/shared_memory.py +++ b/software/python-package/shepherd_sheep/shared_memory.py @@ -5,11 +5,11 @@ from dataclasses import dataclass from datetime import timedelta from types import TracebackType -from typing_extensions import Self import numpy as np from shepherd_core.data_models import GpioTracing from shepherd_core.data_models import PowerTracing +from typing_extensions import Self from . import commons from . import sysfs_interface as sfs diff --git a/software/python-package/shepherd_sheep/shepherd_debug.py b/software/python-package/shepherd_sheep/shepherd_debug.py index e50436cd..2e8d63d4 100644 --- a/software/python-package/shepherd_sheep/shepherd_debug.py +++ b/software/python-package/shepherd_sheep/shepherd_debug.py @@ -1,7 +1,6 @@ import contextlib import time from typing import NoReturn -from typing_extensions import Self import msgpack import msgpack_numpy @@ -15,6 +14,7 @@ from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig from shepherd_core.data_models.content.virtual_source import ConverterPRUConfig from shepherd_core.data_models.testbed import TargetPort +from typing_extensions import Self from . import commons from . import sysfs_interface diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index 6ab8256c..d6e30997 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -4,7 +4,6 @@ from contextlib import ExitStack from datetime import datetime from types import TracebackType -from typing_extensions import Self from shepherd_core import CalibrationPair from shepherd_core import CalibrationSeries @@ -14,6 +13,7 @@ from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig from shepherd_core.data_models.content.virtual_source import ConverterPRUConfig from shepherd_core.data_models.task import EmulationTask +from typing_extensions import Self from . import commons from . import sysfs_interface diff --git a/software/python-package/shepherd_sheep/shepherd_harvester.py b/software/python-package/shepherd_sheep/shepherd_harvester.py index eb5a37ec..2cd34499 100644 --- a/software/python-package/shepherd_sheep/shepherd_harvester.py +++ b/software/python-package/shepherd_sheep/shepherd_harvester.py @@ -4,11 +4,11 @@ import time from contextlib import ExitStack from types import TracebackType -from typing_extensions import Self from shepherd_core import local_tz from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig from shepherd_core.data_models.task import HarvestTask +from typing_extensions import Self from . import sysfs_interface from .eeprom import retrieve_calibration diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 32b9dd22..354d99fa 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -10,7 +10,6 @@ import time from contextlib import suppress from types import TracebackType -from typing_extensions import Self from pydantic import validate_call from shepherd_core import CalibrationEmulator @@ -20,6 +19,7 @@ from shepherd_core.data_models.content.virtual_harvester import HarvesterPRUConfig from shepherd_core.data_models.content.virtual_source import ConverterPRUConfig from shepherd_core.data_models.testbed import TargetPort +from typing_extensions import Self from . import commons from . import sysfs_interface as sfs diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index cb075de2..68475485 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -286,7 +286,9 @@ def write_dac_aux_voltage_raw( voltage_raw = min(voltage_raw, 2**16 - 1) voltage_raw |= int(ch_link) << 20 voltage_raw |= int(cap_out) << 21 - with Path("/sys/shepherd/dac_auxiliary_voltage_raw").open("w", encoding="utf-8") as f: + with Path("/sys/shepherd/dac_auxiliary_voltage_raw").open( + "w", encoding="utf-8" + ) as f: log.debug("Sending raw auxiliary voltage (dac channel B): %d", voltage_raw) f.write(str(voltage_raw)) @@ -604,7 +606,9 @@ def load_pru0_firmware(value: str = "shepherd") -> None: _count = 1 while _count < 6: try: - with Path("/sys/shepherd/pru0_firmware").open("w", encoding="utf-8") as file: + with Path("/sys/shepherd/pru0_firmware").open( + "w", encoding="utf-8" + ) as file: file.write(request) time.sleep(2) with Path("/sys/shepherd/pru0_firmware").open(encoding="utf-8") as file: diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 247f7215..3681ead7 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -6,11 +6,11 @@ import pytest from click.testing import CliRunner -from pyfakefs.fake_filesystem_unittest import patchfs, Patcher - +from pyfakefs.fake_filesystem_unittest import Patcher +from shepherd_sheep import sysfs_interface from shepherd_sheep.sysfs_interface import load_kernel_module from shepherd_sheep.sysfs_interface import remove_kernel_module -from shepherd_sheep import sysfs_interface + def check_beagleboard() -> bool: with suppress(Exception), Path("/proc/cpuinfo").open(encoding="utf-8-sig") as info: diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index cc947baa..b08520e7 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -11,7 +11,6 @@ from types import TracebackType from typing import Any from typing import ClassVar -from typing_extensions import Self import yaml from fabric import Connection @@ -28,6 +27,7 @@ from shepherd_core.data_models.task import extract_tasks from shepherd_core.data_models.task import prepare_task from shepherd_core.data_models.testbed import Testbed +from typing_extensions import Self from .logger import logger diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index 262e47ff..6fb0cc7e 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -3,7 +3,6 @@ from datetime import datetime from pathlib import Path from typing import TypedDict -from typing_extensions import Unpack import click import shepherd_core @@ -12,6 +11,7 @@ from shepherd_core.data_models.task import ProgrammingTask from shepherd_core.data_models.testbed import ProgrammerProtocol from shepherd_core.data_models.testbed import TargetPort +from typing_extensions import Unpack from . import __version__ from .herd import Herd From d88319c89e15d21b2ef13f686ca94589557573cc Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 10:31:27 +0200 Subject: [PATCH 20/35] launcher - proper wd-thread --- .../python-package/shepherd_sheep/launcher.py | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index 45493a05..cc77cdb9 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -14,7 +14,7 @@ from threading import Event from threading import Thread from types import TracebackType -from typing import Callable +from collections.abc import Callable from typing_extensions import Self @@ -26,18 +26,6 @@ from periphery import GPIO -def call_repeatedly(interval: float, func: Callable, *args) -> Callable: - stopped = Event() - - def loop() -> None: - while not stopped.wait(interval): - # the first call is in `interval` secs - func(*args) - - Thread(target=loop).start() - return stopped.set - - class Launcher: """Stores data coming from PRU's in HDF5 format. @@ -67,7 +55,10 @@ def __enter__(self) -> Self: self.gpio_ack_watchdog = GPIO(self.pin_ack_watchdog, "out") self.gpio_button.edge = "falling" log.debug("configured gpio") - self.cancel_wd_timer = call_repeatedly(interval=600, func=self.ack_watchdog) + self.wd_event = Event() + self.wd_interval = 600 + self.wd_thread = threading.Thread(target=self._thread_ack_watchdog, daemon=True) + self.wd_thread.start() sys_bus = dbus.SystemBus() systemd1 = sys_bus.get_object( @@ -91,6 +82,7 @@ def __exit__( tb: TracebackType | None = None, extra_arg: int = 0, ) -> None: + self.wd_event.set() self.gpio_led.close() self.gpio_button.close() @@ -210,12 +202,13 @@ def initiate_shutdown(self, timeout: int = 5) -> None: log.info("shutting down now") self.manager.PowerOff() - def ack_watchdog(self) -> None: + def _thread_ack_watchdog(self) -> None: """prevent system-reset from watchdog hw-rev2 has a watchdog that can turn on the BB every ~60 min """ - self.gpio_ack_watchdog.write(True) - time.sleep(0.002) - self.gpio_ack_watchdog.write(False) - log.debug("Signaled ACK to Watchdog") + while not self.wd_event.wait(self.wd_interval): + self.gpio_ack_watchdog.write(True) + self.wd_event.wait(0.002) + self.gpio_ack_watchdog.write(False) + log.debug("Signaled ACK to Watchdog") From 66c8520822df0fb719ecfd7cd40761f53162bd7d Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 11:53:54 +0200 Subject: [PATCH 21/35] fully typed python --- software/python-package/shepherd_sheep/cli.py | 3 ++- .../python-package/shepherd_sheep/h5_writer.py | 2 +- .../python-package/shepherd_sheep/launcher.py | 6 ++---- .../shepherd_sheep/sysfs_interface.py | 6 ++++-- software/python-package/tests/conftest.py | 15 ++++++++++----- software/python-package/tests/test_eeprom.py | 4 +++- software/python-package/tests/test_harvest.py | 2 +- .../python-package/tests/test_virtual_source.py | 2 +- .../python-package/tests_manual/profiler_cli.py | 3 ++- .../shepherd_cal/cli_helper.py | 3 ++- .../shepherd_cal/profile_calibration.py | 2 ++ software/shepherd-herd/shepherd_herd/herd_cli.py | 3 ++- 12 files changed, 32 insertions(+), 19 deletions(-) diff --git a/software/python-package/shepherd_sheep/cli.py b/software/python-package/shepherd_sheep/cli.py index 0775ee1a..22d7258c 100644 --- a/software/python-package/shepherd_sheep/cli.py +++ b/software/python-package/shepherd_sheep/cli.py @@ -12,6 +12,7 @@ import sys import time from pathlib import Path +from types import FrameType from typing import TypedDict import click @@ -61,7 +62,7 @@ # - redone programmer, emulation -def exit_gracefully(*_args) -> None: +def exit_gracefully(_signum: int, _frame: FrameType | None) -> None: log.warning("Aborted!") sys.exit(0) diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index 18b496d4..c9f88468 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -231,7 +231,7 @@ def write_buffer(self, buffer: DataBuffer) -> None: ] = buffer.gpio_edges.timestamps_ns self.gpio_grp["value"][ self.gpio_pos : gpio_new_pos - ] = buffer.gpio_edges.values + ] = buffer.gpio_edges.values # noqa: PD011, false positive self.gpio_pos = gpio_new_pos if (buffer.util_mean > 95) or (buffer.util_max > 100): diff --git a/software/python-package/shepherd_sheep/launcher.py b/software/python-package/shepherd_sheep/launcher.py index cc77cdb9..cd978053 100644 --- a/software/python-package/shepherd_sheep/launcher.py +++ b/software/python-package/shepherd_sheep/launcher.py @@ -9,12 +9,10 @@ :license: MIT, see LICENSE for more details. """ import os +import threading import time from contextlib import suppress -from threading import Event -from threading import Thread from types import TracebackType -from collections.abc import Callable from typing_extensions import Self @@ -55,7 +53,7 @@ def __enter__(self) -> Self: self.gpio_ack_watchdog = GPIO(self.pin_ack_watchdog, "out") self.gpio_button.edge = "falling" log.debug("configured gpio") - self.wd_event = Event() + self.wd_event = threading.Event() self.wd_interval = 600 self.wd_thread = threading.Thread(target=self._thread_ack_watchdog, daemon=True) self.wd_thread.start() diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index 68475485..2b4657dc 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -287,7 +287,8 @@ def write_dac_aux_voltage_raw( voltage_raw |= int(ch_link) << 20 voltage_raw |= int(cap_out) << 21 with Path("/sys/shepherd/dac_auxiliary_voltage_raw").open( - "w", encoding="utf-8" + "w", + encoding="utf-8", ) as f: log.debug("Sending raw auxiliary voltage (dac channel B): %d", voltage_raw) f.write(str(voltage_raw)) @@ -607,7 +608,8 @@ def load_pru0_firmware(value: str = "shepherd") -> None: while _count < 6: try: with Path("/sys/shepherd/pru0_firmware").open( - "w", encoding="utf-8" + "w", + encoding="utf-8", ) as file: file.write(request) time.sleep(2) diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 3681ead7..6e681c4b 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -1,5 +1,6 @@ import gc from collections.abc import Generator +from collections.abc import Iterable from contextlib import suppress from pathlib import Path from typing import Any @@ -25,7 +26,7 @@ def check_beagleboard() -> bool: pytest.param("fake_hardware", marks=pytest.mark.fake_hardware), ], ) -def fake_hardware(request) -> Generator[Any, None, None]: +def fake_hardware(request: pytest.FixtureRequest) -> Generator[Any, None, None]: if request.param == "fake_hardware": request.fixturenames.append("fs") # needs pyfakefs installed fake_sysfs = request.getfixturevalue("fs") @@ -36,7 +37,7 @@ def fake_hardware(request) -> Generator[Any, None, None]: yield None -def pytest_addoption(parser) -> None: +def pytest_addoption(parser: pytest.Parser) -> None: parser.addoption( "--eeprom-write", action="store_true", @@ -45,7 +46,9 @@ def pytest_addoption(parser) -> None: ) -def pytest_collection_modifyitems(config, items) -> None: +def pytest_collection_modifyitems( + config: pytest.Config, items: Iterable[pytest.Item] +) -> None: skip_fake = pytest.mark.skip(reason="cannot be faked") skip_eeprom_write = pytest.mark.skip(reason="requires --eeprom-write option") skip_missing_hardware = pytest.mark.skip(reason="no hw to test on") @@ -63,7 +66,9 @@ def pytest_collection_modifyitems(config, items) -> None: @pytest.fixture() -def shepherd_up(fake_hardware, shepherd_down: None) -> Generator[None, None, None]: +def shepherd_up( + fake_hardware: pytest.Mark, shepherd_down: None +) -> Generator[None, None, None]: if fake_hardware is not None: files = [ ("/sys/shepherd/state", "idle"), @@ -102,7 +107,7 @@ def shepherd_up(fake_hardware, shepherd_down: None) -> Generator[None, None, Non @pytest.fixture() -def shepherd_down(fake_hardware) -> None: +def shepherd_down(fake_hardware: pytest.Mark) -> None: if fake_hardware is None: remove_kernel_module() diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index 4bb1d851..b4c7d477 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -22,7 +22,9 @@ def data_test_string() -> bytes: @pytest.fixture() -def eeprom_open(request, fake_hardware) -> Generator[EEPROM, None, None]: +def eeprom_open( + request: pytest.FixtureRequest, fake_hardware: pytest.Mark +) -> Generator[EEPROM, None, None]: if fake_hardware is not None: fake_hardware.create_file("/sys/bus/i2c/devices/2-0054/eeprom", st_size=32768) request.applymarker( diff --git a/software/python-package/tests/test_harvest.py b/software/python-package/tests/test_harvest.py index 8eddf533..9e39b3c9 100644 --- a/software/python-package/tests/test_harvest.py +++ b/software/python-package/tests/test_harvest.py @@ -13,7 +13,7 @@ @pytest.fixture(params=["harvester"]) # TODO: there is a second mode now -def mode(request) -> str: +def mode(request: pytest.FixtureRequest) -> str: return request.param diff --git a/software/python-package/tests/test_virtual_source.py b/software/python-package/tests/test_virtual_source.py index cdeab281..fcb08ca0 100644 --- a/software/python-package/tests/test_virtual_source.py +++ b/software/python-package/tests/test_virtual_source.py @@ -11,7 +11,7 @@ @pytest.fixture -def src_cfg(request) -> VirtualSourceConfig: +def src_cfg(request: pytest.FixtureRequest) -> VirtualSourceConfig: marker = request.node.get_closest_marker("src_name") src_name = None if marker is None else marker.args[0] diff --git a/software/python-package/tests_manual/profiler_cli.py b/software/python-package/tests_manual/profiler_cli.py index 7c7e53ec..25c15657 100644 --- a/software/python-package/tests_manual/profiler_cli.py +++ b/software/python-package/tests_manual/profiler_cli.py @@ -18,7 +18,8 @@ Profile Import-Time (properly): sudo python3 -X importtime -c 'from shepherd_sheep.cli import cli' 2> importtime.log -sudo python3 -X importtime -c 'from shepherd_core.data_models.task import EmulationTask' 2> importtime.log +sudo python3 -X importtime -c + 'from shepherd_core.data_models.task import EmulationTask' 2> importtime.log Timing-Optimizations: - import EmulationTask -> from 47 s to 8.4 s diff --git a/software/shepherd-calibration/shepherd_cal/cli_helper.py b/software/shepherd-calibration/shepherd_cal/cli_helper.py index 458ad826..d0565d6b 100644 --- a/software/shepherd-calibration/shepherd_cal/cli_helper.py +++ b/software/shepherd-calibration/shepherd_cal/cli_helper.py @@ -1,5 +1,6 @@ import signal import sys +from types import FrameType import click import shepherd_core @@ -10,7 +11,7 @@ from .logger import set_verbosity -def exit_gracefully(*_args) -> None: +def exit_gracefully(_signum: int, _frame: FrameType | None) -> None: logger.warning("Aborted!") sys.exit(0) diff --git a/software/shepherd-calibration/shepherd_cal/profile_calibration.py b/software/shepherd-calibration/shepherd_cal/profile_calibration.py index 8c53e2b4..35e48b20 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_calibration.py +++ b/software/shepherd-calibration/shepherd_cal/profile_calibration.py @@ -10,6 +10,8 @@ from .logger import logger +# ruff: noqa: PD008 + T_calc = TypeVar("T_calc", NDArray[np.float64], float) diff --git a/software/shepherd-herd/shepherd_herd/herd_cli.py b/software/shepherd-herd/shepherd_herd/herd_cli.py index 6fb0cc7e..0fc6e5b9 100644 --- a/software/shepherd-herd/shepherd_herd/herd_cli.py +++ b/software/shepherd-herd/shepherd_herd/herd_cli.py @@ -2,6 +2,7 @@ import sys from datetime import datetime from pathlib import Path +from types import FrameType from typing import TypedDict import click @@ -27,7 +28,7 @@ # - arguments can be configured in a dict and standardized across tools -def exit_gracefully(*_args) -> None: +def exit_gracefully(_signum: int, _frame: FrameType | None) -> None: log.warning("Aborted!") sys.exit(0) From 9ba30306a068dbfaba9163f3ef28565ae562b633 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 11:54:23 +0200 Subject: [PATCH 22/35] more responsive monitors --- .../python-package/shepherd_sheep/monitor_sysutil.py | 3 +-- software/python-package/shepherd_sheep/monitor_uart.py | 10 ++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/software/python-package/shepherd_sheep/monitor_sysutil.py b/software/python-package/shepherd_sheep/monitor_sysutil.py index 7af3d05e..5f7b6916 100644 --- a/software/python-package/shepherd_sheep/monitor_sysutil.py +++ b/software/python-package/shepherd_sheep/monitor_sysutil.py @@ -93,7 +93,7 @@ def thread_fn(self) -> None: https://psutil.readthedocs.io/en/latest/#cpu :return: none """ - while not self.event.is_set(): + while not self.event.wait(self.poll_intervall): # rate limiter & exit ts_now_ns = int(time.time() * 1e9) if ts_now_ns >= self.log_timestamp_ns: data_length = self.data["time"].shape[0] @@ -125,5 +125,4 @@ def thread_fn(self) -> None: self.position += 1 # TODO: add temp, not working: # https://psutil.readthedocs.io/en/latest/#psutil.sensors_temperatures - self.event.wait(self.poll_intervall) # rate limiter log.debug("[%s] thread ended itself", type(self).__name__) diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index f04f1c3c..fb82d7df 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -70,17 +70,15 @@ def thread_fn(self) -> None: # tried 'S' and opaque-type -> failed with errors # - converting is producing ValueError on certain chars, # errors="backslashreplace" does not help - # TODO: eval https://pyserial.readthedocs.io/en/latest/pyserial_api.html#serial.to_bytes + # https://pyserial.readthedocs.io/en/latest/pyserial_api.html#serial.to_bytes + # TODO: is there a way to signal backpressure? try: # open serial as non-exclusive with serial.Serial(self.uart, self.baudrate, timeout=0) as uart: - while not self.event.is_set(): + while not self.event.wait(self.poll_intervall): # rate limiter & exit if uart.in_waiting > 0: # hdf5 can embed raw bytes, but can't handle nullbytes output = uart.read(uart.in_waiting).replace(b"\x00", b"") - if self.event.is_set(): - # needed because uart.read is blocking - break if len(output) > 0: data_length = self.data["time"].shape[0] if self.position >= data_length: @@ -92,7 +90,7 @@ def thread_fn(self) -> None: ) self.data["message"][self.position] = output self.position += 1 - self.event.wait(self.poll_intervall) # rate limiter + except RuntimeError: log.error("[%s] HDF5-File unavailable - will stop", type(self).__name__) except ValueError as e: From fa385da3c754adb2a1d4bfc4939d581c45853965 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 11:54:50 +0200 Subject: [PATCH 23/35] Update shepherd_io.py --- .../shepherd_sheep/shepherd_io.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 354d99fa..967e7337 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -20,6 +20,8 @@ from shepherd_core.data_models.content.virtual_source import ConverterPRUConfig from shepherd_core.data_models.testbed import TargetPort from typing_extensions import Self +from typing_extensions import TypedDict +from typing_extensions import Unpack from . import commons from . import sysfs_interface as sfs @@ -63,10 +65,10 @@ class ShepherdIO: """ # This _instance-element is part of the singleton implementation - _instance = None + _instance: Self | None = None @classmethod - def __new__(cls, *_args, **_kwds) -> Self: + def __new__(cls, **_kwargs: Unpack[TypedDict]) -> Self: """Implements singleton class.""" if ShepherdIO._instance is None: new_class = object.__new__(cls) @@ -113,7 +115,7 @@ def __init__( self.shared_mem: SharedMemory def __del__(self) -> None: - log.debug("Now deleting ShepherdIO") + log.debug("Now deleting ShepherdIO-Instance") ShepherdIO._instance = None def __enter__(self) -> Self: @@ -138,7 +140,8 @@ def __enter__(self) -> Self: except Exception: log.exception("ShepherdIO.Init caught an exception -> exit now") - self._cleanup() + self._power_down_shp() + self._unload_shared_mem() raise sfs.wait_for_state("idle", 3) @@ -152,7 +155,8 @@ def __exit__( extra_arg: int = 0, ) -> None: log.info("Now exiting ShepherdIO") - self._cleanup() + self._power_down_shp() + self._unload_shared_mem() ShepherdIO._instance = None @staticmethod @@ -260,7 +264,12 @@ def refresh_shared_mem(self) -> None: ) self.shared_mem.__enter__() - def _cleanup(self) -> None: + def _unload_shared_mem(self) -> None: + if self.shared_mem is not None: + self.shared_mem.__exit__() + self.shared_mem = None + + def _power_down_shp(self) -> None: log.debug("ShepherdIO is commanded to power down / cleanup") count = 1 while count < 6 and sfs.get_state() != "idle": @@ -286,10 +295,6 @@ def _cleanup(self) -> None: ) self.set_aux_target_voltage(0.0) - if self.shared_mem is not None: - self.shared_mem.__exit__() - self.shared_mem = None - self.set_io_level_converter(False) self.set_power_state_emulator(False) self.set_power_state_recorder(False) From e5e353ae23c4b021853d2a291ecebd73c983c41f Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 11:55:28 +0200 Subject: [PATCH 24/35] allow uart to release some backpressure after shutdown --- software/python-package/shepherd_sheep/shepherd_emulator.py | 2 ++ software/python-package/shepherd_sheep/shepherd_harvester.py | 1 + 2 files changed, 3 insertions(+) diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index d6e30997..de4255de 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -185,6 +185,8 @@ def __exit__( tb: TracebackType | None = None, extra_arg: int = 0, ) -> None: + super()._power_down_shp() + time.sleep(2) # TODO: experimental - for releasing uart-backpressure self.stack.close() super().__exit__() diff --git a/software/python-package/shepherd_sheep/shepherd_harvester.py b/software/python-package/shepherd_sheep/shepherd_harvester.py index 2cd34499..5cb957b0 100644 --- a/software/python-package/shepherd_sheep/shepherd_harvester.py +++ b/software/python-package/shepherd_sheep/shepherd_harvester.py @@ -117,6 +117,7 @@ def __exit__( tb: TracebackType | None = None, extra_arg: int = 0, ) -> None: + super()._power_down_shp() self.stack.close() super().__exit__() From 83a4ba2822d733913e85485d5bf84d3ef947785c Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 12:12:07 +0200 Subject: [PATCH 25/35] cleanup of linters --- .pre-commit-config.yaml | 1 - pyproject.toml | 18 ++++++++---------- .../python-package/shepherd_sheep/__init__.py | 6 +++--- .../shepherd_sheep/monitor_kernel.py | 2 +- .../shepherd_sheep/monitor_ptp.py | 2 +- .../shepherd_sheep/monitor_uart.py | 4 ++-- .../shepherd_sheep/sysfs_interface.py | 2 +- software/python-package/tests/conftest.py | 6 ++++-- software/python-package/tests/test_eeprom.py | 3 ++- .../tests_manual/profiler_cli.py | 2 +- .../shepherd_cal/calibrator.py | 2 +- .../shepherd_cal/profile_analyzer.py | 4 ++-- software/shepherd-herd/shepherd_herd/herd.py | 6 +++--- 13 files changed, 29 insertions(+), 29 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ead2f24d..55a3c79e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -102,7 +102,6 @@ repos: rev: 'v0.1.1' hooks: - id: ruff - args: ["--fix", "--exit-non-zero-on-fix"] # manual run: ruff check . --preview # TODO: disable, until bug is resolved, maybe connected to diff --git a/pyproject.toml b/pyproject.toml index 2d1d421c..d2b03bf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,28 +51,26 @@ extend-ignore = [ # open() vs os.open() "SCS109", # hideous commas - "C812", - "C815", - "C813", + "C812", "C815", "C813", # DOCString TODO: reduce here - "D100", - "D102", - "D103", + "D100", "D102", "D103", + # handled by ruff "G200", + "S101", "S108", "S404", "S603", + "SCS108", + "E501", "E800", ] per-file-ignores = """ # asserts (in tests are ok), also allow print() instead of logger, -*/tests_manual/*: E501 -*/tests/*: SCS108, S101, T201 +*/tests/*: T201 # commented out code, TODO: finalize model -*_model.py: E800 # telnet for openOCD, for now, TODO: replace by direct ssh.run() software/shepherd-herd/shepherd_herd/open_ocd_cli.py: S401, S312 # allow print() instead of logger, # software/python-package/tests/test_virtual_source.py: T201 # under development - so more -software/firmware/pru0-cython-module/*: E800, T201, F401 +software/firmware/pru0-cython-module/*: T201, F401 """ exclude = "software/shepherd-devicetest/*" diff --git a/software/python-package/shepherd_sheep/__init__.py b/software/python-package/shepherd_sheep/__init__.py index 0a9a25af..505f5c17 100644 --- a/software/python-package/shepherd_sheep/__init__.py +++ b/software/python-package/shepherd_sheep/__init__.py @@ -9,7 +9,7 @@ """ import platform import shutil -import subprocess # noqa: S404 +import subprocess import tempfile import time from contextlib import ExitStack @@ -210,7 +210,7 @@ def run_programmer(cfg: ProgrammingTask) -> bool: log.error("OSError - Failed to initialize Programmer") failed = True except ValueError as xpt: - log.exception("ValueError: %s", str(xpt)) # noqa: G200 + log.exception("ValueError: %s", str(xpt)) failed = True state = "init" @@ -250,7 +250,7 @@ def run_task(cfg: ShpModel | Path | str) -> bool: wrapper = prepare_task(cfg, observer_name) content = extract_tasks(wrapper) except ValueError as xcp: - log.error( # noqa: G200 + log.error( "Task-Set was not usable for this observer '%s', with original error = %s", observer_name, xcp, diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index 8bc5df97..d9ac5a2c 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -1,5 +1,5 @@ import os -import subprocess # noqa: S404 +import subprocess import threading import time from types import TracebackType diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index f5dc0edf..022a2ae5 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -1,5 +1,5 @@ import os -import subprocess # noqa: S404 +import subprocess import threading import time from types import TracebackType diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index fb82d7df..f2db7b47 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -94,7 +94,7 @@ def thread_fn(self) -> None: except RuntimeError: log.error("[%s] HDF5-File unavailable - will stop", type(self).__name__) except ValueError as e: - log.error( # noqa: G200 + log.error( "[%s] PySerial ValueError '%s' - " "couldn't configure serial-port '%s' " "with baudrate=%d -> prevents logging", @@ -104,7 +104,7 @@ def thread_fn(self) -> None: self.baudrate, ) except serial.SerialException as e: - log.error( # noqa: G200 + log.error( "[%s] pySerial SerialException '%s - " "Couldn't open Serial-Port '%s' to target -> prevents logging", type(self).__name__, diff --git a/software/python-package/shepherd_sheep/sysfs_interface.py b/software/python-package/shepherd_sheep/sysfs_interface.py index 2b4657dc..e18aeeef 100644 --- a/software/python-package/shepherd_sheep/sysfs_interface.py +++ b/software/python-package/shepherd_sheep/sysfs_interface.py @@ -8,7 +8,7 @@ :copyright: (c) 2019 Networked Embedded Systems Lab, TU Dresden. :license: MIT, see LICENSE for more details. """ -import subprocess # noqa: S404 +import subprocess import sys import time from pathlib import Path diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 6e681c4b..889fc0da 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -47,7 +47,8 @@ def pytest_addoption(parser: pytest.Parser) -> None: def pytest_collection_modifyitems( - config: pytest.Config, items: Iterable[pytest.Item] + config: pytest.Config, + items: Iterable[pytest.Item], ) -> None: skip_fake = pytest.mark.skip(reason="cannot be faked") skip_eeprom_write = pytest.mark.skip(reason="requires --eeprom-write option") @@ -67,7 +68,8 @@ def pytest_collection_modifyitems( @pytest.fixture() def shepherd_up( - fake_hardware: pytest.Mark, shepherd_down: None + fake_hardware: pytest.Mark, + shepherd_down: None, ) -> Generator[None, None, None]: if fake_hardware is not None: files = [ diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index b4c7d477..c7d8d71c 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -23,7 +23,8 @@ def data_test_string() -> bytes: @pytest.fixture() def eeprom_open( - request: pytest.FixtureRequest, fake_hardware: pytest.Mark + request: pytest.FixtureRequest, + fake_hardware: pytest.Mark, ) -> Generator[EEPROM, None, None]: if fake_hardware is not None: fake_hardware.create_file("/sys/bus/i2c/devices/2-0054/eeprom", st_size=32768) diff --git a/software/python-package/tests_manual/profiler_cli.py b/software/python-package/tests_manual/profiler_cli.py index 25c15657..3ba7a054 100644 --- a/software/python-package/tests_manual/profiler_cli.py +++ b/software/python-package/tests_manual/profiler_cli.py @@ -53,6 +53,6 @@ """, ) -print(f"Routine took {time.time() - time_start} s") # noqa: T201 +print(f"Routine took {time.time() - time_start} s") prof.create_stats() prof.dump_stats(path_log) diff --git a/software/shepherd-calibration/shepherd_cal/calibrator.py b/software/shepherd-calibration/shepherd_cal/calibrator.py index a3299b53..e71a622c 100644 --- a/software/shepherd-calibration/shepherd_cal/calibrator.py +++ b/software/shepherd-calibration/shepherd_cal/calibrator.py @@ -387,7 +387,7 @@ def write( cal_file = Path(cal_file) CalibrationCape.from_file(cal_file) # test data - self._cnx.put(cal_file, temp_file) # noqa: S108 + self._cnx.put(cal_file, temp_file) logger.info("----------EEPROM WRITE------------") logger.info("Local file: %s", cal_file.as_posix()) logger.info("Remote file: %s", temp_file) diff --git a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py index c779508b..e3749430 100644 --- a/software/shepherd-calibration/shepherd_cal/profile_analyzer.py +++ b/software/shepherd-calibration/shepherd_cal/profile_analyzer.py @@ -50,8 +50,8 @@ def analyze_directory( for component in profile.data: for filtered in [True, False]: profile.quiver_setpoints_offset(component, filtered) - # profile.scatter_setpoints_stddev(component, filtered) # noqa: E800 - # profile.scatter_setpoints_dynamic(component, filtered) # noqa: E800 + # profile.scatter_setpoints_stddev(component, filtered) + # profile.scatter_setpoints_dynamic(component, filtered) stat_df = pd.concat(stats_list, axis=0) stat_df.to_csv(stats_path, sep=";", decimal=",", index=False) diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index b08520e7..406b1986 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -198,7 +198,7 @@ def _open(self) -> None: @staticmethod def _thread_run( cnx: Connection, - sudo: bool, # noqa: FBT001 + sudo: bool, cmd: str, results: dict[int, Result], index: int, @@ -264,7 +264,7 @@ def _thread_put( cnx: Connection, src: Path | StringIO, dst: Path, - force_overwrite: bool, # noqa: FBT001 + force_overwrite: bool, ) -> None: if isinstance(src, StringIO): filename = dst.name @@ -281,7 +281,7 @@ def _thread_put( tmp_path = Path("/tmp") / filename # noqa: S108 logger.debug("temp-path for %s is %s", cnx.host, tmp_path) try: - cnx.put(src, str(tmp_path)) # noqa: S108 + cnx.put(src, str(tmp_path)) xtr_arg = "-f" if force_overwrite else "-n" cnx.sudo(f"mv {xtr_arg} {tmp_path} {dst}", warn=True, hide=True) except (NoValidConnectionsError, SSHException, TimeoutError): From 3d10953b9cb6bac4a8d14ea29809a0f8762166ea Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 12:54:34 +0200 Subject: [PATCH 26/35] clean up name collisions --- pyproject.toml | 1 + software/python-package/tests/conftest.py | 33 ++++++++++---------- software/python-package/tests/test_eeprom.py | 13 ++++---- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d2b03bf5..add1bc2c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ extend-ignore = [ per-file-ignores = """ # asserts (in tests are ok), also allow print() instead of logger, +*/tests_manual/*: T201 */tests/*: T201 # commented out code, TODO: finalize model # telnet for openOCD, for now, TODO: replace by direct ssh.run() diff --git a/software/python-package/tests/conftest.py b/software/python-package/tests/conftest.py index 889fc0da..3017319d 100644 --- a/software/python-package/tests/conftest.py +++ b/software/python-package/tests/conftest.py @@ -3,12 +3,10 @@ from collections.abc import Iterable from contextlib import suppress from pathlib import Path -from typing import Any import pytest from click.testing import CliRunner -from pyfakefs.fake_filesystem_unittest import Patcher -from shepherd_sheep import sysfs_interface +from pyfakefs.fake_filesystem import FakeFilesystem from shepherd_sheep.sysfs_interface import load_kernel_module from shepherd_sheep.sysfs_interface import remove_kernel_module @@ -26,10 +24,12 @@ def check_beagleboard() -> bool: pytest.param("fake_hardware", marks=pytest.mark.fake_hardware), ], ) -def fake_hardware(request: pytest.FixtureRequest) -> Generator[Any, None, None]: +def fake_fs( + request: pytest.FixtureRequest, +) -> Generator[FakeFilesystem | None, None, None]: if request.param == "fake_hardware": request.fixturenames.append("fs") # needs pyfakefs installed - fake_sysfs = request.getfixturevalue("fs") + fake_sysfs: FakeFilesystem = request.getfixturevalue("fs") fake_sysfs.create_dir("/sys/class/remoteproc/remoteproc1") fake_sysfs.create_dir("/sys/class/remoteproc/remoteproc2") yield fake_sysfs @@ -68,10 +68,10 @@ def pytest_collection_modifyitems( @pytest.fixture() def shepherd_up( - fake_hardware: pytest.Mark, + fake_fs: FakeFilesystem | None, shepherd_down: None, ) -> Generator[None, None, None]: - if fake_hardware is not None: + if fake_fs is not None: files = [ ("/sys/shepherd/state", "idle"), ("/sys/shepherd/mode", "harvester"), @@ -93,14 +93,13 @@ def shepherd_up( # TODO: design tests for programmer, also check if all hardware-tests need real hw # -> there should be more tests that don't require a pru ] - with Patcher(modules_to_reload=[sysfs_interface]) as patcher: - for file_, content in files: - patcher.fs.create_file(file_, contents=content) - here = Path(__file__).resolve().parent - patcher.fs.add_real_file(here / "_test_config_emulation.yaml") - patcher.fs.add_real_file(here / "_test_config_harvest.yaml") - patcher.fs.add_real_file(here / "_test_config_virtsource.yaml") - yield + for file_, content in files: + fake_fs.create_file(file_, contents=content) + here = Path(__file__).resolve().parent + fake_fs.add_real_file(here / "_test_config_emulation.yaml") + fake_fs.add_real_file(here / "_test_config_harvest.yaml") + fake_fs.add_real_file(here / "_test_config_virtsource.yaml") + yield else: load_kernel_module() yield @@ -109,8 +108,8 @@ def shepherd_up( @pytest.fixture() -def shepherd_down(fake_hardware: pytest.Mark) -> None: - if fake_hardware is None: +def shepherd_down(fake_fs: FakeFilesystem | None) -> None: + if fake_fs is None: remove_kernel_module() diff --git a/software/python-package/tests/test_eeprom.py b/software/python-package/tests/test_eeprom.py index c7d8d71c..89a17473 100644 --- a/software/python-package/tests/test_eeprom.py +++ b/software/python-package/tests/test_eeprom.py @@ -1,6 +1,7 @@ from collections.abc import Generator import pytest +from pyfakefs.fake_filesystem import FakeFilesystem from shepherd_core import CalibrationCape from shepherd_core.data_models.base.calibration import CapeData from shepherd_sheep import EEPROM @@ -24,10 +25,10 @@ def data_test_string() -> bytes: @pytest.fixture() def eeprom_open( request: pytest.FixtureRequest, - fake_hardware: pytest.Mark, + fake_fs: FakeFilesystem | None, ) -> Generator[EEPROM, None, None]: - if fake_hardware is not None: - fake_hardware.create_file("/sys/bus/i2c/devices/2-0054/eeprom", st_size=32768) + if fake_fs is not None: + fake_fs.create_file("/sys/bus/i2c/devices/2-0054/eeprom", st_size=32768) request.applymarker( pytest.mark.xfail( raises=OSError, @@ -111,9 +112,9 @@ def test_write_capedata(eeprom_retained: EEPROM, cape_data: CapeData) -> None: @pytest.mark.eeprom_write @pytest.mark.hardware def test_read_capedata(eeprom_with_data: EEPROM, cape_data: CapeData) -> None: - cape_data = eeprom_with_data._read_cape_data() - for key in cape_data: - assert cape_data[key] == cape_data[key] + cape_data2 = eeprom_with_data._read_cape_data() + for key, _ in cape_data: + assert cape_data[key] == cape_data2[key] @pytest.mark.eeprom_write From a268f909af7efe4d4a4c33fc0275c2c6f7fe786a Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 13:43:46 +0200 Subject: [PATCH 27/35] bump V --- .bumpversion.cfg | 4 +++- software/kernel-module/src/module_base.c | 2 +- software/pps-gmtimer/src/pps-gmtimer.c | 2 +- software/python-package/shepherd_sheep/__init__.py | 2 +- software/shepherd-calibration/shepherd_cal/__init__.py | 2 +- software/shepherd-herd/shepherd_herd/__init__.py | 2 +- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 1047ed87..64cd08fe 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,8 +1,10 @@ [bumpversion] -current_version = 0.7.0 +current_version = 0.7.1 commit = False tag = False +# example: bump2version patch --allow-dirty + [bumpversion:file:software/kernel-module/src/module_base.c] [bumpversion:file:software/python-package/shepherd_sheep/__init__.py] diff --git a/software/kernel-module/src/module_base.c b/software/kernel-module/src/module_base.c index 868862de..c387f1cb 100644 --- a/software/kernel-module/src/module_base.c +++ b/software/kernel-module/src/module_base.c @@ -165,5 +165,5 @@ module_platform_driver(shepherd_driver); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Kai Geissdoerfer"); MODULE_DESCRIPTION("Shepherd kernel module for time synchronization and data exchange to PRUs"); -MODULE_VERSION("0.7.0"); +MODULE_VERSION("0.7.1"); // MODULE_ALIAS("rpmsg:rpmsg-shprd"); // TODO: is this still needed? diff --git a/software/pps-gmtimer/src/pps-gmtimer.c b/software/pps-gmtimer/src/pps-gmtimer.c index 75c7ef98..19548ff0 100644 --- a/software/pps-gmtimer/src/pps-gmtimer.c +++ b/software/pps-gmtimer/src/pps-gmtimer.c @@ -42,7 +42,7 @@ MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Drown"); MODULE_DESCRIPTION("PPS Client Driver using OMAP Timer hardware"); -MODULE_VERSION("0.7.0"); +MODULE_VERSION("0.7.1"); struct pps_gmtimer_platform_data { diff --git a/software/python-package/shepherd_sheep/__init__.py b/software/python-package/shepherd_sheep/__init__.py index 505f5c17..c17e29f4 100644 --- a/software/python-package/shepherd_sheep/__init__.py +++ b/software/python-package/shepherd_sheep/__init__.py @@ -43,7 +43,7 @@ from .sysfs_interface import flatten_list from .target_io import TargetIO -__version__ = "0.7.0" +__version__ = "0.7.1" __all__ = [ "Writer", diff --git a/software/shepherd-calibration/shepherd_cal/__init__.py b/software/shepherd-calibration/shepherd_cal/__init__.py index 8e101950..1629765c 100644 --- a/software/shepherd-calibration/shepherd_cal/__init__.py +++ b/software/shepherd-calibration/shepherd_cal/__init__.py @@ -6,7 +6,7 @@ from .profile_analyzer import analyze_directory from .profiler import Profiler -__version__ = "0.7.0" +__version__ = "0.7.1" __all__ = [ "Calibrator", diff --git a/software/shepherd-herd/shepherd_herd/__init__.py b/software/shepherd-herd/shepherd_herd/__init__.py index 689325f8..e6e0822c 100644 --- a/software/shepherd-herd/shepherd_herd/__init__.py +++ b/software/shepherd-herd/shepherd_herd/__init__.py @@ -14,7 +14,7 @@ from .logger import logger from .logger import set_verbosity -__version__ = "0.7.0" +__version__ = "0.7.1" __all__ = [ "Herd", From 5e3138f1a4fbad24326d1fc9172ae26b1acbfa2b Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 13:44:08 +0200 Subject: [PATCH 28/35] force upgrade packages --- software/python-package/setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/software/python-package/setup.cfg b/software/python-package/setup.cfg index 1b15d2f4..d3f0b500 100644 --- a/software/python-package/setup.cfg +++ b/software/python-package/setup.cfg @@ -64,8 +64,8 @@ dev = pyright test = - pytest - pyfakefs + pytest>7.4.0 + pyfakefs>5.0.0 pytest-timeout pytest-click coverage From f669fe6f81071ae3bd2efe061fda95efecdef6cc Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 13:44:17 +0200 Subject: [PATCH 29/35] fix singleton --- software/python-package/shepherd_sheep/shepherd_io.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index 967e7337..e4a7c132 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -68,15 +68,14 @@ class ShepherdIO: _instance: Self | None = None @classmethod - def __new__(cls, **_kwargs: Unpack[TypedDict]) -> Self: + def __new__(cls, *_args: tuple, **_kwargs: Unpack[TypedDict]) -> Self: """Implements singleton class.""" - if ShepherdIO._instance is None: - new_class = object.__new__(cls) - ShepherdIO._instance = new_class + if cls._instance is None: + cls._instance = object.__new__(cls) # was raising on reuse and stored weakref.ref before - return new_class + return cls._instance log.debug("ShepherdIO-Singleton reused") - return ShepherdIO._instance + return cls._instance def __init__( self, From 1c8054e5f8c5340e886d8db0e2a28d6666554e0b Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 15:44:50 +0200 Subject: [PATCH 30/35] avoid deadlocks --- .../shepherd_sheep/h5_writer.py | 2 +- .../shepherd_sheep/monitor_kernel.py | 7 +++++- .../shepherd_sheep/monitor_ptp.py | 7 +++++- .../shepherd_sheep/monitor_sheep.py | 7 +++++- .../shepherd_sheep/monitor_sysutil.py | 7 +++++- .../shepherd_sheep/monitor_uart.py | 7 +++++- .../shepherd_sheep/shepherd_emulator.py | 23 +++++++++++++++---- .../shepherd_sheep/shepherd_harvester.py | 13 ++++++++--- software/shepherd-herd/shepherd_herd/herd.py | 22 ++++++++++++++---- 9 files changed, 77 insertions(+), 18 deletions(-) diff --git a/software/python-package/shepherd_sheep/h5_writer.py b/software/python-package/shepherd_sheep/h5_writer.py index c9f88468..26678191 100644 --- a/software/python-package/shepherd_sheep/h5_writer.py +++ b/software/python-package/shepherd_sheep/h5_writer.py @@ -247,6 +247,7 @@ def start_monitors( sys: SystemLogging | None = None, gpio: GpioTracing | None = None, ) -> None: + self.monitors.append(SheepMonitor(self.sheep_grp, self._compression)) if sys is not None and sys.dmesg: self.monitors.append(KernelMonitor(self.kernel_grp, self._compression)) if sys is not None and sys.ptp: @@ -261,4 +262,3 @@ def start_monitors( baudrate=gpio.uart_baudrate, ), ) - self.monitors.append(SheepMonitor(self.sheep_grp, self._compression)) diff --git a/software/python-package/shepherd_sheep/monitor_kernel.py b/software/python-package/shepherd_sheep/monitor_kernel.py index d9ac5a2c..34d584ea 100644 --- a/software/python-package/shepherd_sheep/monitor_kernel.py +++ b/software/python-package/shepherd_sheep/monitor_kernel.py @@ -59,7 +59,12 @@ def __exit__( ) -> None: self.event.set() if self.thread is not None: - self.thread.join(timeout=self.poll_intervall) + self.thread.join(timeout=2 * self.poll_intervall) + if self.thread.is_alive(): + log.error( + "[%s] thread failed to end itself - will delete that instance", + type(self).__name__, + ) self.thread = None self.process.terminate() self.data["message"].resize((self.position,)) diff --git a/software/python-package/shepherd_sheep/monitor_ptp.py b/software/python-package/shepherd_sheep/monitor_ptp.py index 022a2ae5..7d249e5f 100644 --- a/software/python-package/shepherd_sheep/monitor_ptp.py +++ b/software/python-package/shepherd_sheep/monitor_ptp.py @@ -60,7 +60,12 @@ def __exit__( ) -> None: self.event.set() if self.thread is not None: - self.thread.join(timeout=self.poll_intervall) + self.thread.join(timeout=2 * self.poll_intervall) + if self.thread.is_alive(): + log.error( + "[%s] thread failed to end itself - will delete that instance", + type(self).__name__, + ) self.thread = None self.process.terminate() self.data["values"].resize((self.position, 3)) diff --git a/software/python-package/shepherd_sheep/monitor_sheep.py b/software/python-package/shepherd_sheep/monitor_sheep.py index a30cbcd6..b2e3e993 100644 --- a/software/python-package/shepherd_sheep/monitor_sheep.py +++ b/software/python-package/shepherd_sheep/monitor_sheep.py @@ -48,7 +48,12 @@ def __exit__( ) -> None: self.event.set() if self.thread is not None: - self.thread.join(timeout=self.poll_intervall) + self.thread.join(timeout=2 * self.poll_intervall) + if self.thread.is_alive(): + log.error( + "[%s] thread failed to end itself - will delete that instance", + type(self).__name__, + ) self.thread = None self.data["message"].resize((self.position,)) self.data["level"].resize((self.position,)) diff --git a/software/python-package/shepherd_sheep/monitor_sysutil.py b/software/python-package/shepherd_sheep/monitor_sysutil.py index 5f7b6916..23d940e9 100644 --- a/software/python-package/shepherd_sheep/monitor_sysutil.py +++ b/software/python-package/shepherd_sheep/monitor_sysutil.py @@ -80,7 +80,12 @@ def __exit__( ) -> None: self.event.set() if self.thread is not None: - self.thread.join(timeout=self.poll_intervall) + self.thread.join(timeout=2 * self.poll_intervall) + if self.thread.is_alive(): + log.error( + "[%s] thread failed to end itself - will delete that instance", + type(self).__name__, + ) self.thread = None self.data["cpu"].resize((self.position,)) self.data["ram"].resize((self.position, 2)) diff --git a/software/python-package/shepherd_sheep/monitor_uart.py b/software/python-package/shepherd_sheep/monitor_uart.py index f2db7b47..5327749c 100644 --- a/software/python-package/shepherd_sheep/monitor_uart.py +++ b/software/python-package/shepherd_sheep/monitor_uart.py @@ -60,7 +60,12 @@ def __exit__( ) -> None: self.event.set() if self.thread is not None: - self.thread.join(timeout=self.poll_intervall) + self.thread.join(timeout=2 * self.poll_intervall) + if self.thread.is_alive(): + log.error( + "[%s] thread failed to end itself - will delete that instance", + type(self).__name__, + ) self.thread = None self.data["message"].resize((self.position,)) super().__exit__() diff --git a/software/python-package/shepherd_sheep/shepherd_emulator.py b/software/python-package/shepherd_sheep/shepherd_emulator.py index de4255de..10ab0385 100644 --- a/software/python-package/shepherd_sheep/shepherd_emulator.py +++ b/software/python-package/shepherd_sheep/shepherd_emulator.py @@ -245,7 +245,14 @@ def run(self) -> None: break if self.writer is not None: - self.writer.write_buffer(emu_buf) + try: + self.writer.write_buffer(emu_buf) + except OSError as _xpt: + log.error( + "Failed to write data to HDF5-File - will STOP! error = %s", + _xpt, + ) + return hrvst_buf = DataBuffer(voltage=dsv, current=dsc) self.return_buffer(idx, hrvst_buf, self.verbose_extra) @@ -255,20 +262,26 @@ def run(self) -> None: try: idx, emu_buf = self.get_buffer(verbose=self.verbose_extra) if emu_buf.timestamp_ns / 1e9 >= ts_end: - break + return if self.writer is not None: self.writer.write_buffer(emu_buf) - except ShepherdIOError as e: + except ShepherdIOError as e: # noqa: PERF203 # We're done when the PRU has processed all emulation data buffers if e.id_num == commons.MSG_DEP_ERR_NOFREEBUF: log.debug("Collected all Buffers from PRU -> begin to exit now") - break + return if e.id_num == ShepherdIOError.ID_TIMEOUT: log.error("Reception from PRU had a timeout -> begin to exit now") - break + return if self.cfg.abort_on_error: raise RuntimeError( "Caught unforgivable ShepherdIO-Exception", ) from e log.warning("Caught an Exception", exc_info=e) + except OSError as _xpt: + log.error( + "Failed to write data to HDF5-File - will STOP! error = %s", + _xpt, + ) + return diff --git a/software/python-package/shepherd_sheep/shepherd_harvester.py b/software/python-package/shepherd_sheep/shepherd_harvester.py index 5cb957b0..d677f548 100644 --- a/software/python-package/shepherd_sheep/shepherd_harvester.py +++ b/software/python-package/shepherd_sheep/shepherd_harvester.py @@ -156,7 +156,7 @@ def run(self) -> None: except ShepherdIOError as e: if e.id_num == ShepherdIOError.ID_TIMEOUT: log.error("Reception from PRU had a timeout -> begin to exit now") - break + return if self.cfg.abort_on_error: raise RuntimeError( "Caught unforgivable ShepherdIO-Exception", @@ -165,7 +165,14 @@ def run(self) -> None: continue if (hrv_buf.timestamp_ns / 1e9) >= ts_end: - break + return - self.writer.write_buffer(hrv_buf) + try: + self.writer.write_buffer(hrv_buf) + except OSError as _xpt: + log.error( + "Failed to write data to HDF5-File - will STOP! error = %s", + _xpt, + ) + return self.return_buffer(idx, verbose=self.verbose_extra) diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 406b1986..2a2052c0 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -191,7 +191,11 @@ def _open(self) -> None: threads[i] = threading.Thread(target=self._thread_open, args=[cnx]) threads[i].start() for thread in threads.values(): - thread.join() + thread.join(timeout=10.0) + if thread.is_alive(): + logger.error( + "Connection.Open() did fail to finish - will delete that thread" + ) del thread # ... overcautious self.group = [cnx for cnx in self.group if cnx.is_connected] @@ -233,7 +237,11 @@ def run_cmd(self, cmd: str, *, sudo: bool = False) -> dict[int, Result]: ) threads[i].start() for thread in threads.values(): - thread.join() + thread.join(timeout=10.0) + if thread.is_alive(): + logger.error( + "Command.Run() did fail to finish - will delete that thread" + ) del thread # ... overcautious if len(results) < 1: raise RuntimeError("ZERO nodes answered - check your config") @@ -330,7 +338,9 @@ def put_file( ) threads[i].start() for thread in threads.values(): - thread.join() + thread.join(timeout=10.0) + if thread.is_alive(): + logger.error("File.Put() did fail to finish - will delete that thread") del thread # ... overcautious @staticmethod @@ -420,7 +430,11 @@ def get_file( hostname = self.hostnames[cnx.host] if replies[i].exited > 0: continue - threads[i].join() + threads[i].join(timeout=10.0) + if threads[i].is_alive(): + logger.error( + "Command.Run() did fail to finish - will delete that thread" + ) del threads[i] # ... overcautious if delete_src: logger.info( From d4d642813410dab98f70a1f483e5c889528e6cec Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sat, 21 Oct 2023 18:10:44 +0200 Subject: [PATCH 31/35] Update herd.py --- software/shepherd-herd/shepherd_herd/herd.py | 45 ++++++++++++-------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/software/shepherd-herd/shepherd_herd/herd.py b/software/shepherd-herd/shepherd_herd/herd.py index 2a2052c0..01d73d38 100644 --- a/software/shepherd-herd/shepherd_herd/herd.py +++ b/software/shepherd-herd/shepherd_herd/herd.py @@ -187,14 +187,16 @@ def _thread_open( def _open(self) -> None: """Open Connection on all Nodes""" threads = {} - for i, cnx in enumerate(self.group): - threads[i] = threading.Thread(target=self._thread_open, args=[cnx]) - threads[i].start() - for thread in threads.values(): + for cnx in self.group: + _name = self.hostnames[cnx.host] + threads[_name] = threading.Thread(target=self._thread_open, args=[cnx]) + threads[_name].start() + for host, thread in threads.items(): thread.join(timeout=10.0) if thread.is_alive(): logger.error( - "Connection.Open() did fail to finish - will delete that thread" + "Connection.Open() did fail to finish on %s - will delete that thread", + host, ) del thread # ... overcautious self.group = [cnx for cnx in self.group if cnx.is_connected] @@ -231,16 +233,18 @@ def run_cmd(self, cmd: str, *, sudo: bool = False) -> dict[int, Result]: threads = {} logger.debug("Sheep-CMD = %s", cmd) for i, cnx in enumerate(self.group): - threads[i] = threading.Thread( + _name = self.hostnames[cnx.host] + threads[_name] = threading.Thread( target=self._thread_run, args=(cnx, sudo, cmd, results, i), ) - threads[i].start() - for thread in threads.values(): - thread.join(timeout=10.0) + threads[_name].start() + for host, thread in threads.items(): + thread.join() # timeout=10.0 if thread.is_alive(): logger.error( - "Command.Run() did fail to finish - will delete that thread" + "Command.Run() did fail to finish on %s - will delete that thread", + host, ) del thread # ... overcautious if len(results) < 1: @@ -331,16 +335,20 @@ def put_file( raise NameError(f"provided path was forbidden ('{dst_path}')") threads = {} - for i, cnx in enumerate(self.group): - threads[i] = threading.Thread( + for cnx in self.group: + _name = self.hostnames[cnx.host] + threads[_name] = threading.Thread( target=self._thread_put, args=(cnx, src_path, dst_path, force_overwrite), ) - threads[i].start() - for thread in threads.values(): - thread.join(timeout=10.0) + threads[_name].start() + for host, thread in threads.items(): + thread.join() # timeout=10.0 if thread.is_alive(): - logger.error("File.Put() did fail to finish - will delete that thread") + logger.error( + "File.Put() did fail to finish on %s - will delete that thread", + host, + ) del thread # ... overcautious @staticmethod @@ -430,10 +438,11 @@ def get_file( hostname = self.hostnames[cnx.host] if replies[i].exited > 0: continue - threads[i].join(timeout=10.0) + threads[i].join() # timeout=10.0 if threads[i].is_alive(): logger.error( - "Command.Run() did fail to finish - will delete that thread" + "Command.Run() did fail to finish on %s - will delete that thread", + hostname, ) del threads[i] # ... overcautious if delete_src: From 380722f1366ea27a17ff5a5a54f9769e07a762bb Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sun, 22 Oct 2023 11:46:43 +0200 Subject: [PATCH 32/35] Update Pipfile.lock --- Pipfile.lock | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 1d09ba8b..22e2d0c0 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -2027,21 +2027,20 @@ "path": "./software/shepherd-calibration" }, "shepherd-core": { - "extras": [], "hashes": [ - "sha256:1cb27b408e56577e3625b6a4605bfef4f54a64c533b1df7aaead1677e4173e61", - "sha256:f6939ad4fdf18f29f15034ec0fa2231486d315b0c57e61ad26d2edd0504f26b4" + "sha256:9c8ff4f740c5e611b487ff8f3e785121b6d5ab599b7092ba7c1871ebab960e8c", + "sha256:a33cc8243434d5b0bab50e7f88001ce7e780c1dcc510e947d675fb6fd0192585" ], "markers": "python_version >= '3.8'", - "version": "==2023.10.1" + "version": "==2023.10.3" }, "shepherd-data": { "hashes": [ - "sha256:28567d89412a5beef79a5a16f48df5275f858a20229aeab697e2c1c28b8f4f0d", - "sha256:52a38ba7f1531d89bb10c77bc53632b6719f80228818731bd830323ff00774d3" + "sha256:0d88400b76a890f5a09fa4b7bd0937acdf87cc13cfe3bd089307568188d6b267", + "sha256:86e3263da2b787c7f6f2ceb8df33c59ca644f2816dd861264227e9f25ffcb69b" ], "markers": "python_version >= '3.8'", - "version": "==2023.10.1" + "version": "==2023.10.3" }, "shepherd-herd": { "editable": true, From 49eb498f2e89e57e66ce7ccd84830b523309723b Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sun, 22 Oct 2023 11:58:13 +0200 Subject: [PATCH 33/35] update submodules --- .gitmodules | 7 ++++--- software/shepherd-datalib | 2 +- software/shepherd-targets | 1 + software/shepherd-webservice | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) create mode 160000 software/shepherd-targets diff --git a/.gitmodules b/.gitmodules index 97359710..a52b2b83 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,17 @@ -[submodule "datalib"] +[submodule "software/shepherd-datalib"] path = software/shepherd-datalib url = https://github.com/orgua/shepherd-datalib.git -[submodule "webservice"] +[submodule "software/shepherd-webservice"] path = software/shepherd-webservice url = https://github.com/orgua/shepherd_webservice.git -[submodule "targets"] +[submodule "software/shepherd-targets"] path = software/shepherd-targets url = https://github.com/orgua/shepherd-targets.git # helpful commands: +# git submodule add # git submodule update --init --recursive # git submodule update --recursive --remote # git submodule foreach git reset --hard diff --git a/software/shepherd-datalib b/software/shepherd-datalib index 3ce65392..359c8a85 160000 --- a/software/shepherd-datalib +++ b/software/shepherd-datalib @@ -1 +1 @@ -Subproject commit 3ce65392552a1b1daed70acdce8a8f65ac90be20 +Subproject commit 359c8a854a39beace9a4d6026a5b3a7ae2852a25 diff --git a/software/shepherd-targets b/software/shepherd-targets new file mode 160000 index 00000000..36429c8e --- /dev/null +++ b/software/shepherd-targets @@ -0,0 +1 @@ +Subproject commit 36429c8e476356550d22e5304f7f2f49facc8ea4 diff --git a/software/shepherd-webservice b/software/shepherd-webservice index ce4e7a94..5f33243a 160000 --- a/software/shepherd-webservice +++ b/software/shepherd-webservice @@ -1 +1 @@ -Subproject commit ce4e7a943fd673c51d7fe4a8a6d90a2d46e8b149 +Subproject commit 5f33243a1dcbeb2fcc4911990a7320380e53e9d9 From 35b0e081246073ce7a12e1c60fb4368b677771e4 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Sun, 22 Oct 2023 12:29:53 +0200 Subject: [PATCH 34/35] remove flake8-commas --- .pre-commit-config.yaml | 1 - pyproject.toml | 3 --- 2 files changed, 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 55a3c79e..8d533842 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -80,7 +80,6 @@ repos: - flake8-comprehensions - flake8-simplify - flake8-eradicate - - flake8-commas ### Limitations - flake8-blind-except diff --git a/pyproject.toml b/pyproject.toml index add1bc2c..9fc7b6e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,6 @@ require-plugins = [ "flake8-comprehensions", "flake8-simplify", "flake8-eradicate", - "flake8-commas", ### Limitations "flake8-blind-except", @@ -50,8 +49,6 @@ extend-ignore = [ "E203", # open() vs os.open() "SCS109", - # hideous commas - "C812", "C815", "C813", # DOCString TODO: reduce here "D100", "D102", "D103", # handled by ruff From 8151f0254b51980d3d7ee1252d88a4c0fa3d63d4 Mon Sep 17 00:00:00 2001 From: Ingmar Splitt Date: Mon, 23 Oct 2023 11:06:18 +0200 Subject: [PATCH 35/35] Update shepherd_io.py --- software/python-package/shepherd_sheep/shepherd_io.py | 1 - 1 file changed, 1 deletion(-) diff --git a/software/python-package/shepherd_sheep/shepherd_io.py b/software/python-package/shepherd_sheep/shepherd_io.py index e4a7c132..4b90b842 100644 --- a/software/python-package/shepherd_sheep/shepherd_io.py +++ b/software/python-package/shepherd_sheep/shepherd_io.py @@ -114,7 +114,6 @@ def __init__( self.shared_mem: SharedMemory def __del__(self) -> None: - log.debug("Now deleting ShepherdIO-Instance") ShepherdIO._instance = None def __enter__(self) -> Self: