@cryptotaxi247 / infra-1 / commits / 8e5be867

add more ruff linter

Jörg Thalheim committed Nov 2, 2024 at 10:49 UTC 8e5be8674f19dd0d38d25ef1cb64b24582e237c3

6 files changed +180 -86

build/datadog/hydra.py

+5 -4

@@ -1,11 +1,12 @@

1 -#! /usr/bin/env python

2 -import checks

3 -import requests

1 import json

3 +import requests

4 +

5 +import checks

6 +

8 class HydraCheck(checks.AgentCheck):

8 - def check(self, instance):

9 + def check(self, instance) -> None:

10 r = requests.get(

11 "http://localhost:3000/status", headers={"Content-Type": "application/json"}

12 )

build/pluto/prometheus/exporters/channel-exporter.py

+7 -7

@@ -1,14 +1,14 @@

1 #!/usr/bin/env python3

3 -import requests

4 -from dateutil.parser import parse

5 -from prometheus_client import Counter, Histogram, Gauge, start_http_server

6 -import time

7 -import sys

3 +import json

4 import logging

5 +import sys

6 +import time

7 from pprint import pprint

10 -import json

9 +import requests

10 +from dateutil.parser import parse

11 +from prometheus_client import Counter, Gauge, Histogram, start_http_server

13 CHANNEL_REVISION = Gauge(

14 "channel_revision",

@@ -87,7 +87,7 @@ if __name__ == "__main__":

87 current=current,

88 ).set(1)

89 CHANNEL_CURRENT.labels(channel=channel).set(current)

90 - print("updated {}".format(channel))

90 + print(f"updated {channel}")

91 previous_revision = revisions.pop(channel, None)

92 revisions[channel] = revision

93 if previous_revision and previous_revision != revision:

build/pluto/prometheus/exporters/hydra-queue-runner-reexporter.py

+23 -30

@@ -1,20 +1,22 @@

1 #!/usr/bin/env nix-shell

2 #!nix-shell -i python3 -p python3 -p python3Packages.requests -p python3Packages.prometheus_client

4 -import requests

4 +import contextlib

5 import json

6 -from prometheus_client.core import GaugeMetricFamily, CounterMetricFamily

7 -from prometheus_client import CollectorRegistry, start_http_server

6 import time

8 +import requests

9 +from prometheus_client import CollectorRegistry, start_http_server

10 +from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily

11 +

11 -def debug_remaining_state(edict):

13 +def debug_remaining_state(edict) -> None:

14 # pprint(edict.remaining_state())

15 pass

18 class EvaporatingDict:

17 - def __init__(self, state):

19 + def __init__(self, state) -> None:

20 self._state = state

22 def preserving_read(self, key):

@@ -22,13 +24,11 @@ class EvaporatingDict:

25 if isinstance(val, dict):

26 return EvaporatingDict(val)

25 - else:

26 - return val

27 + return val

29 def preserving_read_default(self, key, default):

30 try:

30 - val = self.preserving_read(key)

31 - return val

31 + return self.preserving_read(key)

32 except KeyError:

33 return default

@@ -47,7 +47,7 @@ class EvaporatingDict:

47 # todo: log bad reads?

48 return default

50 - def unused_read(self, key):

50 + def unused_read(self, key) -> None:

51 self.destructive_read_default(key, default=None)

53 def remaining_state(self):

@@ -60,7 +60,7 @@ class EvaporatingDict:

62 class HydraScrapeImporter:

63 - def __init__(self, status):

63 + def __init__(self, status) -> None:

64 self._status = EvaporatingDict(status)

66 def collect(self):

@@ -328,7 +328,6 @@ class HydraScrapeImporter:

328 except KeyError:

329 # no key, no metrics

330 s3 = None

331 - pass

331 if s3:

332 # Not in the above try to avoid the try catching mistakes

333 # in the following code

@@ -382,7 +381,7 @@ class HydraScrapeImporter:

381 c.add_metric([], value)

382 return c

383

385 - def unused_metric(self, key):

384 + def unused_metric(self, key) -> None:

385 self._status.unused_read(key)

386

387 def preserving_read(self, key):

@@ -401,12 +400,12 @@ class HydraScrapeImporter:

400 return self._status.remaining_state()

401

402

404 -def blackhole(*args, **kwargs):

403 +def blackhole(*args, **kwargs) -> None:

404 return None

405

406

407 class MachineScrapeImporter:

409 - def __init__(self):

408 + def __init__(self) -> None:

409 labels = ["host"]

410 self.consective_failures = GaugeMetricFamily(

411 "hydra_machine_consecutive_failures",

@@ -450,7 +449,7 @@ class MachineScrapeImporter:

449 labels=labels,

450 )

451

453 - def load_machine(self, name, report):

452 + def load_machine(self, name, report) -> None:

453 report.unused_read("mandatoryFeatures")

454 report.unused_read("supportedFeatures")

455 report.unused_read("systemTypes")

@@ -461,10 +460,8 @@ class MachineScrapeImporter:

460 labels, report.destructive_read("consecutiveFailures")

461 )

462 self.current_jobs.add_metric(labels, report.destructive_read("currentJobs"))

464 - try:

463 + with contextlib.suppress(KeyError):

464 self.idle_since.add_metric(labels, report.destructive_read("idleSince"))

466 - except KeyError:

467 - pass

465 self.disabled_until.add_metric(labels, report.destructive_read("disabledUntil"))

466 self.enabled.add_metric(labels, 1 if report.destructive_read("enabled") else 0)

467 self.last_failure.add_metric(labels, report.destructive_read("lastFailure"))

@@ -492,7 +489,7 @@ class MachineScrapeImporter:

489

490

491 class JobsetScrapeImporter:

495 - def __init__(self):

492 + def __init__(self) -> None:

493 self.seconds = CounterMetricFamily(

494 "hydra_jobset_seconds_total",

495 "Total number of seconds the jobset has been building",

@@ -504,7 +501,7 @@ class JobsetScrapeImporter:

501 labels=["name"],

502 )

503

507 - def load_jobset(self, name, report):

504 + def load_jobset(self, name, report) -> None:

505 self.seconds.add_metric([name], report.destructive_read("seconds"))

506 self.shares_used.add_metric([name], report.destructive_read("shareUsed"))

507 debug_remaining_state(report)

@@ -515,7 +512,7 @@ class JobsetScrapeImporter:

512

513

514 class MachineTypeScrapeImporter:

518 - def __init__(self):

515 + def __init__(self) -> None:

516 self.runnable = GaugeMetricFamily(

517 "hydra_machine_type_runnable",

518 "Number of currently runnable builds",

@@ -537,17 +534,13 @@ class MachineTypeScrapeImporter:

534 labels=["machineType"],

535 )

536

540 - def load_machine_type(self, name, report):

537 + def load_machine_type(self, name, report) -> None:

538 self.runnable.add_metric([name], report.destructive_read("runnable"))

539 self.running.add_metric([name], report.destructive_read("running"))

543 - try:

540 + with contextlib.suppress(KeyError):

541 self.wait_time.add_metric([name], report.destructive_read("waitTime"))

545 - except KeyError:

546 - pass

547 - try:

542 + with contextlib.suppress(KeyError):

543 self.last_active.add_metric([name], report.destructive_read("lastActive"))

549 - except KeyError:

550 - pass

544

545 debug_remaining_state(report)

546

@@ -559,7 +552,7 @@ class MachineTypeScrapeImporter:

552

553

554 class ScrapeCollector:

562 - def __init__(self):

555 + def __init__(self) -> None:

556 pass

557

558 def collect(self):

hydra-packet-importer/import.py

+27 -31

@@ -1,22 +1,22 @@

1 #!/usr/bin/env python3

3 -import json

4 -import packet # type: ignore

3 import base64

4 +import json

5 import sys

7 -from typing import Dict, Any, List, Optional

8 -from typing import TypedDict

6 +from typing import Any, TypedDict

7 +

8 +import packet

10 -DeviceKeys = List[Dict[str, Any]]

10 +DeviceKeys = list[dict[str, Any]]

13 class Metadata(TypedDict):

14 - user: Optional[str]

15 - features: List[str]

16 - mandatory_features: List[str]

14 + user: str | None

15 + features: list[str]

16 + mandatory_features: list[str]

17 max_jobs: int

18 - system_types: List[str]

19 - speed_factor: Optional[int]

18 + system_types: list[str]

19 + speed_factor: int | None

22 class RemoteBuilder(TypedDict):

@@ -50,25 +50,22 @@ class Device(TypedDict):

52 class ProjectDeviceList(TypedDict):

53 - meta: Dict[str, Any]

54 - devices: List[Device]

53 + meta: dict[str, Any]

54 + devices: list[Device]

57 def debug(*args: Any, **kwargs: Any) -> None:

58 print(*args, file=sys.stderr, **kwargs)

61 -def get_builders(manager: Any) -> List[Builder]:

62 - builders: List[Builder] = []

61 +def get_builders(manager: Any) -> list[Builder]:

62 + builders: list[Builder] = []

64 - page: Optional[str] = "projects/{}/devices?page={}".format(config["project_id"], 1)

64 + page: str | None = "projects/{}/devices?page={}".format(config["project_id"], 1)

65 while page is not None:

66 debug(page)

67 data: ProjectDeviceList = manager.call_api(page)

68 - if data["meta"]["next"] is None:

69 - page = None

70 - else:

71 - page = data["meta"]["next"]["href"]

68 + page = None if data["meta"]["next"] is None else data["meta"]["next"]["href"]

70 for device in data["devices"]:

71 if device["state"] != "active":

@@ -95,19 +92,19 @@ def get_builders(manager: Any) -> List[Builder]:

92 return builders

98 -def get_remote_builder_info(manager, device_id: str) -> Optional[RemoteBuilder]:

95 +def get_remote_builder_info(manager, device_id: str) -> RemoteBuilder | None:

96 # ... 50 is probably enough.

97 try:

101 - events_url = "devices/{}/events?per_page=50".format(device_id)

98 + events_url = f"devices/{device_id}/events?per_page=50"

99 debug(events_url)

100 data = manager.call_api(events_url)

101 except Exception:

102 # 404 probably

103 return None

104

108 - host_key: Optional[HostKey] = None

109 - ssh_key: Optional[str] = None

110 - metadata: Optional[Metadata] = None

105 + host_key: HostKey | None = None

106 + ssh_key: str | None = None

107 + metadata: Metadata | None = None

108 for event in data["events"]:

109 if event["type"] == "provisioning.104.01":

110 # we reached a "Device connected to DHCP system" event,

@@ -126,7 +123,7 @@ def get_remote_builder_info(manager, device_id: str) -> Optional[RemoteBuilder]:

123 return None

124 if event["type"] == "user.1001":

125 try:

129 - host_keys: List[HostKey] = [

126 + host_keys: list[HostKey] = [

127 key for key in json.loads(event["body"]) if key["port"] == 22

128 ]

129 host_key = host_keys[0]

@@ -138,16 +135,15 @@ def get_remote_builder_info(manager, device_id: str) -> Optional[RemoteBuilder]:

135 return None

136

137

141 -def strip_ssh_key_comment(key: str) -> Optional[str]:

138 +def strip_ssh_key_comment(key: str) -> str | None:

139 ssh_key_parts = key.rsplit(" ", 1)

140 if len(ssh_key_parts) == 2:

141 return ssh_key_parts[0]

145 - else:

146 - debug("# Skipped due keyscan failed to split on ' '")

147 - return None

142 + debug("# Skipped due keyscan failed to split on ' '")

143 + return None

144

145

150 -def main(config: Dict[str, Any]) -> None:

146 +def main(config: dict[str, Any]) -> None:

147 rows = []

148 manager = packet.Manager(auth_token=config["token"])

149 found = 0

@@ -176,7 +172,7 @@ def main(config: Dict[str, Any]) -> None:

172 )

173 )

174

179 - debug("# {} / {}".format(len(rows), found))

175 + debug(f"# {len(rows)} / {found}")

176 print("\n".join(rows))

177

178

modules/prometheus/nixos-exporter/prometheus_nixos_exporter/__main__.py

+18 -14

@@ -2,34 +2,37 @@

2 #!nix-shell -i python3 -p "python3.withPackages (ps: with ps; [ prometheus-client packaging ])"

5 -import subprocess

5 import json

6 import os

7 +import subprocess

8 import sys

9 import time

10 +from collections.abc import Iterator

11 +

12 from packaging.version import Version

11 -from prometheus_client.core import GaugeMetricFamily

13 from prometheus_client import CollectorRegistry, start_http_server

14 +from prometheus_client.core import GaugeMetricFamily

17 class NixosSystemCollector:

16 - def __init__(self):

18 + def __init__(self) -> None:

19 nix_version = self.get_nix_version()

21 # https://github.com/NixOS/nix/pull/9242

22 self.nix_path_info_returns_object = nix_version >= Version("2.19.0")

22 - def get_nix_version(self):

23 - result = subprocess.run(["nix", "--version"], stdout=subprocess.PIPE)

24 + def get_nix_version(self) -> Version:

25 + result = subprocess.run(

26 + ["nix", "--version"], stdout=subprocess.PIPE, check=False

27 + )

29 if result.returncode == 0:

30 response = result.stdout.decode().strip()

31 return Version(response.split()[-1])

28 - else:

29 - print("Failed to determine nix version", file=sys.stderr)

30 - sys.exit(1)

32 + print("Failed to determine nix version", file=sys.stderr)

33 + sys.exit(1)

32 - def collect(self):

35 + def collect(self) -> Iterator[GaugeMetricFamily]:

36 # note: Gauges because of rollbacks.

37 current_system = GaugeMetricFamily(

38 "nixos_current_system_time_seconds",

@@ -64,22 +67,23 @@ class NixosSystemCollector:

67 current_system_kernel_booted.add_metric([], booted_kernel == current_kernel)

68 yield current_system_kernel_booted

67 - def get_version_id(self, path):

70 + def get_version_id(self, path: str) -> str:

71 result = subprocess.run(

72 ["bash", "-c", f"source {path}/etc/os-release; echo $VERSION_ID"],

73 stdout=subprocess.PIPE,

74 + check=False,

75 )

76 if result.returncode == 0:

77 return result.stdout.decode("utf-8").strip()

79 return None

77 - def get_kernel_out(self, path):

81 + def get_kernel_out(self, path: str) -> str:

82 return os.path.dirname(os.readlink(os.path.join(path, "kernel")))

80 - def get_time(self, path):

84 + def get_time(self, path: str) -> int:

85 result = subprocess.run(

82 - ["nix", "path-info", "--json", path], stdout=subprocess.PIPE

86 + ["nix", "path-info", "--json", path], stdout=subprocess.PIPE, check=False

87 )

88 if result.returncode == 0:

89 parsed = json.loads(result.stdout)

@@ -95,7 +99,7 @@ class NixosSystemCollector:

99 return 0

100

101

98 -def main():

102 +def main() -> None:

103 registry = CollectorRegistry()

104 registry.register(NixosSystemCollector())

105

pyproject.toml new

+100

@@ -0,0 +1,100 @@

1 +[tool.ruff]

2 +line-length = 88

3 +target-version = "py312"

4 +

5 +lint.select = ["ALL"]

6 +

7 +lint.ignore = [

8 + # pydocstyle

9 + "D",

10 + # todo comments

11 + "TD",

12 + # fixmes

13 + "FIX",

14 +

15 + # Unused function argument

16 + "ARG001",

17 +

18 + ## breaks with nix-shell

19 + # Shebang should be at the beginning of the file

20 + "EXE005",

21 + "EXE003",

22 + "EXE001",

23 +

24 + # Missing type annotation for `self` in method

25 + "ANN101",

26 + # Dynamically typed expressions (typing.Any)

27 + "ANN401",

28 + # Trailing comma missing

29 + "COM812",

30 + # Unnecessary `dict` call (rewrite as a literal)

31 + "C408",

32 + # Found commented-out code

33 + "ERA001",

34 + # Boolean-typed positional argument in function definition

35 + "FBT001",

36 + # Logging statement uses f-string

37 + "G004",

38 + # disabled on ruff's recommendation as causes problems with the formatter

39 + "ISC001",

40 + # Use of `assert` detected

41 + "S101",

42 + # `subprocess` call: check for execution of untrusted input

43 + "S603",

44 + # Starting a process with a partial executable path

45 + "S607",

46 + # Boolean default positional argument in function definition

47 + "FBT002",

48 +

49 + # Too many statements

50 + "PLR0915",

51 + # Too many arguments in function definition

52 + "PLR0913",

53 + "PLR0912", # Too many branches

54 + # $X is too complex

55 + "C901",

56 +

57 + "E501", # line too long

58 + "T201", # `print` found

59 + "T203", # `pprint` found

60 + "PLR2004", # Magic value used in comparison

61 +]

62 +

63 +# TODO fixes

64 +[tool.ruff.lint.per-file-ignores]

65 +"modules/prometheus/nixos-exporter/prometheus_nixos_exporter/__main__.py" = [

66 + "PTH115",

67 + "PTH118",

68 + "PTH120"

69 +]

70 +"hydra-packet-importer/import.py" = [

71 + "ANN001",

72 + "PTH123",

73 + "BLE001",

74 + "S110"

75 +]

76 +"build/pluto/prometheus/exporters/**.py" = [

77 + "ANN"

78 +]

79 +"build/datadog/hydra.py" = [

80 + "ANN001",

81 + "ARG002",

82 + "INP001",

83 + "S113",

84 +]

85 +"build/pluto/prometheus/exporters/channel-exporter.py" = [

86 + "BLE001",

87 + "PTH123"

88 +]

89 +"build/pluto/prometheus/exporters/hydra-queue-runner-reexporter.py" = [

90 + "TRY300",

91 + "N806",

92 + "A002",

93 + "PTH123",

94 + "S113"

95 +]

96 +

97 +[[tool.mypy.overrides]]

98 +module = "packet.*"

99 +ignore_missing_imports = true

100 +