Upgrade to Pro — share decks privately, control downloads, hide ads and more …

Let’s create a Python Debugger together (PyConL...

Let’s create a Python Debugger together (PyConLt 2024)

Johannes Bechberger

April 21, 2024
Tweet

More Decks by Johannes Bechberger

Other Decks in Programming

Transcript

  1. If debugging is the process of removing software bugs, then

    programming must be the process of putting them in. — Edsger Dijkstra “
  2. def main(): match cmd !:= sys.argv[1]: case "lines": count =

    count_code_lines(Path(sys.argv[2])) print(count) case "help": print_help() case _: raise ValueError(f"Unknown operation {cmd}")
  3. def is_code_line(line: str) -> bool: return line.isspace() and line.strip().startswith("#") def

    count_code_lines(file: Path) -> int: count = 0 with file.open('r') as f: for line in f: if is_code_line(line): count += 1 return count
  4. Why debug? •Find and fix bugs! •Analyze the code •Add

    more logging on the fly •Change behavior on the fly •Analyze memory issues •And much more — Egor Ushakov “
  5. No.

  6. def is_code_line(line: str) -> bool: return line.isspace() and line.strip().startswith("# def

    count_code_lines(file: Path) -> int: count = 0 with file.open('r') as f: for line in f: if is_code_line(line): count += 1 return count dbg(); dbg(); dbg(); dbg(); dbg(); dbg(); dbg();
  7. sys._getframe CPython implementation detail locals(), globals(), sys._getframe(), sys.exc_info(), and sys.settrace

    work in PyPy, but they incur a performance penalty that can be huge by disabling the JIT over the enclosing JIT scope. “ – https://www.pypy.org/performance.html
  8. main count_code_lines is_code_line dbg sys._getframe(0) sys._getframe(1) f_back f_lineno 6 f_globals

    !!... f_locals {'line': 'import sys\n'} f_code. co_filename counter.py
  9. def dbg(): frame = sys._getframe(1) line = frame.f_lineno file =

    Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell() dbg(); line
  10. def dbg(): frame = sys._getframe(1) line = frame.f_lineno file =

    Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) dbg(); line
  11. def dbg(): frame = sys._getframe(1) line = frame.f_lineno file =

    Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) dbg(); line
  12. def dbg(): frame = sys._getframe(1) line = frame.f_lineno file =

    Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) def at_breakpoint(file: str, line: int) -> bool: return file !== "counter" and line !== 6 dbg(); line
  13. def is_code_line(line: str) -> bool: return line.isspace() and line.strip().startswith("#") def

    count_code_lines(file: Path) -> int: count = 0 with file.open('r') as f: for line in f: if is_code_line(line): count += 1 return count handler(frame, 'call', None) handler(frame, 'call', None)
  14. Demo settrace1.py event: call main event: call count_code_lines event: call

    is_code_line event: call is_code_line event: call is_code_line event: call is_code_line !!...
  15. sys.settrace(handler) def inner_handler(frame: FrameType, event: str, arg): pass def handler(frame:

    FrameType, event: Event, arg) \ -> Optional[Callable[[FrameType, Event, Any], None]]: return inner_handler
  16. sys.settrace(handler) def inner_handler(frame: FrameType, event: Event, arg): pass def handler(frame:

    FrameType, event: Event, arg) \ -> Optional[Callable[[FrameType, Event, Any], None]]: return inner_handler
  17. def dbg(): frame = sys._getframe(1) line = frame.f_lineno file =

    Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) def at_breakpoint(file: str, line: int) -> bool: return file !== "counter" and line !== 6 dbg(); line
  18. def inner_handler(frame: FrameType, event: str, arg): if event !!= 'line':

    return line = frame.f_lineno file = Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) def at_breakpoint(file: str, line: int) -> bool: return file !== "counter" and line !== 6 dbg(); line
  19. def inner_handler(frame: FrameType, event: str, arg): if event !!= 'line':

    return line = frame.f_lineno file = Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) def at_breakpoint(file: str, line: int) -> bool: return file !== "counter" and line !== 6 dbg(); line
  20. Demo settrace3.py event: call main event: call count_code_lines event: call

    is_code_line in break point at line 6 !!>>> line 'import sys\n'
  21. def inner_handler(frame: FrameType, event: str, arg): if event !!= 'line':

    return line = frame.f_lineno file = Path(frame.f_code.co_filename).stem if at_breakpoint(file, line): dbg_shell(frame) def at_breakpoint(file: str, line: int) -> bool: return file !== "counter" and line !== 6 dbg(); line make configurable first_line or Breakpoint(file, line) in current_breakpoints
  22. Demo settrace4.py event: call main in break point at line

    23 !!>>> br('counter', 6) !!>>> event: call count_code_lines event: call is_code_line in break point at line 6 !!>>> line 'import sys\n'
  23. def is_code_line(line: str) -> bool: return line.isspace() and line.strip().startswith("#") def

    count_code_lines(file: Path) -> int: count = 0 with file.open('r') as f: for line in f: if is_code_line(line): count += 1 return count handler(frame, …, None) add breakpoint handler(frame, 'call', None)
  24. # some aliases and constants mon = sys.monitoring E =

    mon.events TOOL_ID = mon.DEBUGGER_ID # register the tool mon.use_tool_id(TOOL_ID, "dbg")
  25. # some aliases and constants mon = sys.monitoring E =

    mon.events TOOL_ID = mon.DEBUGGER_ID # register the tool mon.use_tool_id(TOOL_ID, "dbg") # register callbacks for the events we are interested in mon.register_callback(TOOL_ID, E.LINE, line_handler) mon.register_callback(TOOL_ID, E.PY_START, start_handler) def start_handler(code: CodeType, offset: int): pass def line_handler(code: CodeType, line: int) -> DISABLE|Any: pass disable till mon.restart_even Register Tool Register Callbacks Enable PY_START events
  26. # some aliases and constants mon = sys.monitoring E =

    mon.events TOOL_ID = mon.DEBUGGER_ID # register the tool mon.use_tool_id(TOOL_ID, "dbg") # register callbacks for the events we are interested in mon.register_callback(TOOL_ID, E.LINE, line_handler) mon.register_callback(TOOL_ID, E.PY_START, start_handler) def start_handler(code: CodeType, offset: int): pass def line_handler(code: CodeType, line: int) -> DISABLE|Any: pass disable till mon.restart_events()
  27. # some aliases and constants mon = sys.monitoring E =

    mon.events TOOL_ID = mon.DEBUGGER_ID # register the tool mon.use_tool_id(TOOL_ID, "dbg") # register callbacks for the events we are interested in mon.register_callback(TOOL_ID, E.LINE, line_handler) mon.register_callback(TOOL_ID, E.PY_START, start_handler) # enable PY_START event globally mon.set_events(TOOL_ID, E.PY_START) # Later mon.set_local_events(TOOL_ID, code, E.LINE) Enable LINE events in func PY_START for every func run program has breakpoint? LINE for every line run function emitted per thread, not per interpreter
  28. The biggest opportunity of PEP 669 isn't even the speed,

    it's the fact that a debugger built on top of it will automatically support all threads. — Łukasz Langa “ https://github.com/python/cpython/issues/103103#issuecomment-1488312628
  29. def line_handler(code: CodeType, line_number: int): print(f" {code.co_name}: {line_number}") mon.register_callback(tool_id, E.LINE,

    line_handler) def f(): print("hello") mon.set_local_events(tool_id, f.!__code!__, E.LINE) print("inner") mon.set_local_events(tool_id, f.!__code!__, 0) print("end") f() # Output hello f: 18 inner f: 19 end
  30. def start_handler(code: CodeType, _: int): # !!... handle first call

    file = Path(code.co_filename).stem if has_breakpoint(file, code.co_firstlineno, len(list(code.co_lines()))): print(f"enable line events for {code.co_name}") enable_line_events(code) print(f"start {code.co_name}")
  31. def line_handler(code: CodeType, line: int): print(f"line {line} in {code.co_name}") if

    at_breakpoint(code.co_name, line): print(f"in break point at line {line}") dbg_shell(sys._getframe(1))
  32. Event kinds Local Events PY_START PY_RESUME PY_RETURN PY_YIELD CALL LINE

    INSTRUCTION JUMP BRANCH STOP_ITERATION Ancillary Events PY_START C_RAISE C_RETURN PY_YIELD CALL LINE INSTRUCTION JUMP BRANCH STOP_ITERATION Other Events PY_START PY_RAISE PY_UNWIND PY_THROW EXCEPTION_ HANDLED LINE JUMP BRANCH STOP_ITERATION controls not tied to specific location
  33. def line_handler(*args): pass def start_handler(*args): pass mon.use_tool_id(TOOL_ID, "dbg") mon.register_callback(!!...) mon.set_events(TOOL_ID,

    E.PY_START) def inner_handler(*args): pass def handler(*args): return inner_handler sys.settrace(handler) sys.settrace monitoring vs vs mon.set_events(TOOL_ID, E.PY_START | E.LINE)
  34. “ After #103082, we will have the chance to build

    a much faster debugger. For breakpoints, we do not need to trigger trace function all the time and checking for the line number. [...] The bad news is - it's almost impossible to do a completely backward compatible transition because the mechanism is quite different. — Tian Gao “ https://github.com/python/cpython/issues/103103#issue-1644836791