Share
Embed
Generators Inside Out
Generators Inside Out talk given at PyCon India 2016.
Anand Chitipothu
September 24, 2016
Transcript
65 slides
-
What is this talk about? • Iterators • Generators •
Coroutines • Async • Async IO PyCon India 2016 3 -
Iterating over a list for x in [1, 2, 3,
4]: print(x) ------------------------------------------------------- 1 2 3 4 PyCon India 2016 5 -
Iterating over a string for c in "hello": print(c) -------------------------------------------------------
h e l l o PyCon India 2016 6 -
Iterating over a dictionary for k in {"x": 1, "y":
2, "z": 3}: print(k) ------------------------------------------------------- y x z PyCon India 2016 7 -
The Iteration Protocol >>> x = iter(["a", "b", "c"]) >>>
next(x) 'a' >>> next(x) 'b' >>> next(x) 'c' >>> next(x) Traceback (most recent call last): File "<stdin>", line 1, in <module> StopIteration PyCon India 2016 8 -
win! # Largest word in the dictionary >>> max(open('/usr/share/dict/words'), key=len)
'formaldehydesulphoxylate\n' PyCon India 2016 9 -
What is a generator? def squares(numbers): for n in numbers:
yield n*n ------------------------------------------------------- >>> for x in squares([1, 2, 3]): ... print(x) 1 4 9 PyCon India 2016 11 -
Let me add some prints to understand it better. def
squares(numbers): print("BEGIN squares") for n in numbers: print("Computing square of", n) yield n*n print("END squares") ------------------------------------------------------- >>> sq = squares([1, 2, 3]) >>> sq <generator object squares at 0xb6c73720> PyCon India 2016 12 -
>>> next(sq) BEGIN squares Computing square of 1 1 >>>
next(sq) Computing square of 2 4 >>> next(sq) Computing square of 3 9 >>> next(sq) END squares Traceback (most recent call last): File "<stdin>", line 1, in <module> StopIteration PyCon India 2016 13 -
>>> for x in squares([1, 2, 3]): ... print(x) BEGIN
squares Computing square of 1 1 Computing square of 2 4 Computing square of 3 9 END squares PyCon India 2016 14 -
Write a program to find fibbonacci number. def fibn(n): if
n == 1 or n == 2: return 1 else: return fibn(n-1) + fibn(n-2) ------------------------------------------------------- >>> fibn(10) 55 PyCon India 2016 16 -
Write a program to compute first n fibbonacci numbers. def
fibs(n): result = [] a, b = 1, 1 for i in range(n): result.append(a) a, b = b, a+b return result ------------------------------------------------------- >>> fibs(10) [1, 1, 2, 3, 5, 8, 13, 21, 34, 55] PyCon India 2016 17 -
What is the largest fibbonacci number below one million? def
largest_fib(upperbound): a, b = 1, 1 while b < upperbound: a, b = b, a+b return a ------------------------------------------------------- >>> largest_fib(1000000) 832040 PyCon India 2016 18 -
The generator-based solution def gen_fibs(): """Generates sequence of fibbonacci numbers.
""" a, b = 1, 1 while True: yield a a, b = b, a + b PyCon India 2016 20 -
Let's write some generic generator utilities. def first(seq): """Returns the
first element of a sequence. """ return next(iter(seq)) def last(seq): """Returns the last element of a sequence. """ for x in seq: pass return x PyCon India 2016 21 -
def take(n, seq): """Takes first n elements of a sequence.
""" seq = iter(seq) return (next(seq) for i in range(n)) def nth(n, seq): """Returns n'th element of a sequence. """ return last(take(n, seq)) PyCon India 2016 22 -
def upto(upperbound, seq): """Returns elements in the sequence until they
are less than upper bound. """ for x in seq: if x > upperbound: break yield x def count(seq): """Counts the number of elements in a sequence.""" return sum(1 for x in seq) PyCon India 2016 23 -
# what is 10th fibbonacci number? >>> nth(10, gen_fibs()) 55
# find first 10 fibbinacci numbers >>> list(take(10, gen_fibs())) [1, 1, 2, 3, 5, 8, 13, 21, 34, 55] # find all fibbonacci numbers below 100 >>> list(upto(100, gen_fibs()) [1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89] PyCon India 2016 24 -
# What is the largest fibbonacci number # below one
million? >>> last(upto(1000000, gen_fibs())) 832040 # How many fibbonacci numbes are there # below one million? >>> count(upto(1000000, gen_fibs())) 30 PyCon India 2016 25 -
import os def find(root): """Finds all the files in the
given directory tree. """ for path, dirnames, filenames in os.walk(root): for f in filenames: yield os.path.join(path, f) PyCon India 2016 27 -
def readlines(paths): """Returns a generator over lines in all the
files specified. """ for path in paths: yield from open(path) PyCon India 2016 28 -
def grep(pattern, lines): """Returns only the lines that contain given
pattern. """ return (line for line in lines if pattern in line) PyCon India 2016 29 -
def main(): # find all files in the project filenames
= find("project") # pick only python files filenames = grep('.py', filenames) # read all the lines lines = readlines(filenames) # pick only function definitions lines = grep('def ', lines) # count the total number of functions in your project print(count(lines)) PyCon India 2016 30 -
Let's look at this strange example: def display(values): for v
in values: print(v) yield def main(): g1 = display("ABC") g2 = display("123") next(g1); next(g2) next(g1); next(g2) next(g1); next(g2) PyCon India 2016 32 -
Slightly generalized. def run_all(generators): # Runs all the generators concurrently
# stop when any one of them stops try: while True: for g in generators: next(g) except StopIteration: pass def main2(): g1 = display("ABC") g2 = display("123") run_all([g1, g2]) PyCon India 2016 34 -
How about writing a function to print two sets of
values? def display2(values1, values2): # WARNING: this doesn't work display(values1) display(values2) PyCon India 2016 36 -
def display2(values1, values2): yield from display(values1) yield from display(values2) def
main3(): g1 = display2("ABC", "XYZ") g2 = display2("...", "...") run_all([g1, g2]) PyCon India 2016 37 -
>>> main3() A . B . C . X .
Y . Z . PyCon India 2016 38 -
Let's try to build a simple concurrency library based on
coroutines. from collections import deque _tasks = deque() def run(task): _tasks.append(task) run_all() def spawn(task): _tasks.appendleft(task) yield PyCon India 2016 39 -
def run_all(): while _tasks: task = _tasks.popleft() try: next(task) except
StopIteration: pass else: _tasks.append(task) PyCon India 2016 40 -
Returning a value from a generator def square(x): """Computes square
of a number using square microservice. """ response = send_request("/api/square", number=x) # Let something else run while square is being computed. yield return response.json()['result'] PyCon India 2016 41 -
def sum_of_squares(x, y): x2 = yield from square(x) y2 =
yield from square(y) return x2 + y2 PyCon India 2016 42 -
Generators are overloaded • Used to build and process data
streams • Also used as coroutines Confusing! PyCon India 2016 43 -
async def square(x): return x*x async def sum_of_squares(x, y): x2
= await square(x) y2 = await square(y) return x2+y2 PyCon India 2016 45 -
Coroutine Protocol >>> square(4) <coroutine object square at 0xb57a6510> >>>
x = square(4) >>> x.send(None) Traceback (most recent call last): File "<stdin>", line 1, in <module> StopIteration: 16 PyCon India 2016 46 -
Running Coroutines def run(coroutine): try: while True: coroutine.send(None) except StopIteration
as e: return e.value ------------------------------------------------------- >>> run(square(4)) 16 PyCon India 2016 47 -
async def display(values): for v in values: await aprint(v) -------------------------------------------------------
>>> run(display("ABC")) A B C PyCon India 2016 49 -
"""coro.py - a simple coroutine conncurrency library. """ import types
from collections import deque _tasks = deque() def run(task): _tasks.append(task) run_all() @types.coroutine def spawn(task): _tasks.appendleft(task) yield PyCon India 2016 51 -
def run_all(): while _tasks: task = _tasks.popleft() try: task.send(None) except
StopIteration: pass else: _tasks.append(task) PyCon India 2016 52 -
Async Example from coro import spawn, run import types @types.coroutine
def aprint(x): print(x) yield PyCon India 2016 53 -
async def display(values): for v in values: await aprint(v) async
def main(): await spawn(display("ABC")) await spawn(display("123")) if __name__ == "__main__": run(main()) PyCon India 2016 54 -
"""asocket - simple async socket implementation. """ from socket import
* import types import select # Rename the original socket as _socket as # we are going to write a new socket class _socket = socket PyCon India 2016 57 -
class socket: """Simple async socket. """ def __init__(self, *args): self._sock
= _socket(*args) self._sock.setblocking(0) def __getattr__(self, name): return getattr(self._sock, name) PyCon India 2016 58 -
def connect(self, addr): try: self._sock.connect(addr) except BlockingIOError: pass async def
send(self, data): await wait_for_write(self._sock) return self._sock.send(data) async def recv(self, size): await wait_for_read(self._sock) return self._sock.recv(size) PyCon India 2016 59 -
@types.coroutine def wait_for_read(sock): while True: r, w, e = select.select([sock],
[], [], 0) if r: break yield @types.coroutine def wait_for_write(sock): while True: r, w, e = select.select([], [sock], [], 0) if w: break yield PyCon India 2016 60 -
Async IO Example from asocket import * from coro import
spawn, run async def echo_client(host, port, label): sock = socket(AF_INET, SOCK_STREAM) sock.connect((host, port)) for i in range(3): await sock.send(str(i).encode('ascii')) data = await sock.recv(1024) print(label, data.decode('ascii')) PyCon India 2016 61 -
async def main(): host, port = 'localhost', 1234 await spawn(echo_client(host,
port, 'A')) await spawn(echo_client(host, port, 'B')) await spawn(echo_client(host, port, 'C')) await spawn(echo_client(host, port, 'D')) if __name__ == "__main__": run(main()) PyCon India 2016 62 -
$ python echo_client.py B 0 A 0 D 0 C
0 B 1 B 2 A 1 D 1 C 1 A 2 D 2 C 2 PyCon India 2016 63 -
Summary Generators are awesome. Use them for everything! Coroutines are
good vehicles for concurrency. Explore them more! PyCon India 2016 64