reading this record

code is folded so the argument and saved outputs come first. expand any cell to inspect the implementation. saved outputs are historical evidence from the source notebook, not a new run or independent verification.

inspect code · cell 01

In [3]:

import numpy as np


class Tensor:
    def __init__(self, data, requires_grad=False):
        self.data = np.asarray(data, dtype=float)
        self.requires_grad = requires_grad
        self.grad = None
        self._op = None
        self._parents = ()
    def _accumulate_grad(self, grad):
        if self.grad is None:
            self.grad = np.zeros_like(self.data)
        self.grad += grad

    def backward(self):
        order = topo_sort(self)
        self.grad = np.ones_like(self.data)
        for node in reversed(order):
            if node._op == "mul":
                left, right = node._parents
                left._accumulate_grad(node.grad * right.data)
                right._accumulate_grad(node.grad * left.data)

            elif node._op == "neg":
                parent, = node._parents
                parent._accumulate_grad(-node.grad)

            elif node._op == "add":
                left, right = node._parents
                left._accumulate_grad(node.grad)
                right._accumulate_grad(node.grad)

            elif node._op == "sub":
                left, right = node._parents
                left._accumulate_grad(node.grad)
                right._accumulate_grad(-node.grad)

            elif node._op == "div":
                left, right = node._parents
                left._accumulate_grad(node.grad / right.data)
                right._accumulate_grad(
                    -node.grad * left.data / right.data**2
                )
            
    def __repr__(self):
        return f"Tensor(data={self.data}, grad={self.grad})"

    def __add__(self, other):
        out = Tensor(self.data + other.data, requires_grad=self.requires_grad or other.requires_grad)
        out._op = "add"
        out._parents = (self, other)
        return out

    def __sub__(self, other):
        out = Tensor(self.data - other.data, requires_grad=self.requires_grad or other.requires_grad)
        out._op = "sub"
        out._parents = (self, other)
        return out
    
    def __neg__(self):
        out = Tensor(-self.data, requires_grad=self.requires_grad)
        out._op = "neg"
        out._parents = (self,)
        return out

    def __mul__(self, other):
        out = Tensor(self.data * other.data, requires_grad=self.requires_grad or other.requires_grad)
        out._op = "mul"
        out._parents = (self, other)    
        return out
    
    def __truediv__(self, other):
        out = Tensor(self.data / other.data, requires_grad=self.requires_grad or other.requires_grad)
        out._op = "div"
        out._parents = (self, other)
        return out
def topo_sort(tensor):
    visited = set()
    order = []
    
    def visit(node):
        if node in visited:
            return

        visited.add(node)

        for parent in node._parents:
            visit(parent)

        order.append(node)

    visit(tensor)
    return order

    


a = Tensor([2.0], requires_grad=True)
b = Tensor([3.0], requires_grad=True)

a, b

Out[3]:

(Tensor(data=[2.], grad=None), Tensor(data=[3.], grad=None))

inspect code · cell 02

In [4]:

'topo_sort' in globals()

Out[4]:

True

inspect code · cell 03

In [ ]:

c = a * b
c.backward()

print(c.grad)  # [1.]
print(a.grad)  # [3.]
print(b.grad)  # [2.]

[1.]
[3.]
[2.]

inspect code · cell 04

In [6]:

a = Tensor([2.0], requires_grad=True)
b = Tensor([3.0], requires_grad=True)

c = a * b
d = c + a
e = -d

e.backward()

print(a.grad)  # [-4.]
print(b.grad)  # [-2.]

[-4.]
[-2.]

inspect code · cell 05

In [7]:

a = Tensor([2.0], requires_grad=True)
b = Tensor([5.0], requires_grad=True)

f = (a * b + a) / (b - a)

f.backward()

print("f =", f.data)
print("df/da =", a.grad)
print("df/db =", b.grad)

f = [4.]
df/da = [3.33333333]
df/db = [-0.66666667]

inspect code · cell 06

In [8]:

def f_plain(a, b):
    return (a * b + a) / (b - a)


eps = 1e-5
a0 = 2.0
b0 = 5.0

df_da = (
    f_plain(a0 + eps, b0)
    - f_plain(a0 - eps, b0)
) / (2 * eps)

df_db = (
    f_plain(a0, b0 + eps)
    - f_plain(a0, b0 - eps)
) / (2 * eps)

print("finite difference df/da =", df_da)
print("finite difference df/db =", df_db)

finite difference df/da = 3.3333333333551702
finite difference df/db = -0.6666666666710341

inspect code · cell 07

In [9]:

a = Tensor([2.0], requires_grad=True)
b = Tensor([5.0], requires_grad=True)

f = (a * b + a) / (b - a)

f.backward()

print("autograd df/da =", a.grad)
print("autograd df/db =", b.grad)

autograd df/da = [3.33333333]
autograd df/db = [-0.66666667]