Tensor Manipulation

Imports

import numpy as np
import torch

1 Tensor in pytorch

• Tensor?
  • Vector (Rank = 1)
    $$\begin{bmatrix}1&2&3\end{bmatrix}$$
  • Matrix (Rank = 2) $$\begin{bmatrix}1&2&3\\4&5&6\end{bmatrix}$$
  • Tensor (Rank > 2)
    • Difficult to display on a 2D screen
    • Possible to visualize mentally, though ability varies
  • We recommend reviewing the mathematical definitions of rank, shape, and axis whenever you have time
  • Many related YouTube videos are available

# Syntax similar to Numpy
tensor = torch.FloatTensor([1, 2, 3, 4])
print(tensor)
print(tensor.shape)
print(tensor[1:3])

tensor([1., 2., 3., 4.])
torch.Size([4])
tensor([2., 3.])

# 3 or more dimensions
tensor3d = torch.FloatTensor([ [[1, 2, 3, 4], 
                                [5, 6, 7, 8],
                                [9, 10, 11, 12]],
                              
                               [[13, 14, 15, 16],
                                [17, 18, 19, 20],
                                [21, 22, 23, 24]] ])
print(tensor3d)
print(tensor3d.shape)

tensor([[[ 1.,  2.,  3.,  4.],
         [ 5.,  6.,  7.,  8.],
         [ 9., 10., 11., 12.]],

        [[13., 14., 15., 16.],
         [17., 18., 19., 20.],
         [21., 22., 23., 24.]]])
torch.Size([2, 3, 4])

2 Tensor Combinations

• Concatenation
  • Join arrays by removing their boundaries. $$\begin{bmatrix} 1 & 2 \\ 3 & 4 \end{bmatrix} (\cdot) \begin{bmatrix} 5 & 6 \\ 7 & 8 \end{bmatrix} \rightarrow \begin{bmatrix} 1 & 2 & 5 & 6 \\ 3 & 4 & 7 & 8 \end{bmatrix}$$

x = torch.FloatTensor([[1, 2], [3, 4]])
y = torch.FloatTensor([[5, 6], [7, 8]])
print(x)
print(y)
print(torch.cat([x,y], dim = 0))
print(torch.cat([x,y], dim = 1))

tensor([[1., 2.],
        [3., 4.]])
tensor([[5., 6.],
        [7., 8.]])
tensor([[1., 2.],
        [3., 4.],
        [5., 6.],
        [7., 8.]])
tensor([[1., 2., 5., 6.],
        [3., 4., 7., 8.]])

• Stack
  • Join arrays while keeping their boundaries distinct. $$\begin{bmatrix} 1 & 2 \\ 3 & 4 \end{bmatrix} (*) \begin{bmatrix} 5 & 6 \\ 7 & 8 \end{bmatrix} \rightarrow \begin{bmatrix} \begin{bmatrix} 1 & 2 \\ 3 & 4 \end{bmatrix} & \begin{bmatrix} 5 & 6 \\ 7 & 8 \end{bmatrix} \end{bmatrix}$$

print(torch.stack([x,y], dim = 0))
print(torch.stack([x,y], dim = 1))

tensor([[[1., 2.],
         [3., 4.]],

        [[5., 6.],
         [7., 8.]]])
tensor([[[1., 2.],
         [5., 6.]],

        [[3., 4.],
         [7., 8.]]])

• View / Reshape
  • Change the shape while preserving the data.
    • Since it returns a pointer, use a copy to avoid modifying the original.

x = torch.rand(2, 3, 4)  # [2, 3, 4] 
y = x.view(2, -1)  # [2, 12] 
z = x.reshape(2, -1) # [2, 12]
print(x)
print(y)
print(z)
z[0] = 0.0
print(x)

tensor([[[0.5022, 0.5057, 0.8796, 0.0525],
         [0.8836, 0.7124, 0.8465, 0.8862],
         [0.7256, 0.5428, 0.9198, 0.6916]],

        [[0.4527, 0.6195, 0.5849, 0.9043],
         [0.7626, 0.5965, 0.5295, 0.8113],
         [0.2377, 0.9379, 0.9978, 0.6270]]])
tensor([[0.5022, 0.5057, 0.8796, 0.0525, 0.8836, 0.7124, 0.8465, 0.8862, 0.7256,
         0.5428, 0.9198, 0.6916],
        [0.4527, 0.6195, 0.5849, 0.9043, 0.7626, 0.5965, 0.5295, 0.8113, 0.2377,
         0.9379, 0.9978, 0.6270]])
tensor([[0.5022, 0.5057, 0.8796, 0.0525, 0.8836, 0.7124, 0.8465, 0.8862, 0.7256,
         0.5428, 0.9198, 0.6916],
        [0.4527, 0.6195, 0.5849, 0.9043, 0.7626, 0.5965, 0.5295, 0.8113, 0.2377,
         0.9379, 0.9978, 0.6270]])
tensor([[[0.0000, 0.0000, 0.0000, 0.0000],
         [0.0000, 0.0000, 0.0000, 0.0000],
         [0.0000, 0.0000, 0.0000, 0.0000]],

        [[0.4527, 0.6195, 0.5849, 0.9043],
         [0.7626, 0.5965, 0.5295, 0.8113],
         [0.2377, 0.9379, 0.9978, 0.6270]]])

• Squeeze
  • Reduce unnecessary rank.

_data = np.array([1,2,3,4])
x = torch.from_numpy(_data)
print(x)
print(x.T)
_data = np.array([_data])
x = torch.from_numpy(_data)
print(x)
print(x.T)
print(x.T.squeeze())

tensor([1, 2, 3, 4])
tensor([1, 2, 3, 4])
tensor([[1, 2, 3, 4]])
tensor([[1],
        [2],
        [3],
        [4]])
tensor([1, 2, 3, 4])

3 Tensor Operations

• Arithmetic - Element-wise operations

# Element-wise multiplication
print(tensor)
print(tensor3d)
print(tensor * tensor3d)

tensor([1., 2., 3., 4.])
tensor([[[ 1.,  2.,  3.,  4.],
         [ 5.,  6.,  7.,  8.],
         [ 9., 10., 11., 12.]],

        [[13., 14., 15., 16.],
         [17., 18., 19., 20.],
         [21., 22., 23., 24.]]])
tensor([[[ 1.,  4.,  9., 16.],
         [ 5., 12., 21., 32.],
         [ 9., 20., 33., 48.]],

        [[13., 28., 45., 64.],
         [17., 36., 57., 80.],
         [21., 44., 69., 96.]]])

# Add a constant
print(tensor3d + 3)

tensor([[[ 4.,  5.,  6.,  7.],
         [ 8.,  9., 10., 11.],
         [12., 13., 14., 15.]],

        [[16., 17., 18., 19.],
         [20., 21., 22., 23.],
         [24., 25., 26., 27.]]])

# tensor addition
tensor_a = torch.FloatTensor([1, 2, 3, 4])
tensor_b = torch.FloatTensor([[1], [2], [3]])
tensor_c = torch.FloatTensor([[[1]],[[2]]])
print(tensor3d)
print(tensor3d + tensor_a)
print(tensor3d + tensor_b)
print(tensor3d + tensor_c)

tensor([[[ 1.,  2.,  3.,  4.],
         [ 5.,  6.,  7.,  8.],
         [ 9., 10., 11., 12.]],

        [[13., 14., 15., 16.],
         [17., 18., 19., 20.],
         [21., 22., 23., 24.]]])
tensor([[[ 2.,  4.,  6.,  8.],
         [ 6.,  8., 10., 12.],
         [10., 12., 14., 16.]],

        [[14., 16., 18., 20.],
         [18., 20., 22., 24.],
         [22., 24., 26., 28.]]])
tensor([[[ 2.,  3.,  4.,  5.],
         [ 7.,  8.,  9., 10.],
         [12., 13., 14., 15.]],

        [[14., 15., 16., 17.],
         [19., 20., 21., 22.],
         [24., 25., 26., 27.]]])
tensor([[[ 2.,  3.,  4.,  5.],
         [ 6.,  7.,  8.,  9.],
         [10., 11., 12., 13.]],

        [[15., 16., 17., 18.],
         [19., 20., 21., 22.],
         [23., 24., 25., 26.]]])

• Tensor transpose
  • shape(2, 3, 4) >> shape(4, 3, 2)
  • 3-dimensional cases are hard to visualize mentally.

print(tensor3d)
print(tensor3d.T)

tensor([[[ 1.,  2.,  3.,  4.],
         [ 5.,  6.,  7.,  8.],
         [ 9., 10., 11., 12.]],

        [[13., 14., 15., 16.],
         [17., 18., 19., 20.],
         [21., 22., 23., 24.]]])
tensor([[[ 1., 13.],
         [ 5., 17.],
         [ 9., 21.]],

        [[ 2., 14.],
         [ 6., 18.],
         [10., 22.]],

        [[ 3., 15.],
         [ 7., 19.],
         [11., 23.]],

        [[ 4., 16.],
         [ 8., 20.],
         [12., 24.]]])

• Tensor dot product
  • Similarly, 3-dimensional cases are hard to visualize mentally, so always check that the equations are correct.
  • shape(2,3,4) * shape(4) >> shape(2,3)

print(tensor3d.matmul(tensor))

tensor([[ 30.,  70., 110.],
        [150., 190., 230.]])
tensor([[[ 30.],
         [ 70.],
         [110.]],

        [[150.],
         [190.],
         [230.]]])
tensor([[ 30.,  70., 110.],
        [150., 190., 230.]])

• Mean, Sum, Max

print(tensor3d)
print(tensor3d.mean())
print(tensor3d.shape)
print(tensor3d.mean(dim=0))
print(tensor3d.mean(dim=1))
print(tensor3d.mean(dim=2))

tensor([[[ 1.,  2.,  3.,  4.],
         [ 5.,  6.,  7.,  8.],
         [ 9., 10., 11., 12.]],

        [[13., 14., 15., 16.],
         [17., 18., 19., 20.],
         [21., 22., 23., 24.]]])
tensor(12.5000)
torch.Size([2, 3, 4])
tensor([[ 7.,  8.,  9., 10.],
        [11., 12., 13., 14.],
        [15., 16., 17., 18.]])
tensor([[ 5.,  6.,  7.,  8.],
        [17., 18., 19., 20.]])
tensor([[ 2.5000,  6.5000, 10.5000],
        [14.5000, 18.5000, 22.5000]])

print(tensor3d.sum())
print(tensor3d.sum(dim=0))

tensor(300.)
tensor([[14., 16., 18., 20.],
        [22., 24., 26., 28.],
        [30., 32., 34., 36.]])

print(tensor3d.max())
print(tensor3d.max(dim=1))
values, indices = tensor3d.max(dim=1)
print(values)

tensor(24.)
torch.return_types.max(
values=tensor([[ 9., 10., 11., 12.],
        [21., 22., 23., 24.]]),
indices=tensor([[2, 2, 2, 2],
        [2, 2, 2, 2]]))
tensor([[ 9., 10., 11., 12.],
        [21., 22., 23., 24.]])

• Inverse matrix

tensor_square = torch.FloatTensor([[1, 7], [-4, 6]])
print(torch.inverse(tensor_square))

tensor([[ 0.1765, -0.2059],
        [ 0.1176,  0.0294]])

4 Exercise (Assignment 6)

• Solve a matrix equation (using only PyTorch)
  • Solve the following matrix equation using the pseudoinverse.
  • Assume that $A^{T}A$ is invertible. $$Ax=B$$ $$A = \begin{bmatrix}0 & 1 \\ 1 & 1 \\ 2 & 1 \\ 3 & 1 \end{bmatrix}$$ $$B = \begin{bmatrix}-1 \\ 0.2 \\ 0.9 \\ 2.1 \end{bmatrix}$$