|
| 1 | +#define NONE 0 |
| 2 | +#define TRIGONOMETRIC 1 |
| 3 | +#define ABSOLUTE 2 |
| 4 | + |
| 5 | +submodule(nf_embedding_layer) nf_embedding_layer_submodule |
| 6 | + use nf_base_layer, only: base_layer |
| 7 | + implicit none |
| 8 | +contains |
| 9 | + module function embedding_layer_cons(vocab_size, model_dimension, positional) result(res) |
| 10 | + integer, intent(in) :: vocab_size, model_dimension |
| 11 | + integer, optional :: positional |
| 12 | + type(embedding_layer) :: res |
| 13 | + |
| 14 | + res % vocab_size = vocab_size |
| 15 | + res % model_dimension = model_dimension |
| 16 | + if (.not. present(positional)) then |
| 17 | + res % positional = NONE |
| 18 | + else |
| 19 | + res % positional = positional |
| 20 | + end if |
| 21 | + end function embedding_layer_cons |
| 22 | + |
| 23 | + module subroutine init(self, input_shape) |
| 24 | + class(embedding_layer), intent(in out) :: self |
| 25 | + integer, intent(in) :: input_shape(:) |
| 26 | + |
| 27 | + self % sequence_length = input_shape(1) |
| 28 | + |
| 29 | + allocate(self % output(self % sequence_length, self % model_dimension)) |
| 30 | + |
| 31 | + allocate(self % weights(self % vocab_size, self % model_dimension)) |
| 32 | + self % weights = 0.1 |
| 33 | + |
| 34 | + allocate(self % dw(self % vocab_size, self % model_dimension)) |
| 35 | + self % dw = 0.0 |
| 36 | + end subroutine init |
| 37 | + |
| 38 | + pure module subroutine forward(self, input) |
| 39 | + class(embedding_layer), intent(in out) :: self |
| 40 | + integer, intent(in) :: input(:) |
| 41 | + integer :: i, index |
| 42 | + |
| 43 | + do concurrent(i = 1: self % sequence_length) |
| 44 | + index = input(i) |
| 45 | + if (index > size(self % weights, 1)) then |
| 46 | + index = 1 |
| 47 | + elseif (index == 0) then |
| 48 | + index = 1 |
| 49 | + end if |
| 50 | + |
| 51 | + self % output(i, :) = self % weights(index, :) |
| 52 | + |
| 53 | + if (self % positional == TRIGONOMETRIC) then |
| 54 | + call self % positional_trigonometric(i) |
| 55 | + elseif (self % positional == ABSOLUTE) then |
| 56 | + call self % positional_absolute(i) |
| 57 | + end if |
| 58 | + end do |
| 59 | + end subroutine forward |
| 60 | + |
| 61 | + pure module subroutine backward(self, input, gradient) |
| 62 | + class(embedding_layer), intent(in out) :: self |
| 63 | + integer, intent(in) :: input(:) |
| 64 | + real, intent(in) :: gradient(:, :) |
| 65 | + integer :: i |
| 66 | + |
| 67 | + do concurrent(i = 1: self % sequence_length) |
| 68 | + self % dw(input(i), :) = self % dw(input(i), :) + gradient(i, :) |
| 69 | + end do |
| 70 | + end subroutine backward |
| 71 | + |
| 72 | + pure module subroutine positional_trigonometric(self, pos) |
| 73 | + class(embedding_layer), intent(in out) :: self |
| 74 | + integer, intent(in) :: pos |
| 75 | + integer :: i |
| 76 | + real :: theta |
| 77 | + |
| 78 | + do concurrent(i = 1: floor(real(self % model_dimension) / 2)) |
| 79 | + theta = (pos - 1) / 10000 ** (real(2 * (i-1)) / self % model_dimension) |
| 80 | + self % output(pos, 2 * i - 1) = self % output(pos, 2 * i - 1) + sin(theta) |
| 81 | + self % output(pos, 2 * i) = self % output(pos, 2 * i) + cos(theta) |
| 82 | + end do |
| 83 | + end subroutine positional_trigonometric |
| 84 | + |
| 85 | + pure module subroutine positional_absolute(self, pos) |
| 86 | + class(embedding_layer), intent(in out) :: self |
| 87 | + integer, intent(in) :: pos |
| 88 | + integer :: i |
| 89 | + |
| 90 | + do concurrent(i = 1: self % model_dimension) |
| 91 | + self % output(pos, i) = self % output(pos, i) + pos - 1 |
| 92 | + end do |
| 93 | + end subroutine positional_absolute |
| 94 | + |
| 95 | + pure module function get_num_params(self) result(num_params) |
| 96 | + class(embedding_layer), intent(in) :: self |
| 97 | + integer :: num_params |
| 98 | + num_params = self % vocab_size * self % model_dimension |
| 99 | + end function get_num_params |
| 100 | + |
| 101 | + module function get_params(self) result(params) |
| 102 | + class(embedding_layer), intent(in), target :: self |
| 103 | + real, allocatable :: params(:) |
| 104 | + real, pointer :: w_(:) => null() |
| 105 | + |
| 106 | + w_(1: product(shape(self % weights))) => self % weights |
| 107 | + params = w_ |
| 108 | + end function get_params |
| 109 | + |
| 110 | + module function get_gradients(self) result(gradients) |
| 111 | + class(embedding_layer), intent(in), target :: self |
| 112 | + real, allocatable :: gradients(:) |
| 113 | + real, pointer :: dw_(:) => null() |
| 114 | + |
| 115 | + dw_(1: product(shape(self % dw))) => self % dw |
| 116 | + gradients = dw_ |
| 117 | + end function get_gradients |
| 118 | + |
| 119 | + module subroutine set_params(self, params) |
| 120 | + class(embedding_layer), intent(in out) :: self |
| 121 | + real, intent(in), target :: params(:) |
| 122 | + |
| 123 | + real, pointer :: p_(:,:) => null() |
| 124 | + |
| 125 | + ! check if the number of parameters is correct |
| 126 | + if (size(params) /= self % get_num_params()) then |
| 127 | + error stop 'Error: number of parameters does not match' |
| 128 | + end if |
| 129 | + |
| 130 | + associate(n => self % vocab_size * self % model_dimension) |
| 131 | + ! reshape the weights |
| 132 | + p_(1:self % vocab_size, 1:self % model_dimension) => params(1 : n) |
| 133 | + self % weights = p_ |
| 134 | + end associate |
| 135 | + |
| 136 | + end subroutine set_params |
| 137 | +end submodule nf_embedding_layer_submodule |
0 commit comments