Merge pull request NVIDIA#630 from NVIDIA/gh/release

Updating GNMT/PyT/TF for Ampere, fixes in WnD/TF and Jasper/PyT

Author: nv-kkudrynski

PyTorch/SpeechRecognition/Jasper/.dockerignore

@@ -5,3 +5,5 @@ checkpoints/
 datasets/
 external/tensorrt-inference-server/
 checkpoints/
+triton/model_repo
+triton/deploy

PyTorch/SpeechRecognition/Jasper/.gitmodules

@@ -1,4 +1,4 @@
-[submodule "external/tensorrt-inference-server"]
-	path = external/tensorrt-inference-server
-	url = https://github.com/NVIDIA/tensorrt-inference-server.git
-	branch = r19.06
+[submodule "external/triton-inference-server"]
+	path = external/triton-inference-server
+	url = https://github.com/NVIDIA/triton-inference-server
+	branch = r19.12

PyTorch/SpeechRecognition/Jasper/external/Dockerfile.client.patched

@@ -0,0 +1,95 @@
+# Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Default setting is building on nvidia/cuda:10.1-devel-ubuntu18.04
+ARG BASE_IMAGE=nvidia/cuda:10.1-devel-ubuntu18.04
+
+FROM ${BASE_IMAGE}
+
+# Default to use Python3. Allowed values are "2" and "3".
+ARG PYVER=3
+
+# Ensure apt-get won't prompt for selecting options
+ENV DEBIAN_FRONTEND=noninteractive
+ENV PYVER=$PYVER
+
+RUN PYSFX=`[ "$PYVER" != "2" ] && echo "$PYVER" || echo ""` && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+            software-properties-common \
+            autoconf \
+            automake \
+            build-essential \
+            cmake \
+            curl \
+            git \
+            libopencv-dev \
+            libopencv-core-dev \
+            libssl-dev \
+            libtool \
+            pkg-config \
+            python${PYSFX} \
+            python${PYSFX}-pip \
+            python${PYSFX}-dev && \
+    pip${PYSFX} install --upgrade setuptools wheel
+
+RUN PYSFX=`[ "$PYVER" != "2" ] && echo "$PYVER" || echo ""` && \
+    pip${PYSFX} install --upgrade grpcio-tools
+
+# Build expects "python" executable (not python3).
+RUN rm -f /usr/bin/python && \
+    ln -s /usr/bin/python$PYVER /usr/bin/python
+
+# Build the client library and examples
+WORKDIR /workspace
+COPY VERSION .
+COPY build build
+COPY src/clients src/clients
+COPY src/core src/core
+
+RUN cd build && \
+    cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX:PATH=/workspace/install && \
+    make -j16 trtis-clients
+RUN cd install && \
+    export VERSION=`cat /workspace/VERSION` && \
+    tar zcf /workspace/v$VERSION.clients.tar.gz *
+
+# For CI testing need to install a test script.
+COPY qa/L0_client_tar/test.sh /tmp/test.sh
+
+# Install an image needed by the quickstart and other documentation.
+COPY qa/images/mug.jpg images/mug.jpg
+
+# Install the dependencies needed to run the client examples. These
+# are not needed for building but including them allows this image to
+# be used to run the client examples. The special upgrade and handling
+# of pip is needed to get numpy to install correctly with python2 on
+# ubuntu 16.04.
+RUN python -m pip install --user --upgrade pip && \
+    python -m pip install --upgrade install/python/tensorrtserver-*.whl numpy pillow
+
+ENV PATH //workspace/install/bin:${PATH}
+ENV LD_LIBRARY_PATH /workspace/install/lib:${LD_LIBRARY_PATH}

PyTorch/SpeechRecognition/Jasper/external/triton-inference-server

@@ -0,0 +1 @@
+Subproject commit a1f3860ba65c0fd8f2be3adfcab2673efd039348

PyTorch/SpeechRecognition/Jasper/inference.py

@@ -56,6 +56,10 @@ def parse_args():
     parser.add_argument("--wav", type=str, help='absolute path to .wav file (16KHz)')
     parser.add_argument("--cpu", action="store_true", help="Run inference on CPU")
     parser.add_argument("--ema", action="store_true", help="If available, load EMA model weights")
+
+    # FIXME Unused, but passed by Triton helper scripts
+    parser.add_argument("--pyt_fp16", action='store_true', help='use half precision')
+
     return parser.parse_args()
 
 def calc_wer(data_layer, audio_processor,

PyTorch/SpeechRecognition/Jasper/model.py

@@ -100,7 +100,7 @@ def __init__(self, cfg):
     def forward(self, x):
         sh = x.shape
 
-        mask = torch.zeros(x.shape).byte()
+        mask = torch.zeros(x.shape, dtype=torch.bool)
         for idx in range(sh[0]):
             for _ in range(self.cutout_x_regions):
                 cutout_x_left = int(random.uniform(0, sh[1] - self.cutout_x_width))
@@ -130,7 +130,7 @@ def __init__(self, cfg):
     def forward(self, x):
         sh = x.shape
 
-        mask = torch.zeros(x.shape, dtype=torch.uint8)
+        mask = torch.zeros(x.shape, dtype=torch.bool)
 
         for idx in range(sh[0]):
             for i in range(self.cutout_rect_regions):
@@ -275,7 +275,7 @@ def __init__(self, in_channels, out_channels, kernel_size, stride=1,
 
     def get_seq_len(self, lens):
         return ((lens + 2 * self.padding[0] - self.dilation[0] * (
-            self.kernel_size[0] - 1) - 1) / self.stride[0] + 1)
+            self.kernel_size[0] - 1) - 1) // self.stride[0] + 1)
 
     def forward(self, inp):
         if self.use_conv_mask:

PyTorch/SpeechRecognition/Jasper/optimizers.py

@@ -98,7 +98,7 @@ def step(self, closure=None):
 
                 state['step'] += 1
                 # Decay the first and second moment running average coefficient
-                exp_avg.mul_(beta1).add_(1 - beta1, grad)
+                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
                 exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)
                 if amsgrad:
                     # Maintains the maximum of all 2nd moment running avg. till now
@@ -111,7 +111,7 @@ def step(self, closure=None):
                 bias_correction1 = 1 - beta1 ** state['step']
                 bias_correction2 = 1 - beta2 ** state['step']
                 step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1
-                p.data.add_(-step_size,  torch.mul(p.data, group['weight_decay']).addcdiv_(1, exp_avg, denom) )
+                p.data.add_(torch.mul(p.data, group['weight_decay']).addcdiv_(1, exp_avg, denom), alpha=-step_size)
 
         return loss
 
@@ -201,7 +201,7 @@ def step(self, closure=None):
                 if exp_avg_sq == 0:
                     exp_avg_sq.copy_(norm)
                 else:
-                    exp_avg_sq.mul_(beta2).add_(1 - beta2, norm)
+                    exp_avg_sq.mul_(beta2).add_(norm, alpha=1 - beta2)
 
                 if amsgrad:
                     # Maintains the maximum of all 2nd moment running avg. till now
@@ -213,11 +213,11 @@ def step(self, closure=None):
 
                 grad.div_(denom)
                 if group['weight_decay'] != 0:
-                    grad.add_(group['weight_decay'], p.data)
+                    grad.add_(p.data, alpha=group['weight_decay'])
                 if group['grad_averaging']:
                     grad.mul_(1 - beta1)
                 exp_avg.mul_(beta1).add_(grad)
 
-                p.data.add_(-group['lr'], exp_avg)
+                p.data.add_(exp_avg, alpha=-group['lr'])
 
         return loss

PyTorch/SpeechRecognition/Jasper/scripts/train.sh

@@ -47,9 +47,9 @@ CMD+=" --seed=$SEED"
 CMD+=" --optimizer=novograd"
 CMD+=" --dataset_dir=$DATA_DIR"
 CMD+=" --val_manifest=$DATA_DIR/librispeech-dev-clean-wav.json"
-CMD+=" --train_manifest=$DATA_DIR/librispeech-train-clean-100-wav.json,"
-CMD+="$DATA_DIR/librispeech-train-clean-360-wav.json,"
-CMD+="$DATA_DIR/librispeech-train-other-500-wav.json"
+CMD+=" --train_manifest=$DATA_DIR/librispeech-train-clean-100-wav.json"
+CMD+=",$DATA_DIR/librispeech-train-clean-360-wav.json"
+CMD+=",$DATA_DIR/librispeech-train-other-500-wav.json"
 CMD+=" --weight_decay=1e-3"
 CMD+=" --save_freq=$SAVE_FREQUENCY"
 CMD+=" --eval_freq=100"

PyTorch/SpeechRecognition/Jasper/tensorrt/Dockerfile

@@ -1,8 +1,10 @@
-ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:20.03-py3
+ARG FROM_IMAGE_NAME=nvcr.io/nvidia/pytorch:19.10-py3
 FROM ${FROM_IMAGE_NAME}
 
+RUN apt-get update && apt-get install -y python3
+
 WORKDIR /tmp/onnx-trt
-COPY trt/onnx-trt.patch .
+COPY tensorrt/onnx-trt.patch .
 RUN git clone https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && git checkout 8716c9b && git submodule update --init --recursive && \
     patch -f < ../onnx-trt.patch && mkdir build && cd build && cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr -DGPU_ARCHS="60 70 75" && make -j16 && make install && mv -f /usr/lib/libnvonnx* /usr/lib/x86_64-linux-gnu/ && ldconfig
 
@@ -11,7 +13,7 @@ RUN git clone https://github.com/onnx/onnx-tensorrt.git && cd onnx-tensorrt && g
 # At the same step, also install TRT pip reqs
 WORKDIR /tmp/pipReqs
 COPY requirements.txt /tmp/pipReqs/jocRequirements.txt
-COPY trt/requirements.txt /tmp/pipReqs/trtRequirements.txt
+COPY tensorrt/requirements.txt /tmp/pipReqs/trtRequirements.txt
 RUN pip install --disable-pip-version-check -U -r jocRequirements.txt -r trtRequirements.txt
 
 

PyTorch/SpeechRecognition/Jasper/tensorrt/scripts/docker/build.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
 
 # Constructs a docker image containing dependencies for execution of JASPER through TRT
-echo "docker build . -f ./trt/Dockerfile -t jasper:trt6"
-docker build . -f ./trt/Dockerfile -t jasper:trt6
+echo "docker build . -f ./tensorrt/Dockerfile -t jasper:trt6"
+docker build . -f ./tensorrt/Dockerfile -t jasper:trt6