Merge remote-tracking branch 'benma/nanopb-2'

Author: benma

.ci/run-container-ci

@@ -25,7 +25,7 @@
 set -e
 set -x
 
-CONTAINER=shiftcrypto/firmware_v2:16
+CONTAINER=shiftcrypto/firmware_v2:17
 
 if [ "$1" == "pull" ] ; then
 	docker pull "$CONTAINER"

Dockerfile

@@ -34,19 +34,6 @@ RUN mkdir ~/Downloads &&\
     rm -f gcc.tar.bz2 &&\
     cd ~/Downloads && rsync -a gcc-arm-none-eabi-8-2018-q4-major/ /usr/local/
 
-# Install nanopb
-RUN cd ~/Downloads &&\
-    wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.3.9.2-linux-x86.tar.gz &&\
-    echo "00624f2834066ab31613dd2bb53b258a3b81cd83554df4a7cf49725c5cf34c46 nanopb-0.3.9.2-linux-x86.tar.gz" | sha256sum -c &&\
-    tar -xvzf nanopb-0.3.9.2-linux-x86.tar.gz &&\
-    rm -f nanopb-0.3.9.2-linux-x86.tar.gz &&\
-    mv ~/Downloads/nanopb-0.3.9.2-linux-x86/generator-bin/protoc* /usr/local/bin/ &&\
-    mv ~/Downloads/nanopb-0.3.9.2-linux-x86/generator-bin/nanopb_generator /usr/local/bin/ &&\
-    mv ~/Downloads/nanopb-0.3.9.2-linux-x86/generator-bin/libpython*so* /usr/local/bin/ &&\
-    mv ~/Downloads/nanopb-0.3.9.2-linux-x86/generator-bin/*so* /usr/local/lib/ &&\
-    mv ~/Downloads/nanopb-0.3.9.2-linux-x86/generator-bin/include/* /usr/local/include/ &&\
-    mv ~/Downloads/nanopb-0.3.9.2-linux-x86/generator/proto/google/ /usr/local/include/
-
 # Tools for building
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -111,6 +98,11 @@ RUN python3 -m pip install --upgrade \
     wheel==0.33.6 \
     twine==1.15.0
 
+# For protoc
+RUN apt-get update && apt-get install -y protobuf-compiler
+# Make Python3 the default, so tools/nanopb/generator/*.py run with Python3.
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+
 # Developer tools
 RUN apt-get update && apt-get install -y \
     bash-completion

messages/CMakeLists.txt

@@ -1,5 +1,3 @@
-find_program(PROTOC_GEN_NANOPB protoc-gen-nanopb)
-
 set(PROTO_FILES
   hww.proto
   backup.proto
@@ -18,13 +16,6 @@ set(PROTO_FILES
 string(REGEX REPLACE "\.proto" ".pb.c" OUTPUT_SOURCES "${PROTO_FILES}")
 string(REGEX REPLACE "\.proto" ".pb.h" OUTPUT_HEADERS "${PROTO_FILES}")
 
-if (PROTOC_GEN_NANOPB STREQUAL "PROTOC_GEN_NANOPB-NOTFOUND")
-  message(FATAL_ERROR
-    "Could not find 'protoc-gen-nanopb'.\n"
-    "Please install it from here: https://jpa.kapsi.fi/nanopb/download/\n"
-    "And add generator-bin/ to your path.")
-endif()
-
 # Create absolute paths to protobuf sources
 foreach(i ${PROTO_FILES})
   list(APPEND PROTO_FILES_ABSOLUTE "${CMAKE_CURRENT_SOURCE_DIR}/${i}")
@@ -55,7 +46,11 @@ add_custom_command(
   OUTPUT ${OUTPUT_SOURCES} ${OUTPUT_HEADERS}
   DEPENDS ${PROTO_FILES} ${PROTO_OPTION_FILES}
   COMMAND
-  ${PROTOC} "--proto_path=${CMAKE_CURRENT_SOURCE_DIR}" "--nanopb_out=-I${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_CURRENT_BINARY_DIR}" ${PROTO_FILES_ABSOLUTE}
+  ${CMAKE_SOURCE_DIR}/tools/nanopb/generator/protoc
+    --plugin=protoc-gen-nanopb=${CMAKE_SOURCE_DIR}/tools/nanopb/generator/protoc-gen-nanopb
+    --proto_path=${CMAKE_CURRENT_SOURCE_DIR}
+    '--nanopb_out=-I${CMAKE_CURRENT_SOURCE_DIR}:${CMAKE_CURRENT_BINARY_DIR}'
+    ${PROTO_FILES_ABSOLUTE}
 )
 
 add_custom_target(

messages/backup.options

@@ -12,6 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// skip `bool has_*` fields for submessages
+// https://github.com/nanopb/nanopb/blob/1fdc916e984b27cc0a3824427f70a15def567a2e/docs/migration.rst#submessages-now-have-has-field-in-proto3-mode
+* proto3_singular_msgs:true
+
 Backup.backup_version no_unions:true
 BackupData.seed fixed_length:true max_size:32
 BackupData.generator fixed_length:true max_size:20

messages/btc.options

@@ -13,6 +13,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// skip `bool has_*` fields for submessages
+// https://github.com/nanopb/nanopb/blob/1fdc916e984b27cc0a3824427f70a15def567a2e/docs/migration.rst#submessages-now-have-has-field-in-proto3-mode
+* proto3_singular_msgs:true
+
 BTCScriptConfig.Multisig.xpubs max_count:15
 BTCPubRequest.keypath max_count:10
 BTCSignInitRequest.script_configs max_count:3

src/backup/backup_common.c

@@ -97,9 +97,11 @@ void backup_calculate_checksum(BackupContent* content, BackupData* backup_data,
  * @param[in] field The field that is encoded.
  * @param[in] arg The encode/decode data passed to the callback.
  */
-static bool _encode_backup_data(pb_ostream_t* ostream, const pb_field_t* field, void* const* arg)
+static bool _encode_backup_data(
+    pb_ostream_t* ostream,
+    const pb_field_iter_t* field,
+    void* const* arg)
 {
-    (void)field;
     encode_data_t* encode_data = (encode_data_t*)*arg;
     if (*(encode_data->mode) != BackupMode_PLAINTEXT) {
         return false;
@@ -161,7 +163,7 @@ backup_error_t backup_fill(
 
     // This function is a callback to nanopb when serializing the `data` field.
     // We call it here manually once more to extract the length.
-    _encode_backup_data(&submessage_out_stream, iter.pos, (void* const*)&encode_data);
+    _encode_backup_data(&submessage_out_stream, &iter, (void* const*)&encode_data);
 
     // This length is the serialization of BackupData as protobuf, including the `data` field
     // tag prefix serialization. See the comment in backup.proto for more details.