Add COPY CMD in TS_FILES only for TS_PACKAGE

- Использовать собственную реализацию копирования файлов в bindir для решения проблемы
повторного копирования и отсутствия прав на создание файлов в некоторых случаях
- Для TS_PACKAGE использовать общий для всех остальных модулей механизм копирования исходников вместе сборочных команд копирования
- Макрос TS_FILES не должен создавать сборочные команды копирования - все файлы проекта уже присутствуют в bindir
- Для TS_LARGE_FILES поддержать специальный механизм создающий команду копирования (так как местоположение файла меняется)
- Исправить опечатки и дополнить описание модулей и макросов
commit_hash:c7bbbf35361947845baa84f4806c913103df704d

build/conf/ts/ts.conf

@@ -11,6 +11,9 @@ TS_CONFIG_USE_OUTDIR=
 TS_USE_PREBUILT_NOTS_TOOL=yes
 NOTS_TOOL=${tool:"devtools/frontend_build_platform/nots/builder"}
 
+# Additional commands that module can add (with `&&` as delimiter), if set those will be executed 
+# in build phase just before building the module.
+_TS_PROJECT_SETUP_CMD=
 
 TS_CONFIG_PATH=tsconfig.json
 
@@ -182,12 +185,14 @@ macro _AS_HIDDEN_INPUTS(IN{input}[]) {
   .CMD=${hide;context=TEXT;input:IN}
 }
 
-
-_TS_FILES_COPY_CMD=
+# Input/output directives for files set by use with 
+# SRCS / TS_LARGE_FILES / TS_FILES / TS_FILES_GLOB
+_TS_FILES_INOUTS=
 
 ### @usage: TS_FILES(Files...)
 ###
-### Adds files to output as is. Similar to FILES but works for TS build modules
+### Adds files to output as is. Does not add a command to copy the file to builddir.
+### Similar to FILES but works for TS build modules
 ### Documentation: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-files
 macro TS_FILES(Files...) {
     # TODO: FBP-1795
@@ -207,7 +212,7 @@ macro TS_FILES_GLOB(Glob...) {
 ###
 ### Use large file ether from working copy or from remote storage via placeholder <File>.external
 ### If <File> is present locally (and not a symlink!) it will be copied to build directory.
-### Otherwise macro will try to locate <File>.external, parse it retrieve ot during build phase.
+### Otherwise macro will try to locate <File>.external, parse it and fetch the file during build phase.
 ###
 ### Then file will be copied to DESTINATION folder preserving file structure.
 ### Copied file becomes output of TS_PACKAGE

build/conf/ts/ts_next.conf

@@ -30,13 +30,13 @@ macro TS_NEXT_EXPERIMENTAL_COMPILE() {
     SET(TS_NEXT_COMMAND experimental-compile)
 }
 
-TS_NEXT_CMD=$_TS_FILES_COPY_CMD \
+TS_NEXT_CMD=$_TS_PROJECT_SETUP_CMD \
     && $ADD_VCS_INFO_FILE_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-next $NOTS_TOOL_COMMON_BUILDER_ARGS \
       --ts-next-command ${TS_NEXT_COMMAND} \
       --bundler-config-path ${input:TS_NEXT_CONFIG_PATH} \
       --output-dirs ${TS_NEXT_OUTPUT_DIR} \
-    $_NODE_MODULES_INOUTS ${hide:PEERS} \
+    $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \
     ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \
     ${hide;output:"package.json"} \
     ${hide;kv:"pc magenta"} ${hide;kv:"p TS_NXT"}

build/conf/ts/ts_package.conf

@@ -1,13 +1,12 @@
-
 TS_PACK=$TOUCH_UNIT \
+    && $_TS_PROJECT_SETUP_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-package $_NODE_MODULES_INOUTS \
-    && $COPY_CMD ${input:"package.json"} ${output:"package.json"} \
-    && $_TS_FILES_COPY_CMD \
-    ${hide;kv:"p TS_PKG"} ${hide;kv:"pc magenta"}
+    $_TS_FILES_INOUTS ${hide;kv:"p TS_PKG"} ${hide;kv:"pc magenta"}
 
 ### # internal
 macro _TS_PACKAGE_EPILOGUE() {
-  _TS_PACKAGE_CHECK_FILES()
+    _TS_PACKAGE_CHECK_FILES()
+    TS_FILES(package.json)
 }
 
 ### @usage: TS_PACKAGE()

build/conf/ts/ts_proto.conf

@@ -1,12 +1,11 @@
 _TS_PROTO_OPT=
-_TS_PROTO_IMPL_CMD=$TOUCH_UNIT \
-    && $_TS_FILES_COPY_CMD \
+_TS_PROTO_IMPL_CMD=$_TS_PROJECT_SETUP_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-ts-proto $NOTS_TOOL_COMMON_BUILDER_ARGS \
         ${_TS_PROTO_OPT} \
         --protoc-bin $PROTOC \
         --proto-srcs $_TS_PROTO_SRCS_FILES \
         --proto-paths ./$PROTO_NAMESPACE $ARCADIA_ROOT/$PROTO_NAMESPACE $_PROTO__INCLUDE $ARCADIA_BUILD_ROOT $PROTOBUF_INCLUDE_PATH \
-        $_NODE_MODULES_INOUTS \
+        $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS \
         ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \
         ${hide;output:"package.json"} \
         ${hide:PROTO_FAKEID} \

build/conf/ts/ts_tsc.conf

@@ -1,6 +1,6 @@
-TS_TSC_CMD=$_TS_FILES_COPY_CMD \
+TS_TSC_CMD=$_TS_PROJECT_SETUP_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-tsc $NOTS_TOOL_COMMON_BUILDER_ARGS \
-    $_NODE_MODULES_INOUTS ${hide:PEERS} \
+    $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \
     ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \
     ${hide;output:"package.json"} \
     ${hide;kv:"pc magenta"} ${hide;kv:"p TS_TSC"}

build/conf/ts/ts_vite.conf

@@ -32,13 +32,12 @@ macro VITE_OUTPUT(DirName) {
     SET(VITE_OUTPUT_DIR $DirName)
 }
 
-
-TS_VITE_CMD=$_TS_FILES_COPY_CMD \
+TS_VITE_CMD=$_TS_PROJECT_SETUP_CMD \
     && $ADD_VCS_INFO_FILE_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-vite $NOTS_TOOL_COMMON_BUILDER_ARGS \
       --bundler-config-path ${input:VITE_CONFIG_PATH} \
       --output-dirs ${VITE_OUTPUT_DIR} \
-    $_NODE_MODULES_INOUTS ${hide:PEERS} \
+    $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \
     ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \
     ${hide;output:"package.json"} \
     ${hide;kv:"pc magenta"} ${hide;kv:"p TS_VIT"}

build/conf/ts/ts_webpack.conf

@@ -30,12 +30,12 @@ macro WEBPACK_OUTPUT(FirstDirName, DirNames...) {
     SET(WEBPACK_OUTPUT_DIR $FirstDirName $DirNames)
 }
 
-TS_WEBPACK_CMD=$_TS_FILES_COPY_CMD \
+TS_WEBPACK_CMD=$_TS_PROJECT_SETUP_CMD \
     && $ADD_VCS_INFO_FILE_CMD \
     && $NOTS_TOOL $NOTS_TOOL_BASE_ARGS build-webpack $NOTS_TOOL_COMMON_BUILDER_ARGS \
       --bundler-config-path ${input:WEBPACK_CONFIG_PATH} \
       --output-dirs ${WEBPACK_OUTPUT_DIR} \
-    $_NODE_MODULES_INOUTS ${hide:PEERS} \
+    $_NODE_MODULES_INOUTS $_TS_FILES_INOUTS ${hide:PEERS} \
     ${hide;input:"package.json"} ${TS_CONFIG_FILES} $_AS_HIDDEN_INPUTS(IN $TS_INPUT_FILES) \
     ${hide;output:"package.json"} \
     ${hide;kv:"pc magenta"} ${hide;kv:"p TS_WPK"}

build/plugins/nots.py

@@ -310,6 +310,12 @@ def _build_cmd_input_paths(paths: list[str] | tuple[str], hide=False, disable_in
     return _build_directives([hide_part, disable_ip_part, "input"], paths)
 
 
+def _build_cmd_output_paths(paths: list[str] | tuple[str], hide=False):
+    hide_part = "hide" if hide else ""
+
+    return _build_directives([hide_part, "output"], paths)
+
+
 def _create_erm_json(unit: NotsUnitType):
     from lib.nots.erm_json_lite import ErmJsonLite
 
@@ -517,15 +523,14 @@ def on_setup_build_env(unit: NotsUnitType) -> None:
     unit.set(["NOTS_TOOL_BUILD_ENV", " ".join(options)])
 
 
-def __set_append(unit: NotsUnitType, var_name: str, value: UnitType.PluginArgs) -> None:
+def __set_append(unit: NotsUnitType, var_name: str, value: UnitType.PluginArgs, delimiter: str = " ") -> None:
     """
     SET_APPEND() python naive implementation - append value/values to the list of values
     """
-    previous_value = unit.get(var_name) or ""
-    value_in_str = " ".join(value) if isinstance(value, list) or isinstance(value, tuple) else value
-    new_value = previous_value + " " + value_in_str
-
-    unit.set([var_name, new_value])
+    old_str_value = unit.get(var_name)
+    old_values = [old_str_value] if old_str_value else []
+    new_values = list(value) if isinstance(value, list) or isinstance(value, tuple) else [value]
+    unit.set([var_name, delimiter.join(old_values + new_values)])
 
 
 def __strip_prefix(prefix: str, line: str) -> str:
@@ -920,21 +925,24 @@ def on_set_ts_test_for_vars(unit: NotsUnitType, for_mod: str) -> None:
     unit.set(["TS_TEST_FOR_PATH", rootrel_arc_src(for_mod, unit)])
 
 
-@_with_report_configure_error
-def on_ts_files(unit: NotsUnitType, *files: str) -> None:
-    for f in files:
+def __on_ts_files(unit: NotsUnitType, files_in: list[str], files_out: list[str]) -> None:
+    for f in files_in:
         if f.startswith(".."):
             ymake.report_configure_error(
-                "Macro TS_FILES() does not allow to get files from parent directory.\n"
+                "TS_FILES* macroses are only allowed to use files inside the project directory.\n"
                 f"Got path '{f}'.\n"
                 "Docs: https://docs.yandex-team.ru/frontend-in-arcadia/references/TS_PACKAGE#ts-files."
             )
 
-    new_cmds = ['$COPY_CMD ${{context=TEXT;input:"{0}"}} ${{noauto;output:"{0}"}}'.format(f) for f in files]
-    all_cmds = unit.get("_TS_FILES_COPY_CMD")
-    if all_cmds:
-        new_cmds.insert(0, all_cmds)
-    unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)])
+    new_items = _build_cmd_input_paths(paths=files_in, hide=True, disable_include_processor=True)
+    new_items += _build_cmd_output_paths(paths=files_out, hide=True)
+    __set_append(unit, "_TS_FILES_INOUTS", new_items)
+    logger.print_vars("_TS_FILES_INOUTS")
+
+
+@_with_report_configure_error
+def on_ts_files(unit: NotsUnitType, *files: str) -> None:
+    __on_ts_files(unit, files, files)
 
 
 @_with_report_configure_error
@@ -951,21 +959,21 @@ def on_ts_large_files(unit: NotsUnitType, destination: str, *files: list[str]) -
         )
         return
 
+    in_files = [os.path.join('${BINDIR}', f) for f in files]
+    out_files = [os.path.join('${BINDIR}', destination, f) for f in files]
+
     # TODO: FBP-1795
     # ${BINDIR} prefix for input is important to resolve to result of LARGE_FILES and not to SOURCEDIR
-    new_cmds = [
-        '$COPY_CMD ${{context=TEXT;input:"${{BINDIR}}/{0}"}} ${{noauto;output:"{1}/{0}"}}'.format(f, destination)
-        for f in files
-    ]
-    all_cmds = unit.get("_TS_FILES_COPY_CMD")
-    if all_cmds:
-        new_cmds.insert(0, all_cmds)
-    unit.set(["_TS_FILES_COPY_CMD", " && ".join(new_cmds)])
+    new_items = [f'$COPY_CMD {i} {o}' for (i, o) in zip(in_files, out_files)]
+    __set_append(unit, "_TS_PROJECT_SETUP_CMD", new_items, " && ")
+    logger.print_vars("_TS_PROJECT_SETUP_CMD")
+
+    __on_ts_files(unit, in_files, out_files)
 
 
 @_with_report_configure_error
 def on_ts_package_check_files(unit: NotsUnitType) -> None:
-    ts_files = unit.get("_TS_FILES_COPY_CMD")
+    ts_files = unit.get("_TS_FILES_INOUTS")
     if ts_files == "":
         ymake.report_configure_error(
             "\n"

devtools/frontend_build_platform/nots/builder/api/builders/base_builder.py

@@ -16,12 +16,64 @@
 )
 from build.plugins.lib.nots.typescript import TsConfig
 from devtools.frontend_build_platform.libraries.logging import timeit
-from ..models import BuildError, CommonBuildersOptions
-from ..utils import copy_if_not_exists, extract_peer_tars, popen, resolve_bin
+from ..models import BaseOptions, BuildError, CommonBuildersOptions
+from ..utils import recursive_copy, extract_peer_tars, popen, resolve_bin
+
+
+class BaseBuilderFileManager(object):
+    """
+    Re-usaable basic functionality for managing src files.
+    One do not need to inherit it in builders - inherit BaseBuilder instead and customize the functions.
+    """
+
+    def __init__(self, options: BaseOptions, ts_config_path: str = "", output_dirs: list[str] = []):
+        self.options = options
+        self.ts_config_path = ts_config_path
+        self.output_dirs = output_dirs
+
+    @timeit
+    def _get_copy_ignore_list(self) -> set[str]:
+        return {
+            # IDE's
+            ".idea",
+            ".vscode",
+            # Output dirs
+            "dist",
+            pm_constants.BUILD_DIRNAME,
+            pm_constants.BUNDLE_DIRNAME,
+            # Dependencies
+            pm_constants.NODE_MODULES_DIRNAME,
+            pm_constants.PNPM_LOCKFILE_FILENAME,
+            # ya-make artefacts
+            pm_constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME,
+            "output.tar",  # TODO FBP-1978
+            pm_constants.OUTPUT_TAR_UUID_FILENAME,
+            # Other
+            ".traces",
+            "a.yaml",
+            self.ts_config_path,  # Will be generated inside the builder (merged all the `extends`)
+        }.union(self.output_dirs)
+
+    @timeit
+    def _copy_src_files_to_bindir(self):
+        ignore_list = self._get_copy_ignore_list()
+        for entry in os.scandir(self.options.curdir):
+            if entry.name in ignore_list:
+                continue
+
+            dst = os.path.normpath(os.path.join(self.options.bindir, entry.name))
+            recursive_copy(entry.path, dst)
+
+    @timeit
+    def _copy_package_json(self):
+        shutil.copyfile(
+            pm_utils.build_pj_path(self.options.curdir),
+            pm_utils.build_pj_path(self.options.bindir),
+        )
 
 
 @add_metaclass(ABCMeta)
-class BaseBuilder(object):
+class BaseBuilder(BaseBuilderFileManager):
     @staticmethod
     @timeit
     def load_ts_config(ts_config_file: str, sources_path: str) -> TsConfig:
@@ -70,9 +122,7 @@ def __init__(
         :param external_dependencies: external dependencies which will be linked to node_modules/ (mapping name -> path)
         :type external_dependencies: dict
         """
-        self.options = options
-        self.output_dirs = output_dirs
-        self.ts_config_path = ts_config_path
+        super(BaseBuilder, self).__init__(options=options, ts_config_path=ts_config_path, output_dirs=output_dirs)
         self.copy_package_json = copy_package_json
         self.external_dependencies = external_dependencies
 
@@ -93,57 +143,14 @@ def resolve_bin(self, package_name: str, bin_name: str = None) -> str:
 
     @timeit
     def build(self):
-        self._copy_package_json()
+        if self.copy_package_json:
+            self._copy_package_json()
         self._prepare_dependencies()
         self._copy_src_files_to_bindir()
 
         self._build()
         self._run_javascript_after_build()
 
-    @timeit
-    def _get_copy_ignore_list(self) -> set[str]:
-        return {
-            # IDE's
-            ".idea",
-            ".vscode",
-            # Output dirs
-            "dist",
-            pm_constants.BUILD_DIRNAME,
-            pm_constants.BUNDLE_DIRNAME,
-            # Dependencies
-            pm_constants.NODE_MODULES_DIRNAME,
-            pm_constants.PNPM_LOCKFILE_FILENAME,
-            # ya-make artefacts
-            pm_constants.NODE_MODULES_WORKSPACE_BUNDLE_FILENAME,
-            "output.tar",  # TODO FBP-1978
-            pm_constants.OUTPUT_TAR_UUID_FILENAME,
-            # Other
-            ".traces",
-            "a.yaml",
-            self.ts_config_path,  # Will be generated inside the builder (merged all the `extends`)
-        }.union(self.output_dirs)
-
-    @timeit
-    def _copy_src_files_to_bindir(self):
-        ignore_list = self._get_copy_ignore_list()
-
-        for entry in os.scandir(self.options.curdir):
-            if entry.name in ignore_list:
-                continue
-
-            dst = os.path.normpath(os.path.join(self.options.bindir, entry.name))
-            copy_if_not_exists(entry.path, dst)
-
-    @timeit
-    def _copy_package_json(self):
-        if not self.copy_package_json:
-            return
-
-        shutil.copyfile(
-            pm_utils.build_pj_path(self.options.curdir),
-            pm_utils.build_pj_path(self.options.bindir),
-        )
-
     @timeit
     def __extract_peer_tars(self, *args, **kwargs):
         return extract_peer_tars(*args, **kwargs)