feat: add force flag to import omitted fields [#62]

Author: marcolink

lib/index.ts

@@ -6,7 +6,7 @@ import UpdateRenderer from 'listr-update-renderer'
 import VerboseRenderer from 'listr-verbose-renderer'
 import { startCase } from 'lodash'
 import PQueue from 'p-queue'
-
+import { pipe } from 'lodash/fp'
 import { displayErrorLog, setupLogging, writeErrorLogFile } from 'contentful-batch-libs/dist/logging'
 import { wrapTask } from 'contentful-batch-libs/dist/listr'
 
@@ -17,6 +17,9 @@ import transformSpace from './transform/transform-space'
 import { assertDefaultLocale, assertPayload } from './utils/validations'
 import parseOptions from './parseOptions'
 import { ContentfulMultiError, LogItem } from './utils/errors'
+import { transformers } from './transform/transformers'
+import { ContentTypeProps } from 'contentful-management'
+import { forceDeleteOmittedFieldTransform } from './transform/force-delete-omitted-field-transform'
 
 const ONE_SECOND = 1000
 
@@ -34,28 +37,29 @@ function createListrOptions (options) {
 
 // These type definitions follow what is specified in the Readme
 type RunContentfulImportParams = {
-  spaceId: string,
-  environmentId?: string,
-  managementToken: string,
-  contentFile?: string,
-  content?: object,
-  contentModelOnly?: boolean,
-  skipContentModel?: boolean,
-  skipLocales?: boolean,
-  skipContentPublishing?: boolean,
-  uploadAssets?: boolean,
-  assetsDirectory?: string,
-  host?: string,
-  proxy?: string,
-  rawProxy?: string,
-  rateLimit?: number,
-  headers?: object,
-  errorLogFile?: string,
-  useVerboseRenderer?: boolean,
-  // TODO These properties are not documented in the Readme
-  timeout?: number,
-  retryLimit?: number,
-  config?: string,
+    spaceId: string,
+    environmentId?: string,
+    managementToken: string,
+    contentFile?: string,
+    content?: object,
+    contentModelOnly?: boolean,
+    skipContentModel?: boolean,
+    skipLocales?: boolean,
+    skipContentPublishing?: boolean,
+    uploadAssets?: boolean,
+    assetsDirectory?: string,
+    host?: string,
+    proxy?: string,
+    rawProxy?: string,
+    rateLimit?: number,
+    headers?: object,
+    errorLogFile?: string,
+    useVerboseRenderer?: boolean,
+    // TODO These properties are not documented in the Readme
+    timeout?: number,
+    retryLimit?: number,
+    config?: string,
+    force?: boolean,
 }
 
 async function runContentfulImport (params: RunContentfulImportParams) {
@@ -128,8 +132,12 @@ async function runContentfulImport (params: RunContentfulImportParams) {
     {
       title: 'Apply transformations to source data',
       task: wrapTask(async (ctx) => {
-        const transformedSourceData = transformSpace(ctx.sourceDataUntransformed, ctx.destinationData)
-        ctx.sourceData = transformedSourceData
+        const customTransformers: Partial<typeof transformers> = {}
+        if (options.force) {
+          customTransformers.contentTypes = (contentType: ContentTypeProps) => pipe(transformers.contentTypes, forceDeleteOmittedFieldTransform)(contentType)
+        }
+
+        ctx.sourceData = transformSpace(ctx.sourceDataUntransformed, ctx.destinationData, customTransformers)
       })
     },
     {

lib/transform/force-delete-omitted-field-transform.ts

@@ -0,0 +1,9 @@
+import { ContentTypeProps } from 'contentful-management'
+
+export function forceDeleteOmittedFieldTransform (contentType: ContentTypeProps) {
+  const omittedFields = contentType.fields.filter(field => field.omitted)
+  omittedFields.forEach(field => {
+    contentType.fields = contentType.fields.filter(f => f.id !== field.id)
+  })
+  return contentType
+}

lib/transform/transform-space.ts

@@ -1,6 +1,6 @@
-import { omit, defaults } from 'lodash/object'
+import { omit, defaults } from 'lodash'
 
-import * as defaultTransformers from './transformers'
+import { transformers as defaultTransformers } from './transformers'
 import sortEntries from '../utils/sort-entries'
 import sortLocales from '../utils/sort-locales'
 import { DestinationData, OriginalSourceData, TransformedSourceData } from '../types'
@@ -14,14 +14,16 @@ const spaceEntities = [
  * is a need to transform data when copying it to the destination space
  */
 export default function (
-  sourceData: OriginalSourceData, destinationData: DestinationData, customTransformers?: any, entities = spaceEntities
+  sourceData: OriginalSourceData, destinationData: DestinationData, customTransformers?: Partial<typeof defaultTransformers>, entities = spaceEntities
 ): TransformedSourceData {
   const transformers = defaults(customTransformers, defaultTransformers)
   const baseSpaceData = omit(sourceData, ...entities)
 
   sourceData.locales = sortLocales(sourceData.locales)
   const tagsEnabled = !!destinationData.tags
 
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
   return entities.reduce((transformedSpaceData, type) => {
     // tags don't contain links to other entities, don't need to be sorted
     const sortedEntities = (type === 'tags') ? sourceData[type] : sortEntries(sourceData[type])

lib/transform/transformers.ts

@@ -1,5 +1,6 @@
-import { ContentTypeProps, EntryProps, TagProps, WebhookProps } from 'contentful-management'
-import { find, omit, pick, reduce } from 'lodash'
+import { omit, pick, find, reduce } from 'lodash'
+import { AssetProps, ContentTypeProps, EntryProps, LocaleProps, TagProps, WebhookProps } from 'contentful-management'
+import { MetadataProps } from 'contentful-management/dist/typings/common-types'
 
 /**
  * Default transformer methods for each kind of entity.
@@ -8,19 +9,19 @@ import { find, omit, pick, reduce } from 'lodash'
  * as the whole upload process needs to be followed again.
  */
 
-export function contentTypes (contentType: ContentTypeProps) {
+function contentTypes (contentType: ContentTypeProps) {
   return contentType
 }
 
-export function tags (tag: TagProps) {
+function tags (tag: TagProps) {
   return tag
 }
 
-export function entries (entry: EntryProps, _, tagsEnabled = false) {
+function entries (entry: EntryProps, _, tagsEnabled = false) {
   return removeMetadataTags(entry, tagsEnabled)
 }
 
-export function webhooks (webhook: WebhookProps) {
+function webhooks (webhook: WebhookProps) {
   // Workaround for webhooks with credentials
   if (webhook.httpBasicUsername) {
     delete webhook.httpBasicUsername
@@ -34,17 +35,21 @@ export function webhooks (webhook: WebhookProps) {
   return webhook
 }
 
-export function assets (asset, _, tagsEnabled = false) {
+function assets (asset: AssetProps, _, tagsEnabled = false) {
   const transformedAsset = omit(asset, 'sys')
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
   transformedAsset.sys = pick(asset.sys, 'id')
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
   transformedAsset.fields = pick(asset.fields, 'title', 'description')
   transformedAsset.fields.file = reduce(
     asset.fields.file,
     (newFile, localizedFile, locale) => {
       newFile[locale] = pick(localizedFile, 'contentType', 'fileName')
       if (!localizedFile.uploadFrom) {
         const assetUrl = localizedFile.url || localizedFile.upload
-        newFile[locale].upload = `${/^(http|https):\/\//i.test(assetUrl) ? '' : 'https:'}${assetUrl}`
+        newFile[locale].upload = `${/^(http|https):\/\//i.test(assetUrl!) ? '' : 'https:'}${assetUrl}`
       } else {
         newFile[locale].uploadFrom = localizedFile.uploadFrom
       }
@@ -55,7 +60,7 @@ export function assets (asset, _, tagsEnabled = false) {
   return removeMetadataTags(transformedAsset, tagsEnabled)
 }
 
-export function locales (locale, destinationLocales) {
+function locales (locale: LocaleProps, destinationLocales: Array<LocaleProps>): LocaleProps {
   const transformedLocale = pick(locale, 'code', 'name', 'contentManagementApi', 'contentDeliveryApi', 'fallbackCode', 'optional')
   const destinationLocale = find(destinationLocales, { code: locale.code })
   if (destinationLocale) {
@@ -66,12 +71,23 @@ export function locales (locale, destinationLocales) {
     transformedLocale.sys = pick(destinationLocale.sys, 'id')
   }
 
+  // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+  // @ts-ignore
   return transformedLocale
 }
 
-function removeMetadataTags (entity, tagsEnabled = false) {
+function removeMetadataTags<T extends { metadata?: MetadataProps }> (entity: T, tagsEnabled = false): T {
   if (!tagsEnabled) {
     delete entity.metadata
   }
   return entity
 }
+
+export const transformers = {
+  contentTypes,
+  tags,
+  entries,
+  webhooks,
+  assets,
+  locales
+}

lib/usageParams.ts

@@ -84,5 +84,10 @@ export default yargs
     type: 'string',
     describe: 'Pass an additional HTTP Header'
   })
+  .option('force', {
+    describe: 'force omitted fields to be deleted before importing',
+    type: 'boolean',
+    default: false
+  })
   .config('config', 'An optional configuration JSON file containing all the options for a single run')
   .argv

test/unit/transform/force-delete-omitted-field-transform.test.ts

@@ -0,0 +1,11 @@
+import { cloneMock } from 'contentful-batch-libs/test/mocks/'
+import { ContentTypeProps } from 'contentful-management'
+import { forceDeleteOmittedFieldTransform } from '../../../lib/transform/force-delete-omitted-field-transform'
+
+test('It should transform content types by dropping omitted fields', () => {
+  const contentTypeMock = cloneMock('contentType') as ContentTypeProps
+  contentTypeMock.fields[0].omitted = true
+  expect(contentTypeMock.fields).toHaveLength(1)
+  const transformedContentTypeMock = forceDeleteOmittedFieldTransform(contentTypeMock)
+  expect(transformedContentTypeMock.fields).toHaveLength(0)
+})

test/unit/transform/transform-space.test.ts

@@ -60,6 +60,8 @@ test('applies transformers to give space data', () => {
 
 test('applies custom transformers to give space data', () => {
   const result = transformSpace(space, destinationSpace, {
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
     entries: () => 'transformed'
   })
   expect(result.entries?.[0]?.transformed).toBe('transformed')

test/unit/transform/transformers.test.ts

@@ -1,6 +1,6 @@
 import { cloneMock } from 'contentful-batch-libs/test/mocks/'
 
-import * as transformers from '../../../lib/transform/transformers'
+import { transformers } from '../../../lib/transform/transformers'
 
 const _ = {}
 
@@ -60,8 +60,8 @@ test('It should transform unprocessed asset with uploadFrom', () => {
   const transformedAsset = transformers.assets(assetMock, _)
   expect(transformedAsset.fields.file['en-US'].uploadFrom).toBeTruthy()
   expect(transformedAsset.fields.file['de-DE'].uploadFrom).toBeTruthy()
-  expect(transformedAsset.fields.file['en-US'].uploadFrom.sys.id).toBe(assetMock.fields.file['en-US'].uploadFrom.sys.id)
-  expect(transformedAsset.fields.file['de-DE'].uploadFrom.sys.id).toBe(assetMock.fields.file['de-DE'].uploadFrom.sys.id)
+  expect(transformedAsset.fields.file['en-US'].uploadFrom?.sys.id).toBe(assetMock.fields.file['en-US'].uploadFrom.sys.id)
+  expect(transformedAsset.fields.file['de-DE'].uploadFrom?.sys.id).toBe(assetMock.fields.file['de-DE'].uploadFrom.sys.id)
 })
 
 test('It should transform webhook with credentials to normal webhook', () => {