DataFusionQuickstart.json

{
  "name": "DataFusionQuickstart",
  "appVersion": "f26b6773-edea-11ef-b9c1-0000003a29ff",
  "description": "Data Pipeline Application",
  "change": {
    "author": "vaibhavsethi",
    "creationTimeMillis": 1739877881718,
    "latest": true
  },
  "sourceControlMeta": {
    "fileHash": "e7942469be7e8a0c953b73ec01e7da1d12b2f296",
    "commitId": "c0719fcd97c6cb5aa6115c524e89f84ef7fa24f5",
    "lastSyncedAt": {
      "seconds": 1739877910,
      "nanos": 229000000
    }
  },
  "configuration": "{\"resources\":{\"memoryMB\":2048.0,\"virtualCores\":1.0},\"driverResources\":{\"memoryMB\":2048.0,\"virtualCores\":1.0},\"connections\":[{\"from\":\"NYT Best Sellers Raw Data\",\"to\":\"Parsing and Transformations\"},{\"from\":\"Parsing and Transformations\",\"to\":\"Top Rated Inexpensive Books\"}],\"comments\":[],\"postActions\":[],\"properties\":{},\"processTimingEnabled\":true,\"stageLoggingEnabled\":true,\"stages\":[{\"name\":\"NYT Best Sellers Raw Data\",\"plugin\":{\"name\":\"GCSFile\",\"type\":\"batchsource\",\"label\":\"NYT Best Sellers Raw Data\",\"artifact\":{\"name\":\"google-cloud\",\"version\":\"0.25.0-SNAPSHOT\",\"scope\":\"SYSTEM\"},\"properties\":{\"filenameOnly\":\"false\",\"copyHeader\":\"false\",\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"etlSchemaBody\\\",\\\"fields\\\":[{\\\"name\\\":\\\"body\\\",\\\"type\\\":\\\"string\\\"}]}\",\"path\":\"gs://nyt-bestsellers/nyt2.json\",\"format\":\"text\",\"recursive\":\"false\",\"referenceName\":\"NYT_BestSellers_Raw\",\"serviceFilePath\":\"auto-detect\",\"project\":\"auto-detect\"}},\"outputSchema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"etlSchemaBody\\\",\\\"fields\\\":[{\\\"name\\\":\\\"body\\\",\\\"type\\\":\\\"string\\\"}]}\",\"id\":\"NYT-Best-Sellers-Raw-Data\"},{\"name\":\"Parsing and Transformations\",\"plugin\":{\"name\":\"Wrangler\",\"type\":\"transform\",\"label\":\"Parsing and Transformations\",\"artifact\":{\"name\":\"wrangler-transform\",\"version\":\"4.12.0-SNAPSHOT\",\"scope\":\"SYSTEM\"},\"properties\":{\"workspaceId\":\"b22b1b0c24b12acd45f834ac5e1dfffb\",\"directives\":\"split-to-rows body \\\\n\\nparse-as-json :body 1\\nparse-as-json :body__id 1\\nparse-as-json :body_weeks_on_list 1\\nparse-as-json :body_rank_last_week 1\\nparse-as-json :body_rank 1\\nparse-as-json :body_price 1\\nparse-as-json :body_published_date 2\\nparse-as-json :body_bestsellers_date 2\\ncolumns-replace s/^body_//g\\ncleanse-column-names\\nrename _id__oid id\\nrename weeks_on_list__numberint weeks_on_list\\nrename rank_last_week__numberint rank_last_week\\nrename rank__numberint rank\\nfill-null-or-empty :price__numberint \\u00270\\u0027\\nrename price__numberint price\\nrename published_date__date__numberlong published_date\\nrename bestsellers_date__date__numberlong bestsellers_date\\nfill-null-or-empty :price__numberdouble \\u00270\\u0027\\nrename price__numberdouble price_other\\nfill-null-or-empty :price \\u00270\\u0027\\nfill-null-or-empty :price_other \\u00270\\u0027\\nset-type :price float\\nset-type :price_other float\\nset-column :price_final price+price_other\\ndrop price\\ndrop price_other\\nset-type :published_date long\\nset-type :bestsellers_date long\\nset-type :weeks_on_list integer\\nset-type :rank_last_week integer\\nset-type :rank long\\nfilter-rows-on regex-not-match rank_last_week ^0$\\ndrop rank_last_week\\ndrop weeks_on_list\\ndrop published_date\\ndrop bestsellers_date\\nfilter-rows-on condition-false rank \\u003c\\u003d3\\nfilter-rows-on regex-match price_final ^0.0$\\nfilter-rows-on condition-false price_final \\u003c25.0\",\"schema\":\"{\\\"name\\\":\\\"etlSchemaBody\\\",\\\"type\\\":\\\"record\\\",\\\"fields\\\":[{\\\"name\\\":\\\"amazon_product_url\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"author\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"description\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"publisher\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"title\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"id\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"rank\\\",\\\"type\\\":[\\\"long\\\",\\\"null\\\"]},{\\\"name\\\":\\\"price_final\\\",\\\"type\\\":[\\\"double\\\",\\\"null\\\"]}]}\",\"field\":\"body\",\"precondition\":\"false\"}},\"outputSchema\":\"{\\\"name\\\":\\\"etlSchemaBody\\\",\\\"type\\\":\\\"record\\\",\\\"fields\\\":[{\\\"name\\\":\\\"amazon_product_url\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"author\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"description\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"publisher\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"title\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"id\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"rank\\\",\\\"type\\\":[\\\"long\\\",\\\"null\\\"]},{\\\"name\\\":\\\"price_final\\\",\\\"type\\\":[\\\"double\\\",\\\"null\\\"]}]}\",\"inputSchema\":[{\"name\":\"NYT Best Sellers Raw Data\",\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"etlSchemaBody\\\",\\\"fields\\\":[{\\\"name\\\":\\\"body\\\",\\\"type\\\":\\\"string\\\"}]}\"}],\"id\":\"Parsing-and-Transformations\"},{\"name\":\"Top Rated Inexpensive Books\",\"plugin\":{\"name\":\"BigQueryTable\",\"type\":\"batchsink\",\"label\":\"Top Rated Inexpensive Books\",\"artifact\":{\"name\":\"google-cloud\",\"version\":\"0.25.0-SNAPSHOT\",\"scope\":\"SYSTEM\"},\"properties\":{\"project\":\"auto-detect\",\"serviceFilePath\":\"auto-detect\",\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"etlSchemaBody\\\",\\\"fields\\\":[{\\\"name\\\":\\\"amazon_product_url\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"author\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"description\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"publisher\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"title\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"id\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"rank\\\",\\\"type\\\":[\\\"long\\\",\\\"null\\\"]},{\\\"name\\\":\\\"price_final\\\",\\\"type\\\":[\\\"double\\\",\\\"null\\\"]}]}\",\"referenceName\":\"Top_Rated_Inexpensive_Books\",\"dataset\":\"GCPQuickStart\",\"allowSchemaRelaxation\":\"false\",\"table\":\"top_rated_inexpensive\"}},\"outputSchema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"etlSchemaBody\\\",\\\"fields\\\":[{\\\"name\\\":\\\"amazon_product_url\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"author\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"description\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"publisher\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"title\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"id\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"rank\\\",\\\"type\\\":[\\\"long\\\",\\\"null\\\"]},{\\\"name\\\":\\\"price_final\\\",\\\"type\\\":[\\\"double\\\",\\\"null\\\"]}]}\",\"inputSchema\":[{\"name\":\"Parsing and Transformations\",\"schema\":\"{\\\"type\\\":\\\"record\\\",\\\"name\\\":\\\"etlSchemaBody\\\",\\\"fields\\\":[{\\\"name\\\":\\\"amazon_product_url\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"author\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"description\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"publisher\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"title\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"id\\\",\\\"type\\\":[\\\"string\\\",\\\"null\\\"]},{\\\"name\\\":\\\"rank\\\",\\\"type\\\":[\\\"long\\\",\\\"null\\\"]},{\\\"name\\\":\\\"price_final\\\",\\\"type\\\":[\\\"double\\\",\\\"null\\\"]}]}\"}],\"id\":\"Top-Rated-Inexpensive-Books\"}],\"schedule\":\"0 * * * *\",\"engine\":\"spark\",\"numOfRecordsPreview\":100.0,\"rangeRecordsPreview\":{\"min\":1.0,\"max\":\"5000\"},\"maxConcurrentRuns\":1.0}",
  "datasets": [],
  "programs": [
    {
      "type": "Spark",
      "app": "DataFusionQuickstart",
      "name": "phase-1",
      "description": "Sources \u0027NYT Best Sellers Raw Data\u0027 to sinks \u0027Top Rated Inexpensive Books\u0027."
    },
    {
      "type": "Workflow",
      "app": "DataFusionQuickstart",
      "name": "DataPipelineWorkflow",
      "description": "Data Pipeline Workflow"
    }
  ],
  "plugins": [
    {
      "id": "Top Rated Inexpensive Books",
      "name": "BigQueryTable",
      "type": "batchsink"
    },
    {
      "id": "NYT Best Sellers Raw Data",
      "name": "GCSFile",
      "type": "batchsource"
    },
    {
      "id": "Parsing and Transformations",
      "name": "Wrangler",
      "type": "transform"
    },
    {
      "id": "NYT Best Sellers Raw Data:text",
      "name": "text",
      "type": "validatingInputFormat"
    }
  ],
  "artifact": {
    "name": "cdap-data-pipeline",
    "version": "6.12.0-SNAPSHOT",
    "scope": "SYSTEM"
  }
}