-
Notifications
You must be signed in to change notification settings - Fork 11
feat(input_schema): Enable secret objects #515
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
452cf6f
8a7721e
719a596
c31246f
980f516
9e6908a
3e87a9d
d97c264
fc46b61
b8ceb1e
460297e
fabbaa8
516ae21
6e62f98
667fb2e
0107b23
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,6 +3,7 @@ import type { ValidateFunction } from 'ajv'; | |
import { countries } from 'countries-list'; | ||
|
||
import { PROXY_URL_REGEX, URL_REGEX } from '@apify/consts'; | ||
import { isEncryptedValueForFieldSchema, isEncryptedValueForFieldType } from '@apify/input_secrets'; | ||
|
||
import { parseAjvError } from './input_schema'; | ||
import { m } from './intl'; | ||
|
@@ -133,13 +134,34 @@ export function validateInputUsingValidator( | |
// Process AJV validation errors | ||
if (!isValid) { | ||
errors = validator.errors! | ||
.filter((error) => { | ||
// We are storing encrypted objects/arrays as strings, so AJV will throw type the error here. | ||
// So we need to skip these errors. | ||
if (error.keyword === 'type' && error.instancePath) { | ||
const path = error.instancePath.replace(/^\//, '').split('/')[0]; | ||
const propSchema = inputSchema.properties?.[path]; | ||
const value = input[path]; | ||
|
||
// Check if the property is a secret and if the value is an encrypted value. | ||
// We do additional validation of the field schema in the later part of this function | ||
if ( | ||
propSchema?.isSecret | ||
&& typeof value === 'string' | ||
&& (propSchema.type === 'object' || propSchema.type === 'array') | ||
&& isEncryptedValueForFieldType(value, propSchema.type) | ||
) { | ||
return false; | ||
} | ||
} | ||
return true; | ||
}) | ||
.map((error) => parseAjvError(error, 'input', properties, input)) | ||
.filter((error) => !!error) as any[]; | ||
} | ||
|
||
Object.keys(properties).forEach((property) => { | ||
const value = input[property]; | ||
const { type, editor, patternKey, patternValue } = properties[property]; | ||
const { type, editor, patternKey, patternValue, isSecret } = properties[property]; | ||
const fieldErrors = []; | ||
// Check that proxy is required, if yes, valides that it's correctly setup | ||
if (type === 'object' && editor === 'proxy') { | ||
|
@@ -215,7 +237,7 @@ export function validateInputUsingValidator( | |
} | ||
} | ||
// Check that object items fit patternKey and patternValue | ||
if (type === 'object' && value) { | ||
if (type === 'object' && value && typeof value === 'object') { | ||
if (patternKey) { | ||
const check = new RegExp(patternKey); | ||
const invalidKeys: any[] = []; | ||
|
@@ -249,6 +271,16 @@ export function validateInputUsingValidator( | |
} | ||
} | ||
|
||
// Additional validation for secret fields | ||
if (isSecret && value && typeof value === 'string') { | ||
// If the value is a valid encrypted string for the field type, | ||
// we check if the field schema is likely to be still valid (is unchanged from the time of encryption). | ||
if (isEncryptedValueForFieldType(value, type) && !isEncryptedValueForFieldSchema(value, properties[property])) { | ||
// If not, we add an error message to the field errors and user needs to update the value in the input editor. | ||
fieldErrors.push(m('inputSchema.validation.secretFieldSchemaChanged', { fieldKey: property })); | ||
} | ||
Comment on lines
+276
to
+281
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have a slight concern here, but I'm not sure how big of a problem it will really be. If you have e.g. an I'm not sure what changes to the input schema happen more often, but I would wager it's backwards compatible ones, so we'll be forcing people to update their inputs even when not necessary. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hm, maybe showing this only as a warning in the UI would be better? Ajv does not have anything like warning, but we can updated the What do you think? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe we can keep it it as an error for now, see how many of these validation errors are happening, and possibly change it to a warning later? If we put it as a warning now, but decide to change it to an error later, that would be a breaking change. |
||
} | ||
|
||
if (fieldErrors.length > 0) { | ||
const message = fieldErrors.join(', '); | ||
errors.push({ fieldKey: property, message }); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
import crypto from 'node:crypto'; | ||
|
||
/** | ||
* These keys are omitted from the field schema normalization process | ||
* because they are not relevant for validation of values against the schema. | ||
*/ | ||
const OMIT_KEYS = new Set(['title', 'description', 'sectionCaption', 'sectionDescription', 'nullable', 'example', 'prefill', 'editor']); | ||
|
||
/** | ||
* Normalizes the field schema by removing irrelevant keys and sorting the remaining keys. | ||
*/ | ||
function normalizeFieldSchema(value: any): any { | ||
if (Array.isArray(value)) { | ||
return value.map(normalizeFieldSchema); | ||
} | ||
|
||
if (value && typeof value === 'object') { | ||
const result: Record<string, any> = {}; | ||
Object.keys(value) | ||
.filter((key) => !OMIT_KEYS.has(key)) | ||
.sort() | ||
.forEach((key) => { | ||
result[key] = normalizeFieldSchema(value[key]); | ||
}); | ||
return result; | ||
} | ||
|
||
return value; | ||
} | ||
|
||
/** | ||
* Generates a stable hash for the field schema. | ||
* @param fieldSchema | ||
*/ | ||
export function getFieldSchemaHash(fieldSchema: Record<string, any>): string { | ||
try { | ||
const stringifiedSchema = JSON.stringify(normalizeFieldSchema(fieldSchema)); | ||
// Create a SHA-256 hash of the stringified schema and return the first 10 characters in hex. | ||
return crypto.createHash('sha256').update(stringifiedSchema).digest('hex').slice(0, 10); | ||
} catch (err) { | ||
throw new Error(`The field schema could not be stringified for hash: ${err}`); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This ignores type errors produced by
Ajv
validation when secret string is passed toobject
/array
fields.