-
Notifications
You must be signed in to change notification settings - Fork 11
feat(input_schema): Enable secret objects #515
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
452cf6f
8a7721e
719a596
c31246f
980f516
9e6908a
3e87a9d
d97c264
fc46b61
b8ceb1e
460297e
fabbaa8
516ae21
6e62f98
667fb2e
0107b23
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,6 +3,7 @@ import type { ValidateFunction } from 'ajv'; | |
import { countries } from 'countries-list'; | ||
|
||
import { PROXY_URL_REGEX, URL_REGEX } from '@apify/consts'; | ||
import { isEncryptedValueForFieldSchema, isEncryptedValueForFieldType } from '@apify/input_secrets'; | ||
|
||
import { parseAjvError } from './input_schema'; | ||
import { m } from './intl'; | ||
|
@@ -133,13 +134,34 @@ export function validateInputUsingValidator( | |
// Process AJV validation errors | ||
if (!isValid) { | ||
errors = validator.errors! | ||
.filter((error) => { | ||
// We are storing encrypted objects/arrays as strings, so AJV will throw type the error here. | ||
// So we need to skip these errors. | ||
if (error.keyword === 'type' && error.instancePath) { | ||
const path = error.instancePath.replace(/^\//, '').split('/')[0]; | ||
const propSchema = inputSchema.properties?.[path]; | ||
const value = input[path]; | ||
|
||
// Check if the property is a secret and if the value is an encrypted value. | ||
// We do additional validation of the field schema in the later part of this function | ||
if ( | ||
propSchema?.isSecret | ||
&& typeof value === 'string' | ||
&& (propSchema.type === 'object' || propSchema.type === 'array') | ||
&& isEncryptedValueForFieldType(value, propSchema.type) | ||
) { | ||
return false; | ||
} | ||
} | ||
return true; | ||
}) | ||
Comment on lines
136
to
+157
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This ignores type errors produced by |
||
.map((error) => parseAjvError(error, 'input', properties, input)) | ||
.filter((error) => !!error) as any[]; | ||
} | ||
|
||
Object.keys(properties).forEach((property) => { | ||
const value = input[property]; | ||
const { type, editor, patternKey, patternValue } = properties[property]; | ||
const { type, editor, patternKey, patternValue, isSecret } = properties[property]; | ||
const fieldErrors = []; | ||
// Check that proxy is required, if yes, valides that it's correctly setup | ||
if (type === 'object' && editor === 'proxy') { | ||
|
@@ -215,7 +237,7 @@ export function validateInputUsingValidator( | |
} | ||
} | ||
// Check that object items fit patternKey and patternValue | ||
if (type === 'object' && value) { | ||
if (type === 'object' && value && typeof value === 'object') { | ||
if (patternKey) { | ||
const check = new RegExp(patternKey); | ||
const invalidKeys: any[] = []; | ||
|
@@ -249,6 +271,16 @@ export function validateInputUsingValidator( | |
} | ||
} | ||
|
||
// Additional validation for secret fields | ||
if (isSecret && value && typeof value === 'string') { | ||
// If the value is a valid encrypted string for the field type, | ||
// we check if the field schema is likely to be still valid (is unchanged from the time of encryption). | ||
if (isEncryptedValueForFieldType(value, type) && !isEncryptedValueForFieldSchema(value, properties[property])) { | ||
// If not, we add an error message to the field errors and user needs to update the value in the input editor. | ||
fieldErrors.push(m('inputSchema.validation.secretFieldSchemaChanged', { rootName: 'input', fieldKey: property })); | ||
MFori marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
} | ||
|
||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As we store field schema's hash to encrypted value, here we validate that the field schema (ignoring properties like title, description,..) doesn't change from the time of encryption. |
||
if (fieldErrors.length > 0) { | ||
const message = fieldErrors.join(', '); | ||
errors.push({ fieldKey: property, message }); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
import crypto from 'node:crypto'; | ||
|
||
/** | ||
* These keys are omitted from the field schema normalization process | ||
* because they are not relevant for validation of values against the schema. | ||
*/ | ||
const OMIT_KEYS = new Set(['title', 'description', 'sectionCaption', 'sectionDescription', 'nullable', 'example', 'editor']); | ||
MFori marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
/** | ||
* Normalizes the field schema by removing irrelevant keys and sorting the remaining keys. | ||
*/ | ||
function normalizeFieldSchema(value: any): any { | ||
if (Array.isArray(value)) { | ||
return value.map(normalizeFieldSchema); | ||
} | ||
|
||
if (value && typeof value === 'object') { | ||
const result: Record<string, any> = {}; | ||
Object.keys(value) | ||
.filter((key) => !OMIT_KEYS.has(key)) | ||
.sort() | ||
.forEach((key) => { | ||
result[key] = normalizeFieldSchema(value[key]); | ||
}); | ||
return result; | ||
} | ||
|
||
return value; | ||
} | ||
|
||
/** | ||
* Generates a stable hash for the field schema. | ||
* @param fieldSchema | ||
*/ | ||
export function getFieldSchemaHash(fieldSchema: Record<string, any>): string { | ||
try { | ||
const stringifiedSchema = JSON.stringify(normalizeFieldSchema(fieldSchema)); | ||
return crypto.createHash('sha256').update(stringifiedSchema).digest('base64'); | ||
MFori marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} catch (err) { | ||
throw new Error(`The field schema could not be stringified for hash: ${err}`); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We can now validate these for
secret
strings too.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm not 100% sure here - wouldn't we end up applying the validation to the encrypted value in this case? (That would not be useful)
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Good catch! I removed these three properties for now. We can add it later but would need to catch the Ajv errors