Skip to content

Commit ea4beb1

Browse files
nathan-mellisnathan-mellis
authored
Merge pull request #2 from modzy/improving-samples
Improving samples
2 parents a2c1c7b + 39057a7 commit ea4beb1

File tree

6 files changed

+377
-132
lines changed

6 files changed

+377
-132
lines changed

‎samples/config.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"languages": ["eng", "deu", "fra"]
3+
}

‎samples/job_with_aws_input_sample.js

Lines changed: 104 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,122 @@
11
const logger = require('log4js').getLogger("modzy");
22
const modzy = require('modzy-sdk');
3-
logger.level = "all";
3+
4+
// Always configure the logger level (ie: all, trace, debug, info, warn, error, fatal)
5+
logger.level = "info";
6+
7+
// The system admin can provide the right base API URL, the API key can be downloaded from your profile page on Modzy.
8+
// You can config those params as is described in the readme file (as environment variables, or by using the .env file), or you
9+
// or you can just update the BASE_URL and API_KEY vars and use this sample code (not recommended for production environments).
10+
// The MODZY_BASE_URL should point to the API services route which may be different from the Modzy page URL.
11+
// (ie: https://modzy.example.com/api).
12+
const BASE_URL = process.env.MODZY_BASE_URL;
13+
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character and a private part
14+
// (ie: AzQBJ3h4B1z60xNmhAJF.uQyQh8putLIRDi1nOldh).
15+
const API_KEY = process.env.MODZY_API_KEY;
416

517
// Client initialization
6-
// TODO: set the base url of modzy api and you api key
7-
const modzyClient = new modzy.ModzyClient(MODZY_BASE_URL, MODZY_API_KEY);
18+
// Initialize the ApiClient instance with the BASE_URL and the API_KEY to store those arguments
19+
// for the following API calls.
20+
const modzyClient = new modzy.ModzyClient(BASE_URL, API_KEY);
821

9-
// Create a Job with a text input
22+
// Create a Job with an AWS input, wait, and retrieve results:
1023

1124
async function createJobWithAWSInput(){
1225
try{
26+
// Get the model object:
27+
// If you already know the model identifier (i.e.: you got from the URL of the model details page or the input sample),
28+
// you can skip this step. If you don't you can find the model identifier by using its name as follows:
1329
let model = await modzyClient.getModelByName("Facial Embedding");
14-
logger.info("Model: "+JSON.stringify(model));
30+
// Or if you already know the model id and want to know more about the model, you can use this instead:
31+
//let model = await modzyClient.getModel("f7e252e26a");
32+
33+
// The model identifier is under the modelId key. You can take a look at the other keys by uncommenting the following line
34+
logger.info(Object.keys(model).toString().replace('\n', ' '));
35+
// Or just log the model identifier and the latest version
36+
logger.info(`The model identifier is ${model.modelId} and the latest version is ${model.latestVersion}`);
37+
// Get the model version object:
38+
// If you already know the model version and the input key(s) of the model version you can skip this step. Also, you can
39+
// use the following code block to know about the inputs keys and skip the call on future job submissions.
1540
let modelVersion = await modzyClient.getModelVersion(model.modelId, model.latestVersion);
16-
logger.info("ModelVersion: "+JSON.stringify(modelVersion));
41+
// The info stored in modelVersion provides insights about the amount of time that the model can spend processing, the inputs, and
42+
// output keys of the model.
43+
logger.info(`Ths model version is ${modelVersion.version}`);
44+
logger.info(` timeouts: status ${modelVersion.timeout.status}ms, run ${modelVersion.timeout.run}ms `);
45+
logger.info(" inputs: ");
46+
for(key in modelVersion.inputs){
47+
let input = modelVersion.inputs[key];
48+
logger.info(` key ${input.name}, type ${input.acceptedMediaTypes}, description: ${input.description}`);
49+
}
50+
logger.info(" outputs: ")
51+
for(key in modelVersion.outputs){
52+
let output = modelVersion.outputs[key];
53+
logger.info(` key ${output.name}, type ${output.mediaType}, description: ${output.description}`);
54+
}
55+
// Send the job:
56+
// Amazon Simple Storage Service (AWS S3) is an object storage service (for more info visit: https://aws.amazon.com/s3/?nc1=h_ls).
57+
// It allows to store images, videos, or other content as files. In order to use as input type, provide the following properties:
58+
// AWS Access Key: replace <<AccessKey>>
59+
const ACCESS_KEY="<<AccessKey>>";
60+
// AWS Secret Access Key: replace <<SecretAccessKey>>
61+
const SECRET_ACCESS_KEY="<<SecretAccessKey>>";
62+
// AWS Default Region : replace <<AWSRegion>>
63+
const REGION="<<AWSRegion>>";
64+
// The Bucket Name: replace <<BucketName>>
65+
const BUCKET_NAME="<<BucketName>>";
66+
// The File Key: replace <<FileId>> (remember, this model needs an image as input)
67+
const FILE_KEY="<<FileId>>";
68+
// With the info about the model (identifier), the model version (version string, input/output keys), you are ready to
69+
// submit the job. Just prepare the source object:
70+
let sources = {"source-key": {"image": {'bucket': BUCKET_NAME, 'key': FILE_KEY}}};
71+
// An inference job groups input data that you send to a model. You can send any amount of inputs to
72+
// process and you can identify and refer to a specific input by the key that you assign, for example we can add:
73+
sources["second-key"] = {"image": {'bucket': BUCKET_NAME, 'key': FILE_KEY}};
74+
sources["another-key"] = {"image": {'bucket': BUCKET_NAME, 'key': FILE_KEY}};
75+
// If you send a wrong input key, the model fails to process the input.
76+
sources["wrong-key"] = {"a.wrong.key": {'bucket': BUCKET_NAME, 'key': FILE_KEY}};
77+
// If you send a correct input key, but a wrong AWS S3 value key, the model fails to process the input.
78+
sources["wrong-value"] = {"image": {'bucket': BUCKET_NAME, 'key': "wrong-aws-file-key.png"}};
79+
// When you have all your inputs ready, you can use our helper method to submit the job as follows:
1780
let job = await modzyClient.submitJobAWSS3(
1881
model.modelId,
1982
modelVersion.version,
20-
"MyAccessKeyID", //TODO: Replace with your aws s3 access key id
21-
"MySecretAccessKey", //TODO: Replace with your aws s3 access key secret
22-
"TheRegion", //TODO: Replace with your aws s3 region
23-
{
24-
'input-1': {
25-
'input.txt': {
26-
'bucket': 'the-aws-bucket', //TODO: Replace with your aws s3 bucket
27-
'key': '/path/to/the/file/image.jpeg' //TODO: Replace with your aws s3 file key
28-
}
29-
}
30-
}
31-
); // Facial Embedding
83+
ACCESS_KEY,
84+
SECRET_ACCESS_KEY,
85+
REGION,
86+
sources
87+
);
88+
// Modzy creates the job and queue for processing. The job object contains all the info that you need to keep track
89+
// of the process, the most important being the job identifier and the job status.
3290
logger.info("job: "+job.jobIdentifier+" "+job.status);
91+
// The job moves to SUBMITTED, meaning that Modzy acknowledged the job and sent it to the queue to be processed.
92+
// We provide a helper method to listen until the job finishes processing. it will listen until the job finishes
93+
// and moves to COMPLETED, CANCELED, or TIMEOUT.
3394
job = await modzyClient.blockUntilComplete(job);
34-
logger.info("job: "+job.jobIdentifier+" "+job.status);
35-
let results = await modzyClient.getResult(job.jobIdentifier);
36-
logger.info("results: "+job.jobIdentifier+" "+JSON.stringify(results.results));
95+
// Get the results:
96+
// Check the status of the job. Jobs may be canceled or may reach a timeout.
97+
if( job.status === "COMPLETED" ){
98+
// A completed job means that all the inputs were processed by the model. Check the results for each
99+
// input keys provided in the source object to see the model output.
100+
let result = await modzyClient.getResult(job.jobIdentifier);
101+
// The result object has some useful info:
102+
logger.info(`Result: finished: ${result.finished}, total: ${result.total}, completed: ${result.completed}, failed: ${result.failed}`);
103+
// Notice that we are iterating through the same input sources keys
104+
for( key in sources ){
105+
// The result object has the individual results of each job input. In this case the output key is called
106+
// results.json, so we can get the results as follows:
107+
if( result.results[key] ){
108+
let model_res = result.results[key]["results.json"];
109+
// The output for this model comes in a JSON format, so we can directly log the model results:
110+
logger.info(` ${key}: ${JSON.stringify(model_res)}`);
111+
}
112+
else{
113+
logger.warn(` ${key}: failure ${result.failures[key]['error']}`);
114+
}
115+
}
116+
}
117+
else{
118+
log.warn(`The job ends with status ${job.status}`);
119+
}
37120
}
38121
catch(error){
39122
logger.warn("Unexpected error "+error);

‎samples/job_with_embedded_input_sample.js

Lines changed: 101 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,39 +1,114 @@
11
const logger = require('log4js').getLogger("modzy");
22
const modzy = require('modzy-sdk');
33
const fs = require('fs');
4-
logger.level = "all";
4+
5+
// Always configure the logger level (ie: all, trace, debug, info, warn, error, fatal)
6+
logger.level = "info";
7+
8+
// The system admin can provide the right base API URL, the API key can be downloaded from your profile page on Modzy.
9+
// You can config those params as is described in the readme file (as environment variables, or by using the .env file), or you
10+
// or you can just update the BASE_URL and API_KEY vars and use this sample code (not recommended for production environments).
11+
// The MODZY_BASE_URL should point to the API services route which may be different from the Modzy page URL.
12+
// (ie: https://modzy.example.com/api).
13+
const BASE_URL = process.env.MODZY_BASE_URL;
14+
// The MODZY_API_KEY is your own personal API key. It is composed by a public part, a dot character and a private part
15+
// (ie: AzQBJ3h4B1z60xNmhAJF.uQyQh8putLIRDi1nOldh).
16+
const API_KEY = process.env.MODZY_API_KEY;
517

618
// Client initialization
7-
// TODO: set the base url of modzy api and you api key
8-
const modzyClient = new modzy.ModzyClient(process.env.MODZY_BASE_URL, process.env.MODZY_API_KEY);
19+
// Initialize the ApiClient instance with the BASE_URL and the API_KEY to store those arguments
20+
// for the following API calls.
21+
const modzyClient = new modzy.ModzyClient(BASE_URL, API_KEY);
922

10-
// Create a Job with a embedded input
23+
// Create a Job with an embedded input, wait, and retrieve results:
1124

1225
async function createJobWithEmbeddedInput(){
1326
try {
14-
const imageBytes = fs.readFileSync('samples/image.png');
15-
let model = await modzyClient.getModelByName("NSFW Image Detection");
16-
logger.info("Model: " + JSON.stringify(model));
17-
let modelVersion = await modzyClient.getModelVersion(model.modelId, model.latestVersion);
18-
logger.info("ModelVersion: " + JSON.stringify(modelVersion));
19-
let job = await modzyClient.submitJobEmbedded(
20-
model.modelId,
21-
modelVersion.version,
22-
"image/png",
23-
{
24-
'input-1': {
25-
'image': imageBytes
26-
}
27-
}
28-
); //NSFW Image Detection
29-
logger.info("job: " + job.jobIdentifier + " " + job.status);
30-
job = await modzyClient.blockUntilComplete(job);
31-
logger.info("job: " + job.jobIdentifier + " " + job.status);
32-
let results = await modzyClient.getResult(job.jobIdentifier);
33-
logger.info("results: " + job.jobIdentifier + " " + JSON.stringify(results.results));
27+
// Get the model object:
28+
// If you already know the model identifier (i.e.: you got from the URL of the model details page or the input sample),
29+
// you can skip this step. If you don't you can find the model identifier by using its name as follows:
30+
let model = await modzyClient.getModelByName("Multi-Language OCR");
31+
// Or if you already know the model id and want to know more about the model, you can use this instead:
32+
//let model = await modzyClient.getModel("c60c8dbd79");
33+
34+
// The model identifier is under the modelId key. You can take a look at the other keys by uncommenting the following line
35+
logger.info(Object.keys(model).toString().replace('\n', ' '));
36+
// Or just log the model identifier and the latest version
37+
logger.info(`The model identifier is ${model.modelId} and the latest version is ${model.latestVersion}`);
38+
// Get the model version object:
39+
// If you already know the model version and the input key(s) of the model version you can skip this step. Also, you can
40+
// use the following code block to know about the inputs keys and skip the call on future job submissions.
41+
let modelVersion = await modzyClient.getModelVersion(model.modelId, model.latestVersion);
42+
// The info stored in modelVersion provides insights about the amount of time that the model can spend processing, the inputs, and
43+
// output keys of the model.
44+
logger.info(`Ths model version is ${modelVersion.version}`);
45+
logger.info(` timeouts: status ${modelVersion.timeout.status}ms, run ${modelVersion.timeout.run}ms `);
46+
logger.info(" inputs: ");
47+
for(key in modelVersion.inputs){
48+
let input = modelVersion.inputs[key];
49+
logger.info(` key ${input.name}, type ${input.acceptedMediaTypes}, description: ${input.description}`);
50+
}
51+
logger.info(" outputs: ")
52+
for(key in modelVersion.outputs){
53+
let output = modelVersion.outputs[key];
54+
logger.info(` key ${output.name}, type ${output.mediaType}, description: ${output.description}`);
55+
}
56+
57+
// Send the job:
58+
// An embedded input is a byte array encoded as a string in Base64, that's very handy for small to middle size files, for
59+
// bigger files can be a memory issue because you need to load the file in memory (load + encode).
60+
const imageBytes = fs.readFileSync('samples/image.png');
61+
let configBytes = fs.readFileSync('samples/config.json');
62+
// With the info about the model (identifier), the model version (version string, input/output keys), you are ready to
63+
// submit the job. Just prepare the source object:
64+
let sources = {"source-key": {"input": imageBytes, "config.json": configBytes}};
65+
// An inference job groups input data that you send to a model. You can send any amount of inputs to
66+
// process and you can identify and refer to a specific input by the key that you assign, for example we can add:
67+
sources["second-key"] = {"input": imageBytes, "config.json":configBytes}
68+
// You don't need to load all the inputs from files, just convert to bytes as follows:
69+
configBytes = Buffer.from(JSON.stringify({"languages":["spa"]}));
70+
sources["another-key"] = {"input": imageBytes, "config.json":configBytes}
71+
// If you send a wrong input key, the model fails to process the input.
72+
sources["wrong-key"] = {"a.wrong.key": imageBytes, "config.json":configBytes}
73+
// If you send a correct input key, but some wrong values, the model fails to process the input.
74+
sources["wrong-value"] = {"input": configBytes, "config.json":imageBytes}
75+
// When you have all your inputs ready, you can use our helper method to submit the job as follows:
76+
let job = await modzyClient.submitJobEmbedded(model.modelId,modelVersion.version, "application/octet-stream", sources);
77+
// Modzy creates the job and queue for processing. The job object contains all the info that you need to keep track
78+
// of the process, the most important being the job identifier and the job status.
79+
logger.info("job: "+job.jobIdentifier+" "+job.status);
80+
// The job moves to SUBMITTED, meaning that Modzy acknowledged the job and sent it to the queue to be processed.
81+
// We provide a helper method to listen until the job finishes processing. it will listen until the job finishes
82+
// and moves to COMPLETED, CANCELED, or TIMEOUT.
83+
job = await modzyClient.blockUntilComplete(job);
84+
// Get the results:
85+
// Check the status of the job. Jobs may be canceled or may reach a timeout.
86+
if( job.status === "COMPLETED" ){
87+
// A completed job means that all the inputs were processed by the model. Check the results for each
88+
// input keys provided in the source object to see the model output.
89+
let result = await modzyClient.getResult(job.jobIdentifier);
90+
// The result object has some useful info:
91+
logger.info(`Result: finished: ${result.finished}, total: ${result.total}, completed: ${result.completed}, failed: ${result.failed}`);
92+
// Notice that we are iterating through the same input sources keys
93+
for( key in sources ){
94+
// The result object has the individual results of each job input. In this case the output key is called
95+
// results.json, so we can get the results as follows:
96+
if( result.results[key] ){
97+
let model_res = result.results[key]["results.json"];
98+
// The output for this model comes in a JSON format, so we can directly log the model results:
99+
logger.info(` ${key}: ${JSON.stringify(model_res)}`);
100+
}
101+
else{
102+
logger.warn(` ${key}: failure ${result.failures[key]['error']}`);
103+
}
104+
}
105+
}
106+
else{
107+
log.warn(`The job ends with status ${job.status}`);
108+
}
34109
}
35-
catch(error){
36-
logger.warn("Unexpected error "+error);
110+
catch(error){
111+
logger.warn(error);
37112
}
38113
}
39114

0 commit comments

Comments
 (0)