Skip to content

[feat]: automatically upload adapter on start if the upload is missing for some reason #2902

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 12, 2024
Merged

[feat]: automatically upload adapter on start if the upload is missing for some reason #2902

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5751527
automatically upload adapter on start if the upload is missing for so…
foxriver76 Sep 9, 2024
cf7bd77
merge
foxriver76 Sep 10, 2024
ef9b64f
Merge branch 'master' into fix-upload-problem
foxriver76 Sep 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
* (@foxriver76) enhanced translations for the `diskSpaceIssues` notification category
* (@foxriver76) extend the time to wait until controller is stopped on controller UI upgrade
* (@foxriver76) improved backup/restore process to work for arbitrary large installations
* (@GermanBluefox/@foxriver76) implemented automatic upload on adapter start if version mismatch is detected

## 6.0.11 (2024-08-21) - Kiera
* (foxriver76) only generate `packageUpdates` notification, if new updates detected
Expand Down
37 changes: 6 additions & 31 deletions packages/cli/src/lib/setup/setupInstall.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -814,14 +814,14 @@ export class Install {
let obj;
let err;
try {
obj = await this.objects.getObjectAsync(`system.adapter.${adapter}`);
obj = await this.objects.getObject(`system.adapter.${adapter}`);
} catch (_err) {
err = _err;
}
// Adapter is not installed - install it now
if (err || !obj || !obj.common.installedVersion) {
await this.installAdapter(adapter);
obj = await this.objects.getObjectAsync(`system.adapter.${adapter}`);
obj = await this.objects.getObject(`system.adapter.${adapter}`);
}

if (!obj) {
Expand All @@ -837,7 +837,7 @@ export class Install {
startkey: `${SYSTEM_ADAPTER_PREFIX}${adapter}.`,
endkey: `${SYSTEM_ADAPTER_PREFIX}${adapter}.\u9999`
});
const systemConfig = await this.objects.getObjectAsync('system.config');
const systemConfig = await this.objects.getObject('system.config');
const defaultLogLevel = systemConfig?.common?.defaultLogLevel;
if (!res) {
console.error(`host.${hostname} error: view instanceStats`);
Expand Down Expand Up @@ -1845,34 +1845,9 @@ export class Install {
return;
}

const { installDir } = res;

if (name) {
await this.upload.uploadAdapter(name, true, true);
await this.upload.uploadAdapter(name, false, true);
await this.upload.upgradeAdapterObjects(name);
} else {
// Try to find io-package.json with the newest date
const dirs = fs.readdirSync(installDir);
let date = null;
let dir = null;
for (const _dir of dirs) {
if (fs.existsSync(`${installDir}/${_dir}/io-package.json`)) {
const stat = fs.statSync(`${installDir}/${_dir}/io-package.json`);
if (!date || stat.mtime.getTime() > date.getTime()) {
dir = _dir;
date = stat.mtime;
}
}
}
// if modify time is not older than one hour
if (dir && date && Date.now() - date.getTime() < 3600000) {
name = dir.substring(tools.appName.length + 1);
await this.upload.uploadAdapter(name, true, true);
await this.upload.uploadAdapter(name, false, true);
await this.upload.upgradeAdapterObjects(name);
}
}
await this.upload.uploadAdapter(name, true, true);
await this.upload.uploadAdapter(name, false, true);
await this.upload.upgradeAdapterObjects(name);

// re-enable stopped instances
await this.enableInstances(stoppedList, true);
Expand Down
17 changes: 10 additions & 7 deletions packages/cli/src/lib/setup/setupUpload.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ interface Logger extends InternalLogger {
log(message: string): void;
}

/** Logger without noisy levels */
type MinimalLogger = Omit<Logger, 'info' | 'silly' | 'debug'>;

export class Upload {
private readonly states: StatesRedisClient;
private readonly objects: ObjectsRedisClient;
Expand Down Expand Up @@ -324,7 +327,7 @@ export class Upload {
return `${adapter}/${target}`;
}

async eraseFiles(files: any[], logger: Logger | typeof console): Promise<void> {
async eraseFiles(files: any[], logger: MinimalLogger | typeof console): Promise<void> {
if (files && files.length) {
for (const file of files) {
try {
Expand All @@ -346,7 +349,7 @@ export class Upload {
async collectExistingFilesToDelete(
adapter: string,
path: string,
logger: Logger | typeof console
logger: MinimalLogger | typeof console
): Promise<{ filesToDelete: File[]; dirs: File[] }> {
let _files: File[] = [];
let _dirs: File[] = [];
Expand Down Expand Up @@ -392,11 +395,11 @@ export class Upload {
isAdmin: boolean,
files: string[],
id: string,
logger: Logger | typeof console
logger: MinimalLogger | typeof console
): Promise<string> {
const uploadID = `system.adapter.${adapter}.upload`;

await this.states.setStateAsync(uploadID, { val: 0, ack: true });
await this.states.setState(uploadID, { val: 0, ack: true });

for (let f = 0; f < files.length; f++) {
const file = files[f];
Expand Down Expand Up @@ -494,7 +497,7 @@ export class Upload {
isAdmin: boolean,
forceUpload: boolean,
subTree?: string,
_logger?: Logger
_logger?: MinimalLogger
): Promise<string> {
const id = adapter + (isAdmin ? '.admin' : '');
const adapterDir = tools.getAdapterDir(adapter);
Expand Down Expand Up @@ -708,7 +711,7 @@ export class Upload {
name: string,
ioPack: ioBroker.AdapterObject,
hostname: string,
logger: Logger | typeof console
logger: MinimalLogger | typeof console
): Promise<string> {
// Update all instances of this host
const res = await this.objects.getObjectViewAsync('system', 'instance', {
Expand Down Expand Up @@ -805,7 +808,7 @@ export class Upload {
async upgradeAdapterObjects(
name: string,
ioPack?: ioBroker.AdapterObject,
logger: Logger | typeof console = console
logger: MinimalLogger | typeof console = console
): Promise<string> {
const adapterDir = tools.getAdapterDir(name);
let ioPackFile;
Expand Down
123 changes: 68 additions & 55 deletions packages/controller/src/main.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ interface GetLogFilesResult {
}
interface UploadTask {
adapter: string;
msg: ioBroker.SendableMessage;
msg?: ioBroker.SendableMessage;
}

interface RebuildArgs {
Expand Down Expand Up @@ -224,8 +224,6 @@ let diskWarningLevel = DEFAULT_DISK_WARNING_LEVEL;
let updateIPsTimer: NodeJS.Timeout | null = null;
let lastDiagSend: null | number = null;

const uploadTasks: UploadTask[] = [];

const config = getConfig();

/**
Expand Down Expand Up @@ -1924,23 +1922,21 @@ async function getVersionFromHost(hostId: ioBroker.ObjectIDs.Host): Promise<Host
}

/**
* Upload all adapters which are currently in `uploadTasks` queue
* Upload given adapter
*
* @param task The upload task information containing name and an optional message
*/
async function startAdapterUpload(): Promise<void> {
if (!uploadTasks.length) {
return;
}

async function uploadAdapter(task: UploadTask): Promise<void> {
if (!upload) {
upload = new Upload({
states: states!,
objects: objects!
});
}

const msg = uploadTasks[0].msg;
const msg = task.msg;

const logger = msg.from
const logger = msg?.from
? {
log: (text: string) =>
// @ts-expect-error formally text is not allowed in Message, why not wrapped in message payload property?
Expand All @@ -1954,18 +1950,13 @@ async function startAdapterUpload(): Promise<void> {
}
: undefined;

// @ts-expect-error yes the logger is missing some levels
await upload.uploadAdapter(uploadTasks[0].adapter, true, true, '', logger);
// @ts-expect-error the logger is missing some levels
await upload.upgradeAdapterObjects(uploadTasks[0].adapter, undefined, logger);
// @ts-expect-error yes the logger is missing some levels
await upload.uploadAdapter(uploadTasks[0].adapter, false, true, '', logger);
await upload.uploadAdapter(task.adapter, true, true, '', logger);
await upload.upgradeAdapterObjects(task.adapter, undefined, logger);
await upload.uploadAdapter(task.adapter, false, true, '', logger);
// send response to requester
msg.callback && msg.from && sendTo(msg.from, msg.command, { result: 'done' }, msg.callback);

uploadTasks.shift();

setImmediate(startAdapterUpload);
if (msg?.callback && msg.from) {
sendTo(msg.from, msg.command, { result: 'done' }, msg.callback);
}
}

/**
Expand Down Expand Up @@ -2803,9 +2794,7 @@ async function processMessage(msg: ioBroker.SendableMessage): Promise<null | voi

case 'upload': {
if (msg.message) {
uploadTasks.push({ adapter: msg.message, msg });
// start upload if no tasks running
uploadTasks.length === 1 && startAdapterUpload();
uploadAdapter({ adapter: msg.message, msg });
} else {
logger.error(`${hostLogPrefix} No adapter name is specified for upload command from ${msg.from}`);
}
Expand Down Expand Up @@ -3797,26 +3786,8 @@ async function startInstance(id: ioBroker.ObjectIDs.Instance, wakeUp = false): P
}
}

const isBlocked = await blocklistManager.isAdapterVersionBlocked({
version: instance.common.version,
adapterName: instance.common.name
});

if (isBlocked) {
const message = `Do not start instance "${id}", because the version "${instance.common.version}" has been blocked by the developer`;
logger.error(`${hostLogPrefix} ${message}`);

await notificationHandler.addMessage({
scope: 'system',
category: 'blockedVersions',
message,
instance: SYSTEM_HOST_PREFIX + hostname
});
return;
}

const adapterDir = tools.getAdapterDir(name);
if (!fs.existsSync(adapterDir!)) {
if (adapterDir === null || !fs.existsSync(adapterDir)) {
proc.downloadRetry = proc.downloadRetry || 0;
logger.debug(`${hostLogPrefix} startInstance Queue ${id} for installation`);
installQueue.push({
Expand Down Expand Up @@ -3859,6 +3830,50 @@ async function startInstance(id: ioBroker.ObjectIDs.Instance, wakeUp = false): P
}
}

try {
// check if the io-package content is uploaded to the database
const ioPack = fs.readJSONSync(path.join(adapterDir, 'io-package.json'));

if (ioPack.common.version !== instance.common.version) {
logger.warn(`${hostLogPrefix} Detected missing upload of adapter "${name}" - starting upload now.`);
await uploadAdapter({ adapter: name });
return;
}
} catch (e) {
logger.error(
`${hostLogPrefix} startInstance ${name}.${instanceNo}: Error while ensuring adapter is uploaded: ${e.message}`
);
}

const isBlocked = await blocklistManager.isAdapterVersionBlocked({
version: instance.common.version,
adapterName: instance.common.name
});

if (isBlocked) {
const message = `Do not start instance "${id}", because the version "${instance.common.version}" has been blocked by the developer`;
logger.error(`${hostLogPrefix} ${message}`);

await notificationHandler.addMessage({
scope: 'system',
category: 'blockedVersions',
message,
instance: SYSTEM_HOST_PREFIX + hostname
});
return;
}

// Check if all required adapters installed and have a valid version
if (instance.common.dependencies || instance.common.globalDependencies) {
try {
await checkVersions(id, instance.common.dependencies, instance.common.globalDependencies);
} catch (e) {
logger.error(`${hostLogPrefix} startInstance ${id} ${e.message}`);
// Do not start this instance
return;
}
}

// workaround for old vis
if (instance.common.onlyWWW && name === 'vis') {
instance.common.onlyWWW = false;
Expand Down Expand Up @@ -3886,7 +3901,7 @@ async function startInstance(id: ioBroker.ObjectIDs.Instance, wakeUp = false): P
// read node.js engine requirements
try {
// read directly from disk and not via require to allow "on the fly" updates of adapters.
const packJSON = fs.readJSONSync(`${adapterDir}/package.json`);
const packJSON = fs.readJSONSync(path.join(adapterDir, 'package.json'));
proc.engine = packJSON?.engines?.node;
} catch {
logger.error(
Expand All @@ -3901,16 +3916,14 @@ async function startInstance(id: ioBroker.ObjectIDs.Instance, wakeUp = false): P
`${hostLogPrefix} startInstance ${name}.${instanceNo}: required Node.js version ${proc.engine}, actual version ${process.version}`
);
// disable instance
objects!.getObject(id, (err, obj) => {
if (obj && obj.common && obj.common.enabled) {
obj.common.enabled = false;
objects!.setObject(obj._id, obj, _err =>
logger.warn(
`${hostLogPrefix} startInstance ${name}.${instanceNo}: instance disabled because of Node.js version mismatch`
)
);
}
});
const obj = await objects!.getObject(id);
if (obj?.common?.enabled) {
obj.common.enabled = false;
await objects!.setObject(obj._id, obj);
logger.warn(
`${hostLogPrefix} startInstance ${name}.${instanceNo}: instance disabled because of Node.js version mismatch`
);
}
return;
}
}
Expand Down
Loading