Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: api7 API-level resource type filter #146

Merged
merged 7 commits into from
Jul 16, 2024
Merged

feat: api7 API-level resource type filter #146

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ed9b666
feat: update backend options defination
bzp2010 Jul 15, 2024
5acf3fe
feat: skip fetch request by resource type included or excluded
bzp2010 Jul 15, 2024
513d1f0
test: add is skip cases
bzp2010 Jul 15, 2024
64043d3
chore: add ts path include
bzp2010 Jul 15, 2024
7393234
feat: disable post-resource-type filter for api7
bzp2010 Jul 15, 2024
f65e987
ci: add backend-api7 unit test
bzp2010 Jul 15, 2024
1b000fe
ci: test api7 e2e when push to main branch
bzp2010 Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:

api7:
runs-on: ubuntu-latest
if: contains(github.event.pull_request.labels.*.name, 'test/api7')
if: contains(github.event.pull_request.labels.*.name, 'test/api7') || github.event_name == 'push'
strategy:
matrix:
version: [3.2.13.0, 3.2.14.0]
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/unit.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,7 @@ jobs:
# Run unit tests
- name: Run CLI unit tests
run: npx nx run cli:test
- name: Run API7 Backend unit tests
run: npx nx run backend-api7:test
- name: Run OpenAPI Converter unit tests
run: NODE_OPTIONS=--experimental-vm-modules npx nx run converter-openapi:test
run: npx nx run converter-openapi:test
5 changes: 4 additions & 1 deletion apps/cli/src/command/dump.command.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { BackendAPI7 } from '@api7/adc-backend-api7';
import * as ADCSDK from '@api7/adc-sdk';
import { Listr, ListrTask } from 'listr2';
import { writeFile } from 'node:fs/promises';
Expand Down Expand Up @@ -41,7 +42,9 @@ export const LoadRemoteConfigurationTask = ({
{
title: 'Filter configuration resource type',
enabled: () =>
includeResourceType?.length > 0 || excludeResourceType?.length > 0,
//TODO implement API-level resource filtering on APISIX backend
!(backend instanceof BackendAPI7) &&
(includeResourceType?.length > 0 || excludeResourceType?.length > 0),
task: () => {
ctx.remote = filterResourceType(
ctx.remote,
Expand Down
36 changes: 35 additions & 1 deletion libs/backend-api7/src/fetcher.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import * as ADCSDK from '@api7/adc-sdk';
import { Axios } from 'axios';
import { ListrTask } from 'listr2';
import { isEmpty } from 'lodash';

import { ToADC } from './transformer';
import * as typing from './typing';
Expand All @@ -14,11 +15,15 @@ type FetchTask = ListrTask<{
export class Fetcher {
private readonly toADC = new ToADC();

constructor(private readonly client: Axios) {}
constructor(
private readonly client: Axios,
private readonly backendOpts: ADCSDK.BackendOptions,
) {}

public listServices(): FetchTask {
return {
title: 'Fetch services',
skip: this.isSkip([ADCSDK.ResourceType.SERVICE]),
task: async (ctx, task) => {
const resp = await this.client.get<{ list: Array<typing.Service> }>(
`/api/gateway_groups/${ctx.gatewayGroupId}/services`,
Expand Down Expand Up @@ -62,6 +67,7 @@ export class Fetcher {
public listConsumers(): FetchTask {
return {
title: 'Fetch consumers',
skip: this.isSkip([ADCSDK.ResourceType.CONSUMER]),
task: async (ctx, task) => {
const resp = await this.client.get<{ list: Array<typing.Consumer> }>(
'/apisix/admin/consumers',
Expand All @@ -81,6 +87,7 @@ export class Fetcher {
public listSSLs(): FetchTask {
return {
title: 'Fetch ssls',
skip: this.isSkip([ADCSDK.ResourceType.SSL]),
task: async (ctx, task) => {
const resp = await this.client.get<{ list: Array<typing.SSL> }>(
'/apisix/admin/ssls',
Expand All @@ -100,6 +107,7 @@ export class Fetcher {
public listGlobalRules(): FetchTask {
return {
title: 'Fetch global rules',
skip: this.isSkip([ADCSDK.ResourceType.GLOBAL_RULE]),
task: async (ctx, task) => {
const resp = await this.client.get<{ list: Array<typing.GlobalRule> }>(
'/apisix/admin/global_rules',
Expand All @@ -119,6 +127,7 @@ export class Fetcher {
public listMetadatas(): FetchTask {
return {
title: 'Fetch plugin metadata',
skip: this.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA]),
task: async (ctx, task) => {
const resp = await this.client.get<Array<string>>(
'/apisix/admin/plugins/list',
Expand Down Expand Up @@ -160,6 +169,31 @@ export class Fetcher {
};
}

private isSkip(
requiredTypes: Array<ADCSDK.ResourceType>,
): () => string | undefined {
return () => {
const msg = 'excluded by resource type filters';
if (
this.backendOpts?.includeResourceType &&
!isEmpty(this.backendOpts.includeResourceType) &&
!requiredTypes.some((item) =>
this.backendOpts.includeResourceType.includes(item),
)
) {
return msg;
} else if (
this.backendOpts?.excludeResourceType &&
!isEmpty(this.backendOpts.excludeResourceType) &&
requiredTypes.every((item) =>
this.backendOpts.excludeResourceType.includes(item),
)
) {
return msg;
}
};
}

public allTask() {
return [
this.listServices(),
Expand Down
2 changes: 1 addition & 1 deletion libs/backend-api7/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ export class BackendAPI7 implements ADCSDK.Backend {
}

public async dump(): Promise<Listr<{ remote: ADCSDK.Configuration }>> {
const fetcher = new Fetcher(this.client);
const fetcher = new Fetcher(this.client, this.opts);

return new Listr<{ remote: ADCSDK.Configuration }>(
[
Expand Down
206 changes: 206 additions & 0 deletions libs/backend-api7/test/fetcher.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,206 @@
import * as ADCSDK from '@api7/adc-sdk';
import axios from 'axios';

import { Fetcher } from '../src/fetcher';

describe('Fetcher', () => {
describe('API-level resource type filters', () => {
type TrickFetcher = typeof Fetcher & {
isSkip: Fetcher['isSkip'];
};
const newFetcher = (opts?: Partial<ADCSDK.BackendOptions>) =>
new Fetcher(
axios.create(),
opts as ADCSDK.BackendOptions,
) as unknown as TrickFetcher;
const checkSkip = (func: () => string | undefined) => !!func();

it('should include services', () => {
const fetcher = newFetcher({
includeResourceType: [ADCSDK.ResourceType.SERVICE],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
true,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
true,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(true);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(true);
});

it('should include services, consumers, ssls', () => {
const fetcher = newFetcher({
includeResourceType: [
ADCSDK.ResourceType.SERVICE,
ADCSDK.ResourceType.CONSUMER,
ADCSDK.ResourceType.SSL,
],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
false,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(true);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(true);
});

it('should include all', () => {
const fetcher = newFetcher({
includeResourceType: [
ADCSDK.ResourceType.SERVICE,
ADCSDK.ResourceType.CONSUMER,
ADCSDK.ResourceType.SSL,
ADCSDK.ResourceType.GLOBAL_RULE,
ADCSDK.ResourceType.PLUGIN_METADATA,
],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
false,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(false);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(false);
});

it('should include all (include list defined but empty)', () => {
const fetcher = newFetcher({
includeResourceType: [],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
false,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(false);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(false);
});

it('should exclude services', () => {
const fetcher = newFetcher({
excludeResourceType: [ADCSDK.ResourceType.SERVICE],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
true,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
false,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(false);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(false);
});

it('should exclude services, consumers, ssls', () => {
const fetcher = newFetcher({
excludeResourceType: [
ADCSDK.ResourceType.SERVICE,
ADCSDK.ResourceType.CONSUMER,
ADCSDK.ResourceType.SSL,
],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
true,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
true,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
true,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(false);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(false);
});

it('should exclude all', () => {
const fetcher = newFetcher({
excludeResourceType: [
ADCSDK.ResourceType.SERVICE,
ADCSDK.ResourceType.CONSUMER,
ADCSDK.ResourceType.SSL,
ADCSDK.ResourceType.GLOBAL_RULE,
ADCSDK.ResourceType.PLUGIN_METADATA,
],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
true,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
true,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
true,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(true);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(true);
});

it('should include all (exclude list defined but empty)', () => {
const fetcher = newFetcher({
excludeResourceType: [],
});
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SERVICE]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.CONSUMER]))).toEqual(
false,
);
expect(checkSkip(fetcher.isSkip([ADCSDK.ResourceType.SSL]))).toEqual(
false,
);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.GLOBAL_RULE])),
).toEqual(false);
expect(
checkSkip(fetcher.isSkip([ADCSDK.ResourceType.PLUGIN_METADATA])),
).toEqual(false);
});
});
});
3 changes: 2 additions & 1 deletion libs/backend-api7/tsconfig.spec.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"src/**/*.test.ts",
"src/**/*.spec.ts",
"src/**/*.d.ts",
"e2e/**/*.e2e-spec.ts"
"e2e/**/*.e2e-spec.ts",
"test/**/*.spec.ts"
]
}
4 changes: 4 additions & 0 deletions libs/sdk/src/backend/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ export interface BackendOptions {
tlsClientKeyFile?: string;
tlsSkipVerify?: boolean;
verbose?: number;

labelSelector?: Record<string, string>;
includeResourceType?: Array<ADCSDK.ResourceType>;
excludeResourceType?: Array<ADCSDK.ResourceType>;
}

export interface Backend {
Expand Down
Loading