-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #8 from thulasi-ram/main
Release 0.4.0
- Loading branch information
Showing
44 changed files
with
1,725 additions
and
210 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| name: Dump Discussions Workflow | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| schedule: | ||
| - cron: '0 0 * * *' | ||
|
|
||
| jobs: | ||
| discussions_dump: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-node@v3 | ||
| with: | ||
| node-version: 18 | ||
| cache: 'npm' | ||
| cache-dependency-path: 'package-lock.json' | ||
| - env: | ||
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| run: | | ||
| cd ./scripts | ||
| npm ci | ||
| ls -lahtr | ||
| node ./discussions_dump.js | ||
| - uses: actions/[email protected] | ||
| with: | ||
| name: discussions_dump.json | ||
| path: ./scripts/discussions_dump.json | ||
| if-no-files-found: error | ||
| upload_dump_to_s3: | ||
| runs-on: ubuntu-latest | ||
| needs: [discussions_dump] | ||
| # These permissions are needed to interact with GitHub's OIDC Token endpoint. | ||
| permissions: | ||
| id-token: write | ||
| contents: read | ||
| steps: | ||
| - uses: actions/[email protected] | ||
| - run: ls -lahtr | ||
| - run: pwd | ||
| - uses: aws-actions/configure-aws-credentials@v3 | ||
| with: | ||
| aws-region: us-east-1 | ||
| role-to-assume: ${{ secrets.AWS_SAD_UPLOADER_ROLE }} | ||
| # use recursive: https://github.com/aws/aws-cli/issues/2929 | ||
| - run: aws s3 cp discussions_dump.json s3://simple-android-debloater --recursive |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,4 @@ | ||
| name: 'publish' | ||
| name: 'Publish Cross Platform Builds' | ||
| on: | ||
| push: | ||
| branches: | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,4 +10,5 @@ vite.config.js.timestamp-* | |
| vite.config.ts.timestamp-* | ||
| *.db | ||
| *.db-shm | ||
| *.db-wal | ||
| *.db-wal | ||
| discussions_dump.json | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| // https://stepzen.com/blog/consume-graphql-in-javascript | ||
| // https://stackoverflow.com/questions/42938472/what-is-the-reason-for-having-edges-and-nodes-in-a-connection-in-your-graphql-sc | ||
| // https://docs.github.com/en/graphql/guides/using-the-graphql-api-for-discussions | ||
| import fetch from 'node-fetch'; | ||
| import fs from 'fs'; | ||
|
|
||
| async function getDiscussions(no_of_discussions, next_cursor) { | ||
| const reqData = JSON.stringify({ | ||
| query: `query($no_of_discussions:Int!, $next_cursor: String) { | ||
| repository(owner: "thulasi-ram", name: "simple_android_debloater") { | ||
| discussions(first: $no_of_discussions, after: $next_cursor) { | ||
| # type: DiscussionConnection | ||
| totalCount # Int! | ||
| pageInfo { | ||
| startCursor | ||
| endCursor | ||
| hasNextPage | ||
| hasPreviousPage | ||
| } | ||
| nodes { | ||
| # type: Discussion | ||
| id | ||
| title | ||
| closed | ||
| body | ||
| bodyHTML | ||
| url | ||
| answer { | ||
| body | ||
| bodyHTML | ||
| } | ||
| labels(first: 100) { | ||
| totalCount | ||
| nodes { | ||
| name | ||
| description | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }`, | ||
| variables: `{ | ||
| "no_of_discussions": ${no_of_discussions}, | ||
| "next_cursor": ${next_cursor == null ? null : `"${next_cursor}"`} | ||
| }` | ||
| }); | ||
|
|
||
| const response = await fetch('https://api.github.com/graphql', { | ||
| method: 'post', | ||
| body: reqData, | ||
| headers: { | ||
| 'Content-Type': 'application/json', | ||
| 'Content-Length': reqData.length, | ||
| Authorization: 'bearer ' + process.env.GH_TOKEN, | ||
| 'User-Agent': 'Node' | ||
| } | ||
| }); | ||
| const json = await response.json(); | ||
|
|
||
| let data = json.data; | ||
|
|
||
| if (!data) { | ||
| console.log(response); | ||
| console.log(json); | ||
| throw new Error('Error in response json'); | ||
| } | ||
|
|
||
| const discussions = data?.repository?.discussions; | ||
|
|
||
| if (!discussions) { | ||
| console.log(JSON.stringify(data)); | ||
| } | ||
|
|
||
| return discussions; | ||
| } | ||
|
|
||
| async function getAllDiscussions() { | ||
| // https://stackoverflow.com/questions/71952373/how-to-run-graphql-query-in-a-loop-until-condition-is-not-matched | ||
|
|
||
| let hasNext = true; | ||
| let nextCursor = null; | ||
| let allDiscussions = []; | ||
| while (hasNext) { | ||
| let discussionsData = await getDiscussions(1, nextCursor); | ||
| if (!discussionsData) { | ||
| break; | ||
| } | ||
| hasNext = discussionsData.pageInfo.hasNextPage; | ||
| nextCursor = discussionsData.pageInfo.endCursor; | ||
| allDiscussions.push(...discussionsData.nodes); | ||
| } | ||
|
|
||
| allDiscussions = allDiscussions.map((d) => parseDiscussion(d)).filter((d) => d != null); | ||
| let discussionsDump = JSON.stringify(allDiscussions); | ||
| console.log(discussionsDump); | ||
|
|
||
| fs.writeFile('discussions_dump.json', discussionsDump, 'utf8', function (err) { | ||
| if (err) throw err; | ||
| console.log('Dumped json'); | ||
| }); | ||
| } | ||
|
|
||
| const FILTER_BY_PACKAGE_LABEL = 'pkg'; | ||
|
|
||
| function parseDiscussion(discussion) { | ||
| if (!discussion) { | ||
| return null; | ||
| } | ||
|
|
||
| const labelCount = discussion.labels?.totalCount; | ||
| if (labelCount && labelCount > 100) { | ||
| console.error(`"discussoon ${discussion} has more than 100 labels"`); | ||
| } | ||
| const labels = discussion.labels?.nodes; | ||
| if (!labels) { | ||
| return null; | ||
| } | ||
|
|
||
| let matched = labels.filter((l) => l.name === FILTER_BY_PACKAGE_LABEL); | ||
| if (!matched) { | ||
| return null; | ||
| } | ||
| discussion.labels = labels; | ||
| return discussion; | ||
| } | ||
|
|
||
| getAllDiscussions(); |
Oops, something went wrong.