Verified Commit 50f71f9c authored by Corentin Mors's avatar Corentin Mors
Browse files

Initial release

parents
node_modules/
output.json
# Github Crawler
This tool read an entire org repository and list all files in its projects.
## Run
First install dependencies.
```
npm ci
```
Then run.
```
npm run start
```
const { Octokit } = require("@octokit/rest");
const fs = require("fs");
const octokit = new Octokit({
auth: "1243fc7aaf8b8e32e4f898834245ea0a2ef75f84",
userAgent: 'Crawler v1',
timeZone: 'Europe/Paris',
baseUrl: 'https://api.github.com'
});
const repoTree = async (org, repo, branch) => {
const masterSha = await getMasterSha(org, repo, branch);
const { data: pullRequest } = await octokit.git.getTree({
owner: org,
repo,
tree_sha: masterSha,
recursive: true,
});
const output = pullRequest.tree.map((file) => {
return {
repo: repo,
path: file.path,
url: file.url
};
});
return output;
};
const getMasterSha = async (org, repo, branch) => {
const mainBranch = branch || await getMainBranch(org, repo);
const { data: branchDetails } = await octokit.repos.getBranch({
owner: org,
repo,
branch: mainBranch
});
return branchDetails.commit.sha;
};
const getMainBranch = async (org, repo) => {
const { data: repository } = await octokit.repos.get({
owner: org,
repo
});
return repository.default_branch;
};
const listAllReposOfOrg = async (org) => {
const { data: list } = await octokit.repos.listForOrg({
org
});
const output = list.map((project) => project.name);
return output;
};
const listAllFilesFromAllProjects = async (org) => {
const listProjects = await listAllReposOfOrg(org);
const fileList = [];
for (const repo of listProjects) {
console.log(`Reading ${repo} ...`);
fileList.push(...(await repoTree(org, repo)));
console.log(`Done.`);
};
fs.writeFile('output.json', JSON.stringify(fileList, null, 2), (error) => {
if (error) return console.log(error);
console.log('\nExport done in output.json');
});
// console.log(fileList);
};
listAllFilesFromAllProjects("octokit");
{
"name": "gittree",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"@octokit/auth-token": {
"version": "2.4.3",
"resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-2.4.3.tgz",
"integrity": "sha512-fdGoOQ3kQJh+hrilc0Plg50xSfaCKOeYN9t6dpJKXN9BxhhfquL0OzoQXg3spLYymL5rm29uPeI3KEXRaZQ9zg==",
"requires": {
"@octokit/types": "^5.0.0"
}
},
"@octokit/core": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/@octokit/core/-/core-3.2.1.tgz",
"integrity": "sha512-XfFSDDwv6tclUenS0EmB6iA7u+4aOHBT1Lz4PtQNQQg3hBbNaR/+Uv5URU+egeIuuGAiMRiDyY92G4GBOWOqDA==",
"requires": {
"@octokit/auth-token": "^2.4.0",
"@octokit/graphql": "^4.3.1",
"@octokit/request": "^5.4.0",
"@octokit/types": "^5.0.0",
"before-after-hook": "^2.1.0",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/endpoint": {
"version": "6.0.9",
"resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-6.0.9.tgz",
"integrity": "sha512-3VPLbcCuqji4IFTclNUtGdp9v7g+nspWdiCUbK3+iPMjJCZ6LEhn1ts626bWLOn0GiDb6j+uqGvPpqLnY7pBgw==",
"requires": {
"@octokit/types": "^5.0.0",
"is-plain-object": "^5.0.0",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/graphql": {
"version": "4.5.7",
"resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-4.5.7.tgz",
"integrity": "sha512-Gk0AR+DcwIK/lK/GX+OQ99UqtenQhcbrhHHfOYlrCQe17ADnX3EKAOKRsAZ9qZvpi5MuwWm/Nm+9aO2kTDSdyA==",
"requires": {
"@octokit/request": "^5.3.0",
"@octokit/types": "^5.0.0",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/plugin-paginate-rest": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-2.6.0.tgz",
"integrity": "sha512-o+O8c1PqsC5++BHXfMZabRRsBIVb34tXPWyQLyp2IXq5MmkxdipS7TXM4Y9ldL1PzY9CTrCsn/lzFFJGM3oRRA==",
"requires": {
"@octokit/types": "^5.5.0"
}
},
"@octokit/plugin-request-log": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/@octokit/plugin-request-log/-/plugin-request-log-1.0.2.tgz",
"integrity": "sha512-oTJSNAmBqyDR41uSMunLQKMX0jmEXbwD1fpz8FG27lScV3RhtGfBa1/BBLym+PxcC16IBlF7KH9vP1BUYxA+Eg=="
},
"@octokit/plugin-rest-endpoint-methods": {
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-4.2.1.tgz",
"integrity": "sha512-QyFr4Bv807Pt1DXZOC5a7L5aFdrwz71UHTYoHVajYV5hsqffWm8FUl9+O7nxRu5PDMtB/IKrhFqTmdBTK5cx+A==",
"requires": {
"@octokit/types": "^5.5.0",
"deprecation": "^2.3.1"
}
},
"@octokit/request": {
"version": "5.4.10",
"resolved": "https://registry.npmjs.org/@octokit/request/-/request-5.4.10.tgz",
"integrity": "sha512-egA49HkqEORVGDZGav1mh+VD+7uLgOxtn5oODj6guJk0HCy+YBSYapFkSLFgeYj3Fr18ZULKGURkjyhkAChylw==",
"requires": {
"@octokit/endpoint": "^6.0.1",
"@octokit/request-error": "^2.0.0",
"@octokit/types": "^5.0.0",
"deprecation": "^2.0.0",
"is-plain-object": "^5.0.0",
"node-fetch": "^2.6.1",
"once": "^1.4.0",
"universal-user-agent": "^6.0.0"
}
},
"@octokit/request-error": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-2.0.3.tgz",
"integrity": "sha512-GgD5z8Btm301i2zfvJLk/mkhvGCdjQ7wT8xF9ov5noQY8WbKZDH9cOBqXzoeKd1mLr1xH2FwbtGso135zGBgTA==",
"requires": {
"@octokit/types": "^5.0.1",
"deprecation": "^2.0.0",
"once": "^1.4.0"
}
},
"@octokit/rest": {
"version": "18.0.9",
"resolved": "https://registry.npmjs.org/@octokit/rest/-/rest-18.0.9.tgz",
"integrity": "sha512-CC5+cIx974Ygx9lQNfUn7/oXDQ9kqGiKUC6j1A9bAVZZ7aoTF8K6yxu0pQhQrLBwSl92J6Z3iVDhGhGFgISCZg==",
"requires": {
"@octokit/core": "^3.0.0",
"@octokit/plugin-paginate-rest": "^2.2.0",
"@octokit/plugin-request-log": "^1.0.0",
"@octokit/plugin-rest-endpoint-methods": "4.2.1"
}
},
"@octokit/types": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/@octokit/types/-/types-5.5.0.tgz",
"integrity": "sha512-UZ1pErDue6bZNjYOotCNveTXArOMZQFG6hKJfOnGnulVCMcVVi7YIIuuR4WfBhjo7zgpmzn/BkPDnUXtNx+PcQ==",
"requires": {
"@types/node": ">= 8"
}
},
"@types/node": {
"version": "14.14.9",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.9.tgz",
"integrity": "sha512-JsoLXFppG62tWTklIoO4knA+oDTYsmqWxHRvd4lpmfQRNhX6osheUOWETP2jMoV/2bEHuMra8Pp3Dmo/stBFcw=="
},
"before-after-hook": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.1.0.tgz",
"integrity": "sha512-IWIbu7pMqyw3EAJHzzHbWa85b6oud/yfKYg5rqB5hNE8CeMi3nX+2C2sj0HswfblST86hpVEOAb9x34NZd6P7A=="
},
"deprecation": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz",
"integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ=="
},
"is-plain-object": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz",
"integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q=="
},
"node-fetch": {
"version": "2.6.1",
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
},
"once": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"requires": {
"wrappy": "1"
}
},
"universal-user-agent": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz",
"integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w=="
},
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
}
}
}
{
"name": "gittree",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "node index.js",
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"@octokit/rest": "^18.0.9"
}
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment