diff --git a/.gitignore b/.gitignore index cc2ae1e..0825698 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,7 @@ node_modules .cache dist package-lock.json -website/index.merged.html website/index.html +website/table.html **/.DS_Store diff --git a/buildMetadata.js b/buildMetadata.js index 70a087b..659c745 100644 --- a/buildMetadata.js +++ b/buildMetadata.js @@ -22,7 +22,7 @@ const GITHUB_METADATA_FILE = `data/${dayjs().format( 'YYYY-MM-DDTHH.mm.ss', )}-fetched_repo_data.json`; const GITHUB_REPOS = 'data/list_repos.json'; -const METADATA_TABLE = 'data/table.md'; +// const METADATA_TABLE = 'data/table.md'; // --- HTTP --- const API = 'https://api.github.com/'; const options = { @@ -73,41 +73,41 @@ const ProgressBar = (i, batchSize, total) => { ); }; -const getLastUpdate = updated => { - const updt = Number(dayjs(updated).diff(dayjs(), 'days')); - if (updt < 0) { - if (Math.abs(updt) === 1) return `1 day ago`; - return `${Math.abs(updt)} days ago`; - } else if (updt === 0) return 'today'; - return updated; -}; +// const getLastUpdate = updated => { +// const updt = Number(dayjs(updated).diff(dayjs(), 'days')); +// if (updt < 0) { +// if (Math.abs(updt) === 1) return `1 day ago`; +// return `${Math.abs(updt)} days ago`; +// } else if (updt === 0) return 'today'; +// return updated; +// }; -const createLine = data => { - const { - name, - html_url: repoURL, - description, - homepage, - stargazers_count: stargazers, - updated_at: updated, - language, - license, - owner, - } = data; - if (!data || !name) return '|ERROR |'; +// const createLine = data => { +// const { +// name, +// html_url: repoURL, +// description, +// homepage, +// stargazers_count: stargazers, +// updated_at: updated, +// language, +// license, +// owner, +// } = data; +// if (!data || !name) return '|ERROR |'; - const lineData = [ - `[${name}](${repoURL})`, - description || '-', - homepage || '-', - stargazers, - getLastUpdate(updated), - language, - license && `[${license.name}](${license.url})`, - owner && `[${owner.login}](${owner.html_url})`, - ]; - return `|${lineData.join('|')}|`; -}; +// const lineData = [ +// `[${name}](${repoURL})`, +// description || '-', +// homepage || '-', +// stargazers, +// getLastUpdate(updated), +// language, +// license && `[${license.name}](${license.url})`, +// owner && `[${owner.login}](${owner.html_url})`, +// ]; +// return `|${lineData.join('|')}|`; +// }; async function batchFetchRepoMetadata(githubRepos) { const repos = githubRepos.map(removeHost); @@ -118,37 +118,31 @@ async function batchFetchRepoMetadata(githubRepos) { const batch = repos.slice(i, i + BATCH_SIZE); if (process.env.DEBUG) console.log({ batch }); const res = await fetchAll(batch); - fs.appendFile( - GITHUB_METADATA_FILE, - JSON.stringify(res, null, 2), - printError, - ); metadata.push(...res); ProgressBar(i, BATCH_SIZE, repos.length); - // poor's man rate limiting so github don't blacklist us - // also the file has the time to append the data + // poor man's rate limiting so github don't ban us await delay(DELAY); } ProgressBar(repos.length, BATCH_SIZE, repos.length); return metadata; } -const convertToTable = data => { - const header = ` -# Repository Metadata +// const convertToTable = data => { +// const header = ` +// # Repository Metadata -| Name | Description | Homepage | Star | Updated | Language | License | Author | -| ----------- | ----------- | -------- | ---- | ------- | -------- | :---: | ------:|`; - const table = [header] - .concat( - data - .sort((a, b) => Number(b.stargazers_count) - Number(a.stargazers_count)) - .map(createLine), - ) - .join('\n'); +// | Name | Description | Homepage | Star | Updated | Language | License | Author | +// | ----------- | ----------- | -------- | ---- | ------- | -------- | :---: | ------:|`; +// const table = [header] +// .concat( +// data +// .sort((a, b) => Number(b.stargazers_count) - Number(a.stargazers_count)) +// .map(createLine), +// ) +// .join('\n'); - return writeFile(METADATA_TABLE, table); -}; +// return writeFile(METADATA_TABLE, table); +// }; async function main() { try { @@ -162,11 +156,15 @@ async function main() { const metadata = batchFetchRepoMetadata(githubRepos); - console.log('✅ fetching metadata'); + await writeFile( + GITHUB_METADATA_FILE, + JSON.stringify(metadata, null, 2), + printError, + ); + console.log('✅ metadata saved'); - await convertToTable(metadata); - - console.log('✅ writing metadata table'); + // await convertToTable(metadata); + // console.log('✅ writing metadata table'); } catch (err) { printError(err); } diff --git a/buildTable.js b/buildTable.js new file mode 100644 index 0000000..f1aa296 --- /dev/null +++ b/buildTable.js @@ -0,0 +1,106 @@ +const fs = require('fs'); +// const showdown = require('showdown'); +const cheerio = require('cheerio'); +// const Parcel = require('parcel-bundler'); +const dayjs = require('dayjs'); +const metaData = require('./data/2018-06-06T17.54.30-fetched_repo_data.json'); + +process.env.NODE_ENV = 'production'; + +process.on('unhandledRejection', error => { + console.log('unhandledRejection', error.message); +}); + +// const table = 'data/table.md'; +const templateHTML = 'website/table.tmpl.html'; +// const merged = 'website/table.html'; +const destination = 'website/table.html'; + +const valueNames = [ + 'name', + 'description', + 'homepage', + 'star', + 'updated', + 'language', + 'license', + 'author', +]; + +const getLastUpdate = updated => { + const updt = Number(dayjs(updated).diff(dayjs(), 'days')); + if (updt < 0) { + if (Math.abs(updt) === 1) return `1 day ago`; + return `${Math.abs(updt)} days ago`; + } else if (updt === 0) return 'today'; + return updated; +}; + +const formatEntry = ( + { + name, + html_url: repoURL, + description, + homepage, + stargazers_count: stargazers, + updated_at: updated, + language, + license, + owner, + }, + i, +) => + [ + `
  • `, + `${name}`, + `

    ${description || '-'}

    `, + (homepage && + `🔗 website`) || + '

    ', + `

    ⭐️${stargazers}

    `, + `

    ${getLastUpdate(updated)}

    `, + (language && `

    💻${language}

    `) || '

    ', + (license && + `📃${ + license.name + }`) || + '

    ', + owner && + `Made by ${ + owner.login + }`, + '
    • ', + ].join(''); + +function main() { + const indexTemplate = fs.readFileSync(templateHTML, 'utf8'); + const $ = cheerio.load(indexTemplate); + const btn = valueNames.map( + v => ``, + ); + $('#md').append( + [ + ` ${btn.join('')}`, + '
      ', + metaData.map(formatEntry).join(''), + '
    ', + ].join(''), + ); + + $('body').append( + '', + ); + + $('body').append(` + + `); + console.log('Writing table.html'); + fs.writeFileSync(destination, $.html(), 'utf8'); + console.log('DONE 👍'); +} + +main(); diff --git a/data/2018-06-06T17.54.30-fetched_repo_data.json b/data/2018-06-06T17.54.30-fetched_repo_data.json index 3bfd5c9..58a2ed6 100644 --- a/data/2018-06-06T17.54.30-fetched_repo_data.json +++ b/data/2018-06-06T17.54.30-fetched_repo_data.json @@ -1123,9 +1123,7 @@ }, "network_count": 1333, "subscribers_count": 300 - } -][ - { +},{ "id": 19360616, "node_id": "MDEwOlJlcG9zaXRvcnkxOTM2MDYxNg==", "name": "swarm", @@ -2226,9 +2224,7 @@ }, "network_count": 7, "subscribers_count": 4 - } -][ - { +},{ "id": 67536560, "node_id": "MDEwOlJlcG9zaXRvcnk2NzUzNjU2MA==", "name": "elsy", @@ -3467,9 +3463,7 @@ }, "network_count": 65, "subscribers_count": 40 - } -][ - { +},{ "id": 27701139, "node_id": "MDEwOlJlcG9zaXRvcnkyNzcwMTEzOQ==", "name": "conduit", @@ -4808,9 +4802,7 @@ }, "network_count": 9, "subscribers_count": 4 - } -][ - { +},{ "id": 85515090, "node_id": "MDEwOlJlcG9zaXRvcnk4NTUxNTA5MA==", "name": "docker-flow-monitor", @@ -5977,9 +5969,7 @@ }, "network_count": 11, "subscribers_count": 6 - } -][ - { +},{ "id": 21704134, "node_id": "MDEwOlJlcG9zaXRvcnkyMTcwNDEzNA==", "name": "flannel", @@ -7378,9 +7368,7 @@ }, "network_count": 31, "subscribers_count": 10 - } -][ - { +},{ "id": 72737874, "node_id": "MDEwOlJlcG9zaXRvcnk3MjczNzg3NA==", "name": "haven-platform", @@ -8639,9 +8627,7 @@ }, "network_count": 152, "subscribers_count": 88 - } -][ - { +},{ "id": 26337322, "node_id": "MDEwOlJlcG9zaXRvcnkyNjMzNzMyMg==", "name": "rancher", @@ -9854,9 +9840,7 @@ }, "network_count": 363, "subscribers_count": 205 - } -][ - { +},{ "id": 52381034, "node_id": "MDEwOlJlcG9zaXRvcnk1MjM4MTAzNA==", "name": "workflow", @@ -11167,9 +11151,7 @@ }, "network_count": 2, "subscribers_count": 2 - } -][ - { +},{ "id": 42408804, "node_id": "MDEwOlJlcG9zaXRvcnk0MjQwODgwNA==", "name": "traefik", @@ -12408,9 +12390,7 @@ }, "network_count": 122, "subscribers_count": 52 - } -][ - { +},{ "id": 49986046, "node_id": "MDEwOlJlcG9zaXRvcnk0OTk4NjA0Ng==", "name": "falco", @@ -13629,9 +13609,7 @@ }, "network_count": 98, "subscribers_count": 31 - } -][ - { +},{ "id": 34205411, "node_id": "MDEwOlJlcG9zaXRvcnkzNDIwNTQxMQ==", "name": "docker-unison", @@ -14772,9 +14750,7 @@ }, "network_count": 31, "subscribers_count": 19 - } -][ - { +},{ "id": 77419377, "node_id": "MDEwOlJlcG9zaXRvcnk3NzQxOTM3Nw==", "name": "ctop", @@ -15887,9 +15863,7 @@ }, "network_count": 41, "subscribers_count": 36 - } -][ - { +},{ "id": 42531199, "node_id": "MDEwOlJlcG9zaXRvcnk0MjUzMTE5OQ==", "name": "dvm", @@ -17002,9 +16976,7 @@ }, "network_count": 1, "subscribers_count": 3 - } -][ - { +},{ "id": 34074446, "node_id": "MDEwOlJlcG9zaXRvcnkzNDA3NDQ0Ng==", "name": "wharfee", @@ -18137,9 +18109,7 @@ }, "network_count": 369, "subscribers_count": 109 - } -][ - { +},{ "id": 51555295, "node_id": "MDEwOlJlcG9zaXRvcnk1MTU1NTI5NQ==", "name": "rapid", @@ -19212,9 +19182,7 @@ }, "network_count": 4, "subscribers_count": 9 - } -][ - { +},{ "id": 100341708, "node_id": "MDEwOlJlcG9zaXRvcnkxMDAzNDE3MDg=", "name": "lstags", @@ -20387,9 +20355,7 @@ }, "network_count": 36, "subscribers_count": 26 - } -][ - { +},{ "id": 39589297, "node_id": "MDEwOlJlcG9zaXRvcnkzOTU4OTI5Nw==", "name": "dlayer", @@ -21582,9 +21548,7 @@ }, "network_count": 16, "subscribers_count": 43 - } -][ - { +},{ "id": 36849312, "node_id": "MDEwOlJlcG9zaXRvcnkzNjg0OTMxMg==", "name": "runlike", @@ -22659,9 +22623,7 @@ }, "network_count": 399, "subscribers_count": 52 - } -][ - { +},{ "id": 49360444, "node_id": "MDEwOlJlcG9zaXRvcnk0OTM2MDQ0NA==", "name": "Dockerfiles", @@ -23802,9 +23764,7 @@ }, "network_count": 5, "subscribers_count": 5 - } -][ - { +},{ "id": 29760781, "node_id": "MDEwOlJlcG9zaXRvcnkyOTc2MDc4MQ==", "name": "docket", @@ -25003,9 +24963,7 @@ }, "network_count": 139, "subscribers_count": 103 - } -][ - { +},{ "id": 12467623, "node_id": "MDEwOlJlcG9zaXRvcnkxMjQ2NzYyMw==", "name": "dockerode", @@ -26164,9 +26122,7 @@ }, "network_count": 246, "subscribers_count": 56 - } -][ - { +},{ "id": 16607898, "node_id": "MDEwOlJlcG9zaXRvcnkxNjYwNzg5OA==", "name": "drone", @@ -27365,9 +27321,7 @@ }, "network_count": 4, "subscribers_count": 1 - } -][ - { +},{ "id": 18931097, "node_id": "MDEwOlJlcG9zaXRvcnkxODkzMTA5Nw==", "name": "devstep", @@ -28408,9 +28362,7 @@ }, "network_count": 9, "subscribers_count": 11 - } -][ - { +},{ "id": 78498440, "node_id": "MDEwOlJlcG9zaXRvcnk3ODQ5ODQ0MA==", "name": "lando", @@ -29603,9 +29555,7 @@ }, "network_count": 618, "subscribers_count": 225 - } -][ - { +},{ "id": 59715467, "node_id": "MDEwOlJlcG9zaXRvcnk1OTcxNTQ2Nw==", "name": "docker-lambda", @@ -30778,9 +30728,7 @@ }, "network_count": 65, "subscribers_count": 40 - } -][ - { +},{ "id": 13003799, "node_id": "MDEwOlJlcG9zaXRvcnkxMzAwMzc5OQ==", "name": "azk", @@ -31947,9 +31895,7 @@ }, "network_count": 36, "subscribers_count": 12 - } -][ - { +},{ "id": 72878982, "node_id": "MDEwOlJlcG9zaXRvcnk3Mjg3ODk4Mg==", "name": "landscape", @@ -33242,9 +33188,7 @@ }, "network_count": 66, "subscribers_count": 69 - } -][ - { +},{ "id": 30508772, "node_id": "MDEwOlJlcG9zaXRvcnkzMDUwODc3Mg==", "name": "Docker", @@ -34399,9 +34343,7 @@ }, "network_count": 343, "subscribers_count": 167 - } -][ - { +},{ "id": 23929024, "node_id": "MDEwOlJlcG9zaXRvcnkyMzkyOTAyNA==", "name": "dockerfiles", diff --git a/fetchRepos.js b/fetchRepos.js index 0de374b..acd87a3 100644 --- a/fetchRepos.js +++ b/fetchRepos.js @@ -85,19 +85,22 @@ async function main() { const repos = githubRepos.map(removeHost); + const metadata = []; /* eslint-disable no-await-in-loop */ for (let i = 0; i < repos.length; i += BATCH_SIZE) { const batch = repos.slice(i, i + BATCH_SIZE); if (process.env.DEBUG) console.log({ batch }); const res = await Promise.all(batch.map(async path => get(path))); - fs.appendFile( - GITHUB_METADATA_FILE, - JSON.stringify(res, null, 2), - err => err && console.error(err), - ); + metadata.push(...res); ProgressBar(i, BATCH_SIZE, repos.length); await delay(DELAY); } + + fs.writeFile( + GITHUB_METADATA_FILE, + JSON.stringify(metadata, null, 2), + err => err && console.error(err), + ); ProgressBar(repos.length, BATCH_SIZE, repos.length); } catch (err) { console.error('ERROR:', err); diff --git a/package.json b/package.json index 2611e52..34050cb 100644 --- a/package.json +++ b/package.json @@ -23,9 +23,11 @@ "homepage": "https://github.com/veggiemonk/awesome-docker#readme", "dependencies": { "cheerio": "^1.0.0-rc.2", - "critical": "^1.3.2", - "dayjs": "^1.6.4", + "critical": "^1.3.3", + "dayjs": "^1.6.6", "draftlog": "^1.0.12", + "jsdom": "^11.11.0", + "list.js": "^1.5.0", "node-fetch": "^2.1.2", "parcel-bundler": "^1.8.1", "rimraf": "^2.6.2", @@ -39,6 +41,6 @@ "eslint-plugin-import": "^2.12.0", "eslint-plugin-jsx-a11y": "^6.0.3", "eslint-plugin-prettier": "^2.6.0", - "prettier": "^1.13.0" + "prettier": "^1.13.4" } } diff --git a/website/index.js b/website/index.js index 28f3ac5..c96e45f 100644 --- a/website/index.js +++ b/website/index.js @@ -1,3 +1,5 @@ +const list = require('list.js'); + const main = () => { console.log('hi!'); }; diff --git a/website/table.tmpl.html b/website/table.tmpl.html new file mode 100644 index 0000000..8b0843e --- /dev/null +++ b/website/table.tmpl.html @@ -0,0 +1,257 @@ + + + + + + + + Awesome-docker + + + + + + + + + + +
    +

    Awesome-docker

    +

    A curated list of Docker resources and projects

    + View on GitHub +
    + + Star +
    +
    + + + + + + \ No newline at end of file