Github Actions for PR

.eslintrc.js

@@ -1,22 +1,24 @@
 module.exports = {
   env: {
     browser: true,
-    node: true
+    node: true,
   },
   extends: [
     'airbnb-base',
     'plugin:import/errors',
     'plugin:import/warnings',
-    'prettier'
+    'prettier',
+    'eslint:recommended',
   ],
   plugins: ['import', 'prettier'],
   rules: {
+    camelcase: 0,
     'import/order': [
       'error',
       {
         groups: ['builtin', 'external', 'parent', 'sibling', 'index'],
-        'newlines-between': 'never'
+        'newlines-between': 'never',
-      }
+      },
     ],
     'no-console': 0,
     'prefer-template': 2,
@@ -24,8 +26,8 @@ module.exports = {
       'error',
       {
         singleQuote: true,
-        trailingComma: 'all'
+        trailingComma: 'all',
-      }
+      },
-    ]
+    ],
-  }
+  },
 };

.github/workflow/pull_request.yml

@@ -0,0 +1,30 @@
+name: Pull Requests
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@01aecccf739ca6ff86c0539fbc67a7a5007bbc81
+      - uses: actions/setup-node@83c9f7a7df54d6b57455f7c57ac414f2ae5fb8de
+        with:
+          node-version: 12
+
+      - uses: actions/cache@70655ec8323daeeaa7ef06d7c56e1b9191396cbe
+        id: cache
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Install Dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts --no-audit --no-progress --prefer-offline
+      - run: npm run test
+        with:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -666,13 +666,11 @@ Services to securely store your Docker images.
 - [Awesome Sysadmin](https://github.com/n1trux/awesome-sysadmin) by [@n1trux](https://github.com/n1trux)
 - [ToolsOfTheTrade](https://github.com/cjbarber/ToolsOfTheTrade) a list of SaaS and On premise applications by [@cjbarber](https://github.com/cjbarber)
 
-
 ## Demos and Examples
 
 - [Webstack-micro](https://github.com/ferbs/webstack-micro) Demo web app showing how Docker Compose might be used to set up an API Gateway, centralized authentication, background workers, and WebSockets as containerized services.
 - [An Annotated Docker Config for Frontend Web Development](https://nystudio107.com/blog/an-annotated-docker-config-for-frontend-web-development) A local development environment with Docker allows you to shrink-wrap the devops your project needs as config, making onboarding frictionless.
 
-
 ## Good Tips
 
 - [Dealing with linked containers dependency in docker-compose](http://brunorocha.org/python/dealing-with-linked-containers-dependency-in-docker-compose.html) by [@rochacbruno](https://github.com/rochacbruno)

build.js

@@ -2,7 +2,6 @@ const fs = require('fs-extra');
 const cheerio = require('cheerio');
 const showdown = require('showdown');
 const Parcel = require('parcel-bundler');
-// const sm = require('sitemap');
 const { SitemapStream, streamToPromise } = require('sitemap');
 
 process.env.NODE_ENV = 'production';
@@ -13,7 +12,7 @@ const LOG = {
     if (process.env.DEBUG) console.log('💡 DEBUG: ', { ...args });
   },
 };
-const handleFailure = err => {
+const handleFailure = (err) => {
   LOG.error(err);
   process.exit(1);
 };
@@ -90,7 +89,7 @@ const bundle = () => {
       smStream.end();
       return streamToPromise(smStream);
     })
-    .then(sm =>
+    .then((sm) =>
       // Creates a sitemap object given the input configuration with URLs
       fs.outputFile(
         'dist/sitemap.xml',

old_build_beta.js

@@ -0,0 +1,132 @@
+const fs = require('fs-extra');
+const fetch = require('node-fetch');
+
+require('draftlog').into(console);
+
+const LOG = {
+  error: (...args) => console.error('  ERROR', { ...args }),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log('💡 DEBUG: ', { ...args });
+  },
+};
+const handleFailure = (err) => {
+  LOG.error(err);
+  process.exit(1);
+};
+
+process.on('unhandledRejection', handleFailure);
+
+if (!process.env.GITHUB_TOKEN) {
+  LOG.error('no credentials found.');
+  process.exit(1);
+}
+
+const TOKEN = process.env.GITHUB_TOKEN;
+
+// --- ENV VAR ---
+const BATCH_SIZE = parseInt(process.env.BATCH_SIZE, 10) || 10;
+const DELAY = parseInt(process.env.DELAY, 10) || 3000;
+const INTERVAL = parseInt(process.env.INTERVAL, 10) || 1;
+const INTERVAL_UNIT = process.env.INTERVAL_UNIT || 'days';
+
+// --- FILES ---
+const DATA_FOLDER = 'data';
+const README = 'README.md';
+const LATEST_FILENAME = `${DATA_FOLDER}/latest`;
+const GITHUB_REPOS = `${DATA_FOLDER}/repository.json`;
+const Authorization = `token ${TOKEN}`;
+
+// --- HTTP ---
+const API = 'https://api.github.com/';
+const options = {
+  method: 'GET',
+  headers: {
+    'User-Agent': 'awesome-docker script listing',
+    'Content-Type': 'application/json',
+    Authorization,
+  },
+};
+
+// ----------------------------------------------------------------------------
+const removeHost = (x) => x.slice('https://github.com/'.length, x.length);
+
+const delay = (ms) =>
+  new Promise((resolve) => {
+    setTimeout(() => resolve(), ms);
+  });
+
+const get = (pathURL, opt) => {
+  LOG.debug(`Fetching ${pathURL}`);
+  return fetch(`${API}repos/${pathURL}`, {
+    ...options,
+    ...opt,
+  })
+    .catch(handleFailure)
+    .then((response) => {
+      if (response.ok) return response.json();
+      throw new Error('Network response was not ok.');
+    })
+    .catch(handleFailure);
+};
+
+const fetchAll = (batch) =>
+  Promise.all(batch.map(async (pathURL) => get(pathURL)));
+
+const extractAllLinks = (markdown) => {
+  const re = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/g;
+  return markdown.match(re);
+};
+
+const extractAllRepos = (markdown) => {
+  const re = /https:\/\/github\.com\/([a-zA-Z0-9-._]+)\/([a-zA-Z0-9-._]+)/g;
+  const md = markdown.match(re);
+  return [...new Set(md)];
+};
+
+const ProgressBar = (i, batchSize, total) => {
+  const progress = Math.round((i / total) * 100);
+  const units = Math.round(progress / 2);
+  const barLine = console.draft('Starting batch...');
+  return barLine(
+    `[${'='.repeat(units)}${' '.repeat(50 - units)}] ${progress}%  -  # ${i}`,
+  );
+};
+// ----------------------------------------------------------------------------
+async function batchFetchRepoMetadata(githubRepos) {
+  const repos = githubRepos.map(removeHost);
+
+  const metadata = [];
+  /* eslint-disable no-await-in-loop */
+  for (let i = 0; i < repos.length; i += BATCH_SIZE) {
+    const batch = repos.slice(i, i + BATCH_SIZE);
+    LOG.debug({ batch });
+    const res = await fetchAll(batch);
+    LOG.debug('batch fetched...');
+    metadata.push(...res);
+    ProgressBar(i, BATCH_SIZE, repos.length);
+    // poor man's rate limiting so github doesn't ban us
+    await delay(DELAY);
+  }
+  ProgressBar(repos.length, BATCH_SIZE, repos.length);
+  return metadata;
+}
+
+async function main() {
+  try {
+    const markdown = await fs.readFile(README, 'utf8');
+    const links = extractAllLinks(markdown);
+    const githubRepos = extractAllRepos(markdown);
+    LOG.debug('writing repo list to disk...');
+    await fs.outputJSON(GITHUB_REPOS, githubRepos, { spaces: 2 });
+
+    LOG.debug('fetching data...');
+    const metadata = await batchFetchRepoMetadata(githubRepos);
+
+    LOG.debug('gracefully shutting down.');
+    process.exit();
+  } catch (err) {
+    handleFailure(err);
+  }
+}
+
+main();

package-lock.json

@@ -1229,6 +1229,14 @@
       "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz",
       "integrity": "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg=="
     },
+    "awesome-readme-to-data": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/awesome-readme-to-data/-/awesome-readme-to-data-0.0.3.tgz",
+      "integrity": "sha512-DBbPrfFxk/TGgIzzjpraiAf8zssav50E3ufMnleFv9fr28khbnAZlAQx153KQDQVbAEWt2EJfuwJdzEYRKM7/Q==",
+      "requires": {
+        "marked": "^0.8.2"
+      }
+    },
     "aws-sign2": {
       "version": "0.7.0",
       "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
@@ -4878,6 +4886,11 @@
         "object-visit": "^1.0.0"
       }
     },
+    "marked": {
+      "version": "0.8.2",
+      "resolved": "https://registry.npmjs.org/marked/-/marked-0.8.2.tgz",
+      "integrity": "sha512-EGwzEeCcLniFX51DhTpmTom+dSA/MG/OBUDjnWtHbEnjAH180VzUeAw+oE4+Zv+CoYBWyRlYOTR0N8SO9R1PVw=="
+    },
     "md5.js": {
       "version": "1.3.5",
       "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz",
@@ -5008,18 +5021,11 @@
       }
     },
     "mkdirp": {
-      "version": "0.5.1",
+      "version": "0.5.5",
-      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
-      "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
+      "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
       "requires": {
-        "minimist": "0.0.8"
+        "minimist": "^1.2.5"
-      },
-      "dependencies": {
-        "minimist": {
-          "version": "0.0.8",
-          "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
-          "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0="
-        }
       }
     },
     "ms": {
@@ -5073,6 +5079,11 @@
       "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-1.7.1.tgz",
       "integrity": "sha512-2+DuKodWvwRTrCfKOeR24KIc5unKjOh8mz17NCzVnHWfjAdDqbfbjqh7gUT+BkXBRQM52+xCHciKWonJ3CbJMQ=="
     },
+    "node-fetch": {
+      "version": "2.6.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz",
+      "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA=="
+    },
     "node-forge": {
       "version": "0.7.6",
       "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.7.6.tgz",

package.json

@@ -4,7 +4,8 @@
   "description": "A curated list of Docker resources and projects Inspired by @sindresorhus and improved by amazing contributors",
   "main": "build.js",
   "scripts": {
-    "build": "rimraf ./dist/ && node build.js"
+    "build": "rimraf ./dist/ && node build.js",
+    "test": "node pull_request.js"
   },
   "repository": {
     "type": "git",
@@ -17,9 +18,11 @@
   },
   "homepage": "https://github.com/veggiemonk/awesome-docker#readme",
   "dependencies": {
+    "awesome-readme-to-data": "0.0.3",
     "cheerio": "1.0.0-rc.3",
     "draftlog": "1.0.12",
     "fs-extra": "9.0.0",
+    "node-fetch": "2.6.0",
     "parcel-bundler": "1.12.4",
     "rimraf": "3.0.2",
     "showdown": "1.9.1",

pull_request.js

@@ -0,0 +1,177 @@
+const fs = require('fs-extra');
+const fetch = require('node-fetch');
+
+function envvar_undefined(variable_name) {
+  throw new Error(`${variable_name} must be defined`);
+}
+console.log({
+  DEBUG: process.env.DEBUG,
+});
+
+const README = 'README.md';
+const GITHUB_GQL_API = 'https://api.github.com/graphql';
+const TOKEN = process.env.GITHUB_TOKEN || envvar_undefined('GITHUB_TOKEN');
+
+const LINKS_OPTIONS = {
+  redirect: 'error',
+  headers: {
+    'Content-Type': 'application/json',
+    'user-agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+  },
+};
+
+const Authorization = `token ${TOKEN}`;
+
+const make_GQL_options = (query) => ({
+  method: 'POST',
+  headers: {
+    Authorization,
+    'Content-Type': 'application/json',
+    'user-agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+  },
+  body: JSON.stringify({ query }),
+});
+
+const LOG = {
+  error: (...args) => console.error('❌ ERROR', { ...args }),
+  error_string: (...args) =>
+    console.error('❌ ERROR', JSON.stringify({ ...args })),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args });
+  },
+  debug_string: (...args) => {
+    if (process.env.DEBUG)
+      console.log('>>> DEBUG: ', JSON.stringify({ ...args }));
+  },
+};
+
+const handleFailure = (error) => {
+  console.error(`${error.message}: ${error.stack}`, { error });
+  process.exit(1);
+};
+
+process.on('unhandledRejection', handleFailure);
+
+const extract_all_links = (markdown) => {
+  // if you have a problem and you try to solve it with a regex,
+  // now you have two problems
+  const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/\\\w]*))?)/g;
+  return markdown.match(re);
+};
+
+const find_duplicates = (arr) => {
+  const hm = {};
+  const dup = [];
+  arr.forEach((e) => {
+    if (hm[e]) dup.push(e);
+    else hm[e] = null;
+  });
+  return dup;
+};
+
+const partition = (arr, func) => {
+  const ap = [[], []];
+  arr.forEach((e) => (func(e) ? ap[0].push(e) : ap[1].push(e)));
+  return ap;
+};
+
+async function fetch_link(url) {
+  try {
+    const { ok, statusText, redirected } = await fetch(url, LINKS_OPTIONS);
+    return [url, { ok, status: statusText, redirected }];
+  } catch (error) {
+    return [url, { ok: false, status: error.message }];
+  }
+}
+
+async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) {
+  const result = [];
+  /* eslint-disable no-await-in-loop */
+  for (let i = 0; i < arr.length; i += BATCH_SIZE) {
+    const batch = arr.slice(i, i + BATCH_SIZE);
+    LOG.debug({ batch });
+    let res = await Promise.all(batch.map(get));
+    LOG.debug('batch fetched...');
+    res = post_filter_func ? res.filter(post_filter_func) : res;
+    LOG.debug_string({ res });
+    result.push(...res);
+  }
+  return result;
+}
+
+const extract_repos = (arr) =>
+  arr
+    .map((e) => e.substr('https://github.com/'.length).split('/'))
+    .filter((r) => r.length === 2 && r[1] !== '');
+
+const generate_GQL_query = (arr) =>
+  `query AWESOME_REPOS{ ${arr
+    .map(
+      ([owner, name]) =>
+        `repo_${owner.replace(/(-|\.)/g, '_')}_${name.replace(
+          /(-|\.)/g,
+          '_',
+        )}: repository(owner: "${owner}", name:"${name}"){ nameWithOwner } `,
+    )
+    .join('')} }`;
+
+// =============================================================
+// const batch_github_repos = async (github_links) => {
+//   const BATCH_SIZE = 50;
+//   const repos = extract_repos(github_links);
+//   for (let i = 0; i < repos.length; i += BATCH_SIZE) {
+//     const batch = repos.slice(i, i + BATCH_SIZE);
+//     const query = generate_GQL_query(batch);
+//     LOG.debug({ query });
+//     const gql_response = await fetch(
+//       'https://api.github.com/graphql',
+//       make_GQL_options(query),
+//     ).then((r) => r.json());
+//     LOG.debug({ gql_response });
+//   }
+// };
+// =============================================================
+
+async function main() {
+  const markdown = await fs.readFile(README, 'utf8');
+  const links = extract_all_links(markdown);
+  const duplicates = find_duplicates(links);
+  if (duplicates.length > 0) {
+    LOG.error_string({ duplicates });
+  }
+  const [github_links, other_links] = partition(links, (link) =>
+    link.startsWith('https://github.com'),
+  );
+
+  const other_links_error = await batch_fetch({
+    arr: other_links,
+    get: fetch_link,
+    post_filter_func: (x) => !x[1].ok,
+    BATCH_SIZE: 8,
+  });
+  if (other_links_error.length > 0) {
+    LOG.error_string({ other_links_error });
+  }
+
+  const repos = extract_repos(github_links);
+  const query = generate_GQL_query(repos);
+  const options = make_GQL_options(query);
+  const gql_response = await fetch(GITHUB_GQL_API, options).then((r) =>
+    r.json(),
+  );
+
+  const { data } = gql_response;
+  if (gql_response.errors) {
+    LOG.error_string({ errors: gql_response.errors });
+  }
+  const repos_fetched = Object.entries(data)
+    .map(([, /* k , */ v]) => v.nameWithOwner)
+    .sort((a, b) => b - a);
+
+  console.log({ repos_fetched: repos_fetched.length });
+}
+
+console.log('starting...');
+main();