awesome-docker/tests/common.mjs

import fetch from 'node-fetch';
import { isRedirect } from 'node-fetch';
import {readFileSync} from 'fs';

const LINKS_OPTIONS = {
  redirect: 'manual',
  headers: {
    'Content-Type': 'application/json',
    'user-agent':
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
  },
  timeout: 60000, // 1m
  signal: AbortSignal.timeout(60000),
};

const LOG = {
  error: (...args) => console.error('❌ ERROR', args),
  error_string: (...args) =>
    console.error('❌ ERROR', JSON.stringify({ ...args }, null, '  ')),
  debug: (...args) => {
    if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args });
  },
  debug_string: (...args) => {
    if (process.env.DEBUG)
      console.log('>>> DEBUG: ', JSON.stringify({ ...args }, null, '  '));
  },
};

const handleFailure = (error) => {
  console.error(`${error.message}: ${error.stack}`, { error });
  process.exit(1);
};

process.on('unhandledRejection', handleFailure);

const extract_all_links = (markdown) => {
  // if you have a problem and you try to solve it with a regex,
  // now you have two problems
  // TODO: replace this mess with a mardown parser ?
  const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/@\-\\\w]*))?)/g;
  return markdown.match(re);
};

const find_duplicates = (arr) => {
  const hm = {};
  const dup = [];
  arr.forEach((e) => {
    if (hm[e]) dup.push(e);
    else hm[e] = true;
  });
  return dup;
};

const partition = (arr, func) => {
  const ap = [[], []];
  arr.forEach((e) => (func(e) ? ap[0].push(e) : ap[1].push(e)));
  return ap;
};

async function fetch_link(url) {
  try {
    const { headers, ok, status, statusText } = await fetch(url, LINKS_OPTIONS);
    const redirect = isRedirect(status) ? { redirect: { src: url, dst: headers.get("location") } } : {};
    return [url, { ok, status: statusText, ...redirect }];
  } catch (error) {
    return [url, { ok: false, status: error.message }];
  }
}

async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) {
  const result = [];
  /* eslint-disable no-await-in-loop */
  for (let i = 0; i < arr.length; i += BATCH_SIZE) {
    const batch = arr.slice(i, i + BATCH_SIZE);
    LOG.debug_string({ batch });
    let res = await Promise.all(batch.map(get));
    console.log(`batch fetched...${i + BATCH_SIZE}`);
    res = post_filter_func ? res.filter(post_filter_func) : res;
    LOG.debug_string({ res });
    result.push(...res);
  }
  return result;
}

const data = readFileSync('./tests/exclude_in_test.json')
const exclude = JSON.parse(data)
const exclude_length = exclude.length;
const exclude_from_list = (link) => {
  let is_excluded = false;
  for (let i = 0; i < exclude_length; i += 1) {
    if (link.startsWith(exclude[i])) {
      is_excluded = true;
      break;
    }
  }
  return is_excluded;
};

export default {
  LOG,
  handleFailure,
  extract_all_links,
  find_duplicates,
  partition,
  fetch_link,
  batch_fetch,
  exclude_from_list,
};