change to module

tests/common.mjs

@@ -0,0 +1,102 @@
+import fetch from 'node-fetch';
+import exclude from './exclude_in_test.json';
+
+const LINKS_OPTIONS = {
+  redirect: 'error',
+  headers: {
+    'Content-Type': 'application/json',
+    'user-agent':
+      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36',
+  },
+};
+
+const LOG = {
+  error: (...args) => console.error('❌ ERROR', args),
+  error_string: (...args) =>
+    console.error('❌ ERROR', JSON.stringify({ ...args }, null, '  ')),
+  debug: (...args) => {
+    if (process.env.DEBUG) console.log('>>> DEBUG: ', { ...args });
+  },
+  debug_string: (...args) => {
+    if (process.env.DEBUG)
+      console.log('>>> DEBUG: ', JSON.stringify({ ...args }, null, '  '));
+  },
+};
+
+const handleFailure = (error) => {
+  console.error(`${error.message}: ${error.stack}`, { error });
+  process.exit(1);
+};
+
+process.on('unhandledRejection', handleFailure);
+
+const extract_all_links = (markdown) => {
+  // if you have a problem and you try to solve it with a regex,
+  // now you have two problems
+  // TODO: replace this mess with a mardown parser ?
+  const re = /(((https:(?:\/\/)?)(?:[-;:&=+$,\w]+@)?[A-Za-z0-9.-]+|(?:www\.|[-;:&=+$,\w]+@)[A-Za-z0-9.-]+)((?:\/[+~%/.\w\-_]*)?\??(?:[-+=&;%@.\w_]*)#?(?:[.!/@\-\\\w]*))?)/g;
+  return markdown.match(re);
+};
+
+const find_duplicates = (arr) => {
+  const hm = {};
+  const dup = [];
+  arr.forEach((e) => {
+    if (hm[e]) dup.push(e);
+    else hm[e] = true;
+  });
+  return dup;
+};
+
+const partition = (arr, func) => {
+  const ap = [[], []];
+  arr.forEach((e) => (func(e) ? ap[0].push(e) : ap[1].push(e)));
+  return ap;
+};
+
+async function fetch_link(url) {
+  try {
+    const { ok, statusText, redirected } = await fetch(url, LINKS_OPTIONS);
+    return [url, { ok, status: statusText, redirected }];
+  } catch (error) {
+    return [url, { ok: false, status: error.message }];
+  }
+}
+
+async function batch_fetch({ arr, get, post_filter_func, BATCH_SIZE = 8 }) {
+  const result = [];
+  /* eslint-disable no-await-in-loop */
+  for (let i = 0; i < arr.length; i += BATCH_SIZE) {
+    const batch = arr.slice(i, i + BATCH_SIZE);
+    LOG.debug_string({ batch });
+    let res = await Promise.all(batch.map(get));
+    console.log(`batch fetched...${i + BATCH_SIZE}`);
+    res = post_filter_func ? res.filter(post_filter_func) : res;
+    LOG.debug_string({ res });
+    result.push(...res);
+  }
+  return result;
+}
+
+const exclude_length = exclude.length;
+const exclude_from_list = (link) => {
+  let is_excluded = false;
+  for (let i = 0; i < exclude_length; i += 1) {
+    if (link.startsWith(exclude[i])) {
+      is_excluded = true;
+      break;
+    }
+  }
+  return is_excluded;
+};
+
+module.exports = {
+  LOG,
+  handleFailure,
+  extract_all_links,
+  find_duplicates,
+  partition,
+  fetch_link,
+  batch_fetch,
+  exclude_from_list,
+};