add dockerfiles

Signed-off-by: Niv Sardi <xaiki@evilgiggle.com>

Dockerfile.famaf

@@ -0,0 +1,14 @@
+FROM docker.io/nvidia/cuda:11.4.3-runtime-ubuntu20.04
+MAINTAINER Niv Sardi <x@filtra.me>
+WORKDIR /app
+
+RUN apt update && \
+    export DEBIAN_FRONTEND=noninteractive && \
+    apt install -y libcairo2 libglib2.0-0 libgl1 python3-pip && \
+    rm -rf /var/cache/apt
+COPY ./yolov5/requirements.txt ./requirements.txt
+RUN pip install -r ./requirements.txt --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111
+RUN pip3 install torch==1.8.2 torchvision==0.9.2 torchaudio==0.8.2 --extra-index-url https://download.pytorch.org/whl/lts/1.8/cu111
+
+CMD sh -c 'while true; do  echo "🐍 waiting" ; sleep 2h; done'
+

deno/index.ts

@@ -4,65 +4,9 @@ import { Application, Router } from "https://deno.land/x/oak@v9.0.0/mod.ts";
 import * as CSV from './csv.ts';
 import Puppet from './puppet.ts';
 import selectors from './selectors.ts';
+import { fetch_logos } from './logos.ts';
 
 const puppet = new Puppet();
-async function get_logos(page, selector): {}[] {
-  const logos = await page.$$(selector) || [];
-  for (const i in logos) {
-    const bb = await page.evaluate(e => {
-      const { x, y, width, height } = e.getBoundingClientRect();
-      return {
-        x, y, width, height, top: window.screen.top, left: window.screen.left
-      }
-    }, logos[i])
-    logos[i].box = bb;
-  }
-  return logos;
-}
-
-async function fetch_logos(page, id, dest) {
-  console.error(`getting logos for: ${id}`)
-        try {
-          const imgs = await get_logos(page, selectors.img_logo);
-          const ids = await get_logos(page, selectors.id_logo);
-          const cls = await get_logos(page, selectors.class_logo);
-          const logos = [
-            ...imgs, ...ids, ...cls
-          ]
-
-          let annotations = '';
-          for (const i in logos) {
-            const bb = logos[i].box
-            if (!bb
-              || (bb.width < 10)
-              || (bb.height < 10)
-              || (bb.x + bb.width < 0)
-              || (bb.y + bb.height < 0)) continue;
-            console.log('got bb',  bb)
-
-            try {
-              await logos[i].screenshot({
-                path: dest
-                  .replace('images', 'logos')
-                  .replace('.png', `.${i}.png`)
-              })
-              annotations +=
-                `${id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
-            } catch (e) {
-              console.error(`couldn't screenshot logo: ${e}`);
-            }
-          }
-          if (logos.length) {
-            await Deno.writeTextFile(dest
-              .replace('images', 'labels')
-              .replace('png', 'txt'),
-                                     annotations);
-        }
-        } catch (err) {
-          console.error(`error in screenshot: ${err}`);
-        }
-}
-
 const app = new Application();
 const router = new Router();
 
@@ -70,33 +14,52 @@ const stats = {
   in_flight: 0,
   done: 0
 }
+
+export const DEFAULT_VIEWPORTS = [{
+  width: 640,
+  height: 480
+}, {
+  width: 1080,
+  height: 800
+}, {
+  width: 640,
+  height: 640
+}, {
+  width: 600,
+  height: 800,
+  hasTouch: true,
+  isMobile: true
+}]
+
 router.post('/screenshot', async (ctx) => {
-  const {request, response} = ctx;
-  const q = await request.body().value;
+  const { request, response } = ctx;
+  const { url, path = './debug.png', logos = false, viewports = DEFAULT_VIEWPORTS } = await request.body().value;
 
   stats.in_flight++;
-  const ret = await puppet.run(async page => {
-    console.error('running', q, stats)
-    await page.goto(q.url, {waitUntil: 'networkidle2', timeout: 60000})
-    await page.screenshot({ path: q.path, fullPage: true })
-    if (q.logos) {
-      await fetch_logos(page, q.id, q.logos)
-    }
-    console.error(`screenshot ok: ${q.path}`)
-    return {response: 'ok'}
+  let i = 0, v = {};
+  viewports.map(async (v, i) => {
+    const ret = await puppet.run(async page => {
+      await page.setViewport(v)
+      console.error('running', url, stats, v)
+      try {
+        const npath = path.replace('.png', `.${i}.png`)
+
+        await page.goto(url, { waitUntil: 'networkidle2', timeout: 60000 })
+        await page.screenshot({ path: npath, fullPage: true })
+        if (logos) {
+          await fetch_logos(page, npath)
+        }
+        console.error(`screenshot ${v} / ${i} ok: ${path}`)
+      } catch (e) {
+        console.error(e)
+      }
+      return { response: 'ok' }
+    })
   })
   stats.in_flight--;
   stats.done++
-  response.body = ret
+  response.body = { response: 'ok' }
 })
-router.post('/bco', async (ctx) => {
-  const {request, response} = ctx;
-  const q = await request.body().value;
-  const ret = await process(q)
-
-  console.error(`ret: ${ret}`)
-  response.body = ret
-});
 
 app.use(router.routes())
 app.use(router.allowedMethods())

deno/logos.ts

@@ -0,0 +1,58 @@
+import selectors from './selectors.ts';
+
+export async function get_logos(page, selector): {}[] {
+    const logos = await page.$$(selector) || [];
+    for (const i in logos) {
+        const bb = await page.evaluate(e => {
+            const { x, y, width, height } = e.getBoundingClientRect();
+            return {
+                x, y, width, height, top: window.screen.top, left: window.screen.left
+            }
+        }, logos[i])
+        logos[i].box = bb;
+    }
+    return logos;
+}
+
+export async function fetch_logos(page, dest) {
+    console.error(`getting logos for: ${dest}`)
+    try {
+        const imgs = await get_logos(page, selectors.img_logo);
+        const ids = await get_logos(page, selectors.id_logo);
+        const cls = await get_logos(page, selectors.class_logo);
+        const logos = [
+            ...imgs, ...ids, ...cls
+        ]
+
+        let annotations = '';
+        for (const i in logos) {
+            const bb = logos[i].box
+            if (!bb
+                || (bb.width < 10)
+                || (bb.height < 10)
+                || (bb.x + bb.width < 0)
+                || (bb.y + bb.height < 0)) continue;
+            console.log('got bb', bb)
+
+            try {
+                await logos[i].screenshot({
+                    path: dest
+                        .replace('images', 'logos')
+                        .replace('.png', `.${i}.png`)
+                })
+                annotations +=
+                    `${o.id} ${bb.x + bb.width / 2} ${bb.y + bb.height / 2} ${bb.width} ${bb.height}\n`
+            } catch (e) {
+                console.error(`couldn't screenshot logo: ${e}`);
+            }
+        }
+        if (logos.length) {
+            await Deno.writeTextFile(dest
+                .replace('images', 'labels')
+                .replace('png', 'txt'),
+                annotations);
+        }
+    } catch (err) {
+        console.error(`error in screenshot: ${err}`);
+    }
+}

deno/process.ts

@@ -0,0 +1,34 @@
+router.post('/bco', async (ctx) => {
+    const { request, response } = ctx;
+    const q = await request.body().value;
+    const ret = await process(q)
+
+    console.error(`ret: ${ret}`)
+    response.body = ret
+})
+
+function process(o: { id: int, url: string, bco: string, name: string }): Promise<void> {
+    const promises: Promise<void>[] = [];
+
+    return puppet.run(async page => {
+        const url = o.url.replace('http:', 'https:');
+        promises.push(new Promise<void>((accept, _reject) => {
+            page.once('load', async () => {
+                const dest = `./data/images/${e.bco}.chrome.full.png`
+                await fetch_logos(page, bco.name, dest)
+                await page.screenshot({ path: dest, fullPage: true })
+                console.log(`screenshot ok for ${o.name}`);
+
+                accept()
+            })
+        }))
+
+        try {
+            await page.goto(url)
+                .catch(() => page.goto(o.url))
+        } catch (e) {
+            console.error(`got error: ${e}`);
+        }
+        await Promise.all(promises);
+    })
+}

poormans-nc.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+PORT=1234
+while true; do
+      perl -MIO::Socket::INET -ne "BEGIN{\$l=IO::Socket::INET->new(
+        LocalPort=>$PORT,Proto=>\"tcp\",Listen=>5,ReuseAddr=>1);
+          \$l=\$l->accept} die"
+      ps auxf | grep geckodriver | grep -v sh | grep -v grep | awk "{print $2}" | xargs -n1 kill
+      sleep 1
+done