initial commit

app.py

@@ -0,0 +1,60 @@
+import timm
+from fastai.vision.all import *
+import gradio as gr
+import os
+
+import pathlib
+temp = pathlib.PosixPath
+pathlib.PosixPath = pathlib.WindowsPath
+
+
+title = 'LEGO sets&creations theme classifier'
+description = f'''
+# {title}
+This demo showcases the LEGO theme classifier built with the help of fast.ai. A model was trained using over 1800 images of sets released in 2005-19 scraped from the Brickset LEGO database.
+To test how much overfitting might be present due to the model memorizing the color(s) associated with a particular theme, I ran the training again using the same set of images, but in grayscale. Hence two available models.
+
+I was especially intrested in how the model will do on MOCS a.k.a. community creations, since the boundries between themes are not well-defined. Enjoy!
+'''
+
+themes = sorted(('City', 'Technic', 'Star-Wars', 'Creator', 'Ninjago', 'Architecture', 'Duplo', 'Friends', 'DC-Comics-Super-Heroes'))
+learn_color = load_learner('models/lego_convnext_small_4ep_sets05-19.pkl')
+learn_gray = load_learner('models/lego_convnext_small_4ep_grayscale.pkl')
+
+def classify(img, is_color):
+    if is_color == 'Grayscale model':
+        _, _, probs = learn_gray.predict(img)
+    else:
+        _, _, probs = learn_color.predict(img)
+    return dict(zip(themes, map(float, probs)))
+
+
+examples_sets = [[f'images/sets/{img_name}', img_name.split('2', 1)[0].capitalize(), img_name.split('.', 1)[0][-4:]] for img_name in os.listdir('images/sets')]
+examples_mocs = [['images/mocs/modernlibrary.jpg', 'Modern library MOC'],
+                 ['images/mocs/keanu.jpg', 'Keanu Reeves himself'],
+                 ['images/mocs/solaris.jfif', 'Solaris Urbino articulated bus'],
+                 ['images/mocs/aroundtheworld.jpg', '"Around the World" MOC'],
+                 ['images/mocs/walkingminicooper.jpg', 'Walking mini cooper. Yes, walking mini cooper']]
+
+with gr.Blocks() as app:
+    gr.Markdown(description)
+    with gr.Row(equal_height=True):
+        with gr.Column():
+            img = gr.components.Image(shape=(192, 192), label="Input image")
+            is_color = gr.components.Radio(['Color model', 'Grayscale model'], value='Color model', show_label=False)
+            real_label = gr.components.Textbox("", label='Real theme', interactive=False)
+            run_btn = gr.Button("Predict!")
+            # placeholders for additional info
+            name = gr.components.Textbox("", label='Name', visible=False)
+            year = gr.components.Textbox("", label='Release year', visible=False)
+        with gr.Column():
+            prediction = gr.components.Label(label='Prediction')
+    with gr.Row():
+        with gr.Column():
+            ex_sets = gr.Examples(examples_sets, inputs=[img, real_label, year], outputs=prediction, label='Examples - official sets')
+        with gr.Column():
+            ex_mocs = gr.Examples(examples_mocs, inputs=[img, name], outputs=prediction, label='Examples - community creations')
+
+    run_btn.click(fn=classify, inputs=[img, is_color], outputs=prediction)
+
+app.launch()

deploy.ipynb

@@ -0,0 +1,397 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import timm\n",
+    "from fastai.vision.all import *\n",
+    "import gradio as gr\n",
+    "import os\n",
+    "\n",
+    "import pathlib\n",
+    "temp = pathlib.PosixPath\n",
+    "pathlib.PosixPath = pathlib.WindowsPath"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "themes = sorted(('City', 'Technic', 'Star-Wars', 'Creator', 'Ninjago', 'Architecture', 'Duplo', 'Friends', 'DC-Comics-Super-Heroes'))\n",
+    "learn_color = load_learner('models/lego_convnext_small_4ep_sets05-19.pkl')\n",
+    "learn_gray = load_learner('models/lego_convnext_small_4ep_grayscale.pkl')\n",
+    "\n",
+    "def classify(img, *args):\n",
+    "    if args[-1] == 'Color mode':\n",
+    "        _, _, probs = learn_color.predict(img)\n",
+    "    else:\n",
+    "        _, _, probs = learn_gray.predict(img)\n",
+    "    return dict(zip(themes, map(float, probs)))\n",
+    "\n",
+    "\n",
+    "img = gr.components.Image(shape=(192, 192), label=\"Input image\")\n",
+    "is_color = gr.components.Radio(['Color mode', 'Grayscale mode'], value='Color mode', show_label=False)\n",
+    "real_label = gr.components.Textbox(\"\", label='Theme', interactive=False)\n",
+    "year = gr.components.Textbox(\"\", label='Release year', visible=False)\n",
+    "\n",
+    "label = gr.components.Label(label='Predictions')\n",
+    "examples = [[f'test_images/{img_name}', img_name.split('2', 1)[0].capitalize(), img_name.split('.', 1)[0][-4:]] for img_name in os.listdir('test_images')]\n",
+    "\n",
+    "gr.Interface(fn=classify, inputs=[img, real_label, year, is_color], outputs=label, examples=examples).launch(\n",
+    "    # inline=False\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "title = 'LEGO sets&creations theme classifier'\n",
+    "description = f'''\n",
+    "# {title}\n",
+    "This demo showcases the LEGO theme classifier built with the help of fast.ai. A model was trained using over 1800 images of sets released in 2005-19 scraped from the Brickset LEGO database.\n",
+    "To test how much overfitting might be present due to the model memorizing the color(s) associated with a particular theme, I ran the training again using the same set of images, but in grayscale. Hence two available models.\n",
+    "\n",
+    "I was especially intrested in how the model will do on MOCS a.k.a. community creations, since the boundries between themes are not well-defined. Enjoy!\n",
+    "'''"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7867\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7867/\" width=\"900\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": [
+       "(<gradio.routes.App at 0x18df584c880>, 'http://127.0.0.1:7867/', None)"
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "<style>\n",
+       "    /* Turns off some styling */\n",
+       "    progress {\n",
+       "        /* gets rid of default border in Firefox and Opera. */\n",
+       "        border: none;\n",
+       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "        background-size: auto;\n",
+       "    }\n",
+       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
+       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
+       "    }\n",
+       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "        background: #F44336;\n",
+       "    }\n",
+       "</style>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "<style>\n",
+       "    /* Turns off some styling */\n",
+       "    progress {\n",
+       "        /* gets rid of default border in Firefox and Opera. */\n",
+       "        border: none;\n",
+       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "        background-size: auto;\n",
+       "    }\n",
+       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
+       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
+       "    }\n",
+       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "        background: #F44336;\n",
+       "    }\n",
+       "</style>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "<style>\n",
+       "    /* Turns off some styling */\n",
+       "    progress {\n",
+       "        /* gets rid of default border in Firefox and Opera. */\n",
+       "        border: none;\n",
+       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "        background-size: auto;\n",
+       "    }\n",
+       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
+       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
+       "    }\n",
+       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "        background: #F44336;\n",
+       "    }\n",
+       "</style>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "<style>\n",
+       "    /* Turns off some styling */\n",
+       "    progress {\n",
+       "        /* gets rid of default border in Firefox and Opera. */\n",
+       "        border: none;\n",
+       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "        background-size: auto;\n",
+       "    }\n",
+       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
+       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
+       "    }\n",
+       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "        background: #F44336;\n",
+       "    }\n",
+       "</style>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "<style>\n",
+       "    /* Turns off some styling */\n",
+       "    progress {\n",
+       "        /* gets rid of default border in Firefox and Opera. */\n",
+       "        border: none;\n",
+       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "        background-size: auto;\n",
+       "    }\n",
+       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
+       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
+       "    }\n",
+       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "        background: #F44336;\n",
+       "    }\n",
+       "</style>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "\n",
+       "<style>\n",
+       "    /* Turns off some styling */\n",
+       "    progress {\n",
+       "        /* gets rid of default border in Firefox and Opera. */\n",
+       "        border: none;\n",
+       "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
+       "        background-size: auto;\n",
+       "    }\n",
+       "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
+       "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
+       "    }\n",
+       "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
+       "        background: #F44336;\n",
+       "    }\n",
+       "</style>\n"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "themes = sorted(('City', 'Technic', 'Star-Wars', 'Creator', 'Ninjago', 'Architecture', 'Duplo', 'Friends', 'DC-Comics-Super-Heroes'))\n",
+    "learn_color = load_learner('models/lego_convnext_small_4ep_sets05-19.pkl')\n",
+    "learn_gray = load_learner('models/lego_convnext_small_4ep_grayscale.pkl')\n",
+    "\n",
+    "def classify(img, is_color):\n",
+    "    if is_color == 'Grayscale model':\n",
+    "        _, _, probs = learn_gray.predict(img)\n",
+    "    else:\n",
+    "        _, _, probs = learn_color.predict(img)\n",
+    "    return dict(zip(themes, map(float, probs)))\n",
+    "\n",
+    "\n",
+    "examples_sets = [[f'images/sets/{img_name}', img_name.split('2', 1)[0].capitalize(), img_name.split('.', 1)[0][-4:]] for img_name in os.listdir('images/sets')]\n",
+    "examples_mocs = [['images/mocs/modernlibrary.jpg', 'Modern library MOC'],\n",
+    "                 ['images/mocs/keanu.jpg', 'Keanu Reeves himself'],\n",
+    "                 ['images/mocs/solaris.jfif', 'Solaris Urbino articulated bus'],\n",
+    "                 ['images/mocs/aroundtheworld.jpg', '\"Around the World\" MOC'],\n",
+    "                 ['images/mocs/walkingminicooper.jpg', 'Walking mini cooper. Yes, walking mini cooper']]\n",
+    "\n",
+    "with gr.Blocks() as app:\n",
+    "    gr.Markdown(description)\n",
+    "    with gr.Row(equal_height=True):\n",
+    "        with gr.Column():\n",
+    "            img = gr.components.Image(shape=(192, 192), label=\"Input image\")\n",
+    "            is_color = gr.components.Radio(['Color model', 'Grayscale model'], value='Color model', show_label=False)\n",
+    "            real_label = gr.components.Textbox(\"\", label='Real theme', interactive=False)\n",
+    "            run_btn = gr.Button(\"Predict!\")\n",
+    "            # placeholders for additional info\n",
+    "            name = gr.components.Textbox(\"\", label='Name', visible=False)\n",
+    "            year = gr.components.Textbox(\"\", label='Release year', visible=False)\n",
+    "        with gr.Column():\n",
+    "            prediction = gr.components.Label(label='Prediction')\n",
+    "    with gr.Row():\n",
+    "        with gr.Column():\n",
+    "            ex_sets = gr.Examples(examples_sets, inputs=[img, real_label, year], outputs=prediction, label='Examples - official sets')\n",
+    "        with gr.Column():\n",
+    "            ex_mocs = gr.Examples(examples_mocs, inputs=[img, name], outputs=prediction, label='Examples - community creations')\n",
+    "\n",
+    "    run_btn.click(fn=classify, inputs=[img, is_color], outputs=prediction)\n",
+    "\n",
+    "app.launch()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.10.4 ('ml')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.4"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "575b27b03c8f4938561cc9027b66655be84e7082a51e87d8eb0fbf4ab5514768"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

flagged/Predictions/tmpewniup3s.json

@@ -0,0 +1 @@
+{"label": "Star-Wars", "confidences": [{"label": "Star-Wars", "confidence": 0.9999582767486572}, {"label": "Creator", "confidence": 2.1909216229687445e-05}, {"label": "Architecture", "confidence": 1.8172977434005588e-05}, {"label": "Technic", "confidence": 8.87583098574396e-07}, {"label": "DC-Comics-Super-Heroes", "confidence": 6.561281793437956e-07}, {"label": "Ninjago", "confidence": 4.70658036988425e-08}, {"label": "Duplo", "confidence": 4.34802096549447e-08}, {"label": "Friends", "confidence": 2.3506199653411386e-08}, {"label": "City", "confidence": 2.532207221683791e-10}]}

flagged/Predictions/tmpy12e8hcc.json

@@ -0,0 +1 @@
+{"label": "Star-Wars", "confidences": [{"label": "Star-Wars", "confidence": 0.9999582767486572}, {"label": "Creator", "confidence": 2.1909216229687445e-05}, {"label": "Architecture", "confidence": 1.8172977434005588e-05}, {"label": "Technic", "confidence": 8.87583098574396e-07}, {"label": "DC-Comics-Super-Heroes", "confidence": 6.561281793437956e-07}, {"label": "Ninjago", "confidence": 4.70658036988425e-08}, {"label": "Duplo", "confidence": 4.34802096549447e-08}, {"label": "Friends", "confidence": 2.3506199653411386e-08}, {"label": "City", "confidence": 2.532207221683791e-10}]}

flagged/log.csv

@@ -0,0 +1,6 @@
+img,output,flag,username,timestamp
+h:\machine learning\legoclassification\flagged\img\tmprm6d_bsn.jpg,h:\machine learning\legoclassification\flagged\output\tmpunv7yoa4.json,,,2022-09-21 21:34:50.877120
+h:\machine learning\legoclassification\flagged\img\tmp6dzn1kjt.jpg,h:\machine learning\legoclassification\flagged\output\tmpnd3lfp3g.json,,,2022-09-21 21:34:52.176488
+h:\machine learning\legoclassification\flagged\img\tmp92aivevy.jpg,h:\machine learning\legoclassification\flagged\output\tmpsvbuiqim.json,,,2022-09-21 21:34:52.578412
+h:\machine learning\lego_classification\flagged\Input image\tmpal_f3hxe.jpg,Starwars,2021,Color mode,h:\machine learning\lego_classification\flagged\Predictions\tmpy12e8hcc.json,,,2022-09-28 23:30:41.377124
+h:\machine learning\lego_classification\flagged\Input image\tmphw36rajo.jpg,Starwars,2021,Color mode,h:\machine learning\lego_classification\flagged\Predictions\tmpewniup3s.json,,,2022-09-28 23:30:42.359274

flagged/output/tmpnd3lfp3g.json

@@ -0,0 +1 @@
+{"label": "Architecture", "confidences": [{"label": "Architecture", "confidence": 0.9984230995178223}, {"label": "Duplo", "confidence": 0.0009100447059608996}, {"label": "DC-Comics-Super-Heroes", "confidence": 0.0003492557443678379}, {"label": "Technic", "confidence": 0.00022252422058954835}, {"label": "Star-Wars", "confidence": 4.270664067007601e-05}, {"label": "City", "confidence": 4.061772415298037e-05}, {"label": "Ninjago", "confidence": 5.997831976856105e-06}, {"label": "Creator", "confidence": 5.755870461143786e-06}, {"label": "Friends", "confidence": 1.5410030584916967e-07}]}

flagged/output/tmpsvbuiqim.json

@@ -0,0 +1 @@
+{"label": "Architecture", "confidences": [{"label": "Architecture", "confidence": 0.9984230995178223}, {"label": "Duplo", "confidence": 0.0009100447059608996}, {"label": "DC-Comics-Super-Heroes", "confidence": 0.0003492557443678379}, {"label": "Technic", "confidence": 0.00022252422058954835}, {"label": "Star-Wars", "confidence": 4.270664067007601e-05}, {"label": "City", "confidence": 4.061772415298037e-05}, {"label": "Ninjago", "confidence": 5.997831976856105e-06}, {"label": "Creator", "confidence": 5.755870461143786e-06}, {"label": "Friends", "confidence": 1.5410030584916967e-07}]}

flagged/output/tmpunv7yoa4.json

@@ -0,0 +1 @@
+{"label": "Architecture", "confidences": [{"label": "Architecture", "confidence": 0.9984230995178223}, {"label": "Duplo", "confidence": 0.0009100447059608996}, {"label": "DC-Comics-Super-Heroes", "confidence": 0.0003492557443678379}, {"label": "Technic", "confidence": 0.00022252422058954835}, {"label": "Star-Wars", "confidence": 4.270664067007601e-05}, {"label": "City", "confidence": 4.061772415298037e-05}, {"label": "Ninjago", "confidence": 5.997831976856105e-06}, {"label": "Creator", "confidence": 5.755870461143786e-06}, {"label": "Friends", "confidence": 1.5410030584916967e-07}]}

models/lego_convnext_small_4ep_grayscale.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:00104f57d372faa8f35867dde04e7c1ef748dc5248a951d261586dcaa6485400
+size 201271983

models/lego_convnext_small_4ep_sets05-19.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ca1ebea93afe6b9c6b7209f9087073750f4cec29defe1bd40c6165b77255a31
+size 201278191