Expand cleanup

Makefile

+out = clean_data
+
+clean:
+	@$(RM) -rf ./$(out)/*
\ No newline at end of file

src/cleanup.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Cleanup\n",
+    "This file is dedicated to data cleanup. See the corresponding markdown headers to get more information on the steps applied"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 425,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Imports and constants\n",
+    "import pandas, os, re, requests, json, math, numbers\n",
+    "from lib.scraping import get_course, get_complete_catalogue_for\n",
+    "import lib.constants as CONSTANTS"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 426,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def normalize_type(type: str) -> str:\n",
+    "\tnormalized_identifiers = {\n",
+    "\t\t\"V\": \"VL\",\n",
+    "\t\t\"Vl\": \"VL\",\n",
+    "\t\t\"VL+Ü\": \"VL\",\n",
+    "\t\t\"VLeKo\": \"VL\",\n",
+    "\t\t\"SLeKo\": \"S\"\n",
+    "\t}\n",
+    "\t# Using get prevents KeyErrors by types that already are right\n",
+    "\treturn normalized_identifiers.get(type, type)\n",
+    "\n",
+    "def make_plural(category: str) -> str:\n",
+    "\tplural_list = {\n",
+    "\t\t\"Vorlesung\": \"Vorlesungen\",\n",
+    "\t\t\"Übung\": \"Übungen\",\n",
+    "\t\t\"Seminar\": \"Seminare\"\n",
+    "\t}\n",
+    "\n",
+    "\treturn plural_list.get(category, category)\n",
+    "\n",
+    "def request_questions():\n",
+    "\tres = requests.get(\"http://localhost:50000/v1/questions\")\n",
+    "\treturn json.loads(res.text)\n",
+    "\n",
+    "def choose(a,b):\n",
+    "\tif type(a) == str:\n",
+    "\t\treturn b if a == \"\" else a\n",
+    "\telif isinstance(a, numbers.Number):\n",
+    "\t\treturn b if math.isnan(a) else a\n",
+    "\telse:\n",
+    "\t\treturn b if a == None else a"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Generate Paths\n",
+    "Converts all files in the `raw_data` folder into pandas dataframes"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 427,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "paths = []\n",
+    "\n",
+    "for (root, dirs, files) in os.walk(CONSTANTS.RAW_DATA_PATH):\n",
+    "\tif len(files) > 0 and 'daten.csv' in files:\n",
+    "\t\tpaths.append((root, pandas.read_csv(os.path.join(root, 'daten.csv'), sep=\";\", encoding=\"ISO-8859-1\")))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Remove unnecessary columns\n",
+    "Some columns (like text answers) are not needed for the pipeline and get excluded here."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 428,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "to_remove = [\"language\", \"Zulassungsbeschr.\", \"Bearbeitungsstand\", \"zeit\", \"Ausfülldauer\", \"Ausfülldauer (s)\"]\n",
+    "multiples = [\"Was hat Ihnen an dieser Lehrveranstaltung gut gefallen?\", \"Was könnte der Dozent/die Dozentin an dieser Lehrveranstaltung verbessern?\"]\n",
+    "\n",
+    "# Easier handling for doubles columns\n",
+    "for i in range(1, 8):\n",
+    "\tfor j in multiples:\n",
+    "\t\tto_remove.append(j + \".\" + str(i))\n",
+    "to_remove.extend(multiples)\n",
+    "\n",
+    "for (path, df) in paths:\n",
+    "\tto_drop = [column for column in to_remove if column in df]\n",
+    "\tdf.drop(columns=to_drop, inplace=True, axis=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Normalize Course Type\n",
+    "The pipeline expects four questionnaire types `VL` for lectures, `S` for seminars, `Ü` for tutorials and `LeKo` for teaching competence. The provided data has multiple possible notations. \n",
+    "- It is quite common to see `VL`, `Vl` or `V` for lectures. In small cases they are even denoted by `VL + Ü`. To unify these types, they will all be converted to `VL`. \n",
+    "- Teaching competence has multiple ways to be categorized. Older data may have `VLeKo` or `SLeKo` for lectures or seminars evaluated according to teaching competence. This will be normalized to `LeKo`. In this case the course type will be changed in the corresponding course type column, to preserve the original type of the course"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 429,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for (path, df) in paths:\n",
+    "\tdf[\"Veranstaltung-Typ\"] = df[\"Veranstaltung-Typ\"].map(normalize_type)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Normalize Course Category\n",
+    "The pipeline handles the three course types lectures (`Vorlesungen`), tutorials (`Übungen`) and seminars (`Seminare`). The provided date knows a few more types.\n",
+    "- Tutorials are mostly noted with the singular `Übung`. This gets translated into plural to keep it in line with the other types, which all are plural\n",
+    "- Data has a type called `Projekte / Praktika`, which don't have an own questionnaire type, so the name gets normalized to the questionnaire type it belongs to\n",
+    "### Problems:\n",
+    "- if a course has `LeKo` as a questionnaire and course type, it's impossible to handle. Possible Solution: VV Webscraper\n",
+    "- if a course has `Prak` as questionnaire type and `Projekte / Praktika` as course type, it can't be categorized. Possible Solution: Always count as seminar"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 430,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "normalized_categories = {\n",
+    "\t\"VL\": \"Vorlesungen\",\n",
+    "\t\"Ü\": \"Übungen\",\n",
+    "\t\"S\": \"Seminare\"\n",
+    "}\n",
+    "for (path, df) in paths:\n",
+    "\t# Iterates over dataframe rows\n",
+    "\tfor i in df.index:\n",
+    "\t\tvalue = df.loc[i]\n",
+    "\n",
+    "\t\t# Normalized courses can be skipped\n",
+    "\t\tif value.loc[\"Veranstaltung-Kategorie\"] in [\"Vorlesungen\", \"Übungen\", \"Seminare\"]: continue\n",
+    "\n",
+    "\t\tevaluation_type = value.loc[\"Veranstaltung-Typ\"]\n",
+    "\t\t\n",
+    "\t\tif evaluation_type in normalized_categories:\n",
+    "\t\t\tdf.at[i, \"Veranstaltung-Kategorie\"] = normalized_categories[evaluation_type]\n",
+    "\t\telse:\n",
+    "\t\t\t# Pluralizing\n",
+    "\t\t\tdf.at[i, \"Veranstaltung-Kategorie\"] = make_plural(value.loc[\"Veranstaltung-Kategorie\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Fix Course Numbers\n",
+    "Sometimes Course Numbers contain a space at the end. This hinders comparing them with the same number that does not contain a space. This part removes leading and trailing spaces"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 431,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for (path, df) in paths:\n",
+    "\tdf[\"Veranstaltung-Nr.\"] = df[\"Veranstaltung-Nr.\"].map(lambda x: str(x).strip())"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Merge questions\n",
+    "Since all questions of all questionnaires are put into a row, some questions are duplicated. But since questions are the header row, they need to be unique. To counteract this, pandas appends numbers to duplicates, thus turning them unique. However a duplicated question most likely will not be answered twice, thus enabling to merge duplicated questions into a single column, where text takes precedence above empty cells."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 432,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "end_regex = re.compile('\\.\\d+$')\n",
+    "\n",
+    "for (path, df) in paths:\n",
+    "\tfor column in df:\n",
+    "\t\t# If this line matches, a sufficient requirement for a duplicated question is met\n",
+    "\t\tif (match := end_regex.search(column)) != None:\n",
+    "\t\t\tclean_name = column[:-(len(match.group()))]\n",
+    "\t \t\t# Merges duplicated column into the original, deletes duplicate\n",
+    "\t\t\tdf[clean_name] = df[clean_name].combine(df[column], choose)\n",
+    "\t\t\tdf.drop(columns=[column], inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Replace Questions with IDs\n",
+    "Takes the IDs from the api and converts the questions into them. This allows to extract the ID programmatically"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 433,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "questions = request_questions()\n",
+    "not_indexable = []\n",
+    "for (path, df) in paths:\n",
+    "\n",
+    "\tfor column in df:\n",
+    "\t\t# Strip is needed, because some questions have a trailing whitespace\n",
+    "\t\t# TODO: Document the regex somewhere\n",
+    "\t\tprepared_string = re.sub(r\"(^\\.+)|(\\.+$)|(\\s?\\?$)|(^\\?)\", \"\", column.strip())\n",
+    "\t\t# In a few files a single question exists with two spaces, this unifies them to one\n",
+    "\t\tprepared_string = ' '.join(prepared_string.split())\n",
+    "\n",
+    "\t\tid = [item for item in questions if item[\"content\"].replace(\"Der/die Lehrende \", \"\") == prepared_string]\n",
+    "\t\tif len(id) > 0:\n",
+    "\t\t\tname = id[0][\"id\"]\n",
+    "\t\t\tdf.rename(columns={column: name}, inplace=True)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Split into course types\n",
+    "Splits data into the four questionnaire types and writes them separately into files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 434,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for (path, df) in paths:\n",
+    "\tinformation = path.split(\"/\")[-1]\n",
+    "\t(institute, semester) = information.split(\"_\")\n",
+    "\tinstitute_path = os.path.join(CONSTANTS.CLEAN_DATA_PATH, institute)\n",
+    "\tsemester_path = os.path.join(institute_path, semester)\n",
+    "\n",
+    "\tif not os.path.isdir(institute_path):\n",
+    "\t\tos.mkdir(institute_path)\n",
+    "\tif not os.path.isdir(semester_path):\n",
+    "\t\tos.mkdir(semester_path)\n",
+    "\n",
+    "\tquestionnaire_types = [\"VL\", \"Ü\", \"S\", \"LeKo\"]\n",
+    "\n",
+    "\t# Separate into types\n",
+    "\tfor q_type in questionnaire_types:\n",
+    "\t\tres = df[df[\"Veranstaltung-Typ\"] == q_type]\n",
+    "\t\tresult_path = os.path.join(semester_path, q_type + \".csv\")\n",
+    "\n",
+    "\t\tres.to_csv(result_path, encoding=\"utf-8\", index=False)\n",
+    "\t\n",
+    "\t# If Outliers exist\n",
+    "\trest = df[~df[\"Veranstaltung-Typ\"].isin(questionnaire_types)]\n",
+    "\n",
+    "\tif len(rest.index) > 0:\n",
+    "\t\trest.to_csv(os.path.join(semester_path, \"rest.csv\"), encoding=\"utf-8\", index=False)\n",
+    "\n",
+    "\t# If everything needs to be handled at once\n",
+    "\tdf.to_csv(os.path.join(semester_path, \"all.csv\"), encoding=\"utf-8\", index=False)\n",
+    "\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/db/questions.json

@@ -99,7 +99,7 @@
 		["Das Thema der Lehrveranstaltung hat mich schon vorher interessiert", "scale_applies_alt"],
 		["Ich verfügte bereits vor dem Lehrveranstaltungsbesuch über umfangreiches Wissen zu den in der LV behandelten Themengebieten", "scale_applies_alt"],
 		["Ich verfügte bereits vor der Lehrveranstaltung über viel Erfahrung mit E-Learning", "scale_applies_alt"],
-		["Ich habe an mindestens zwei Drittel der Termine dieser Lehrveranstaltung teilgenommen bzw. die Inhalte Lerneinheiten bearbeitet", "y_n"],
+		["Ich habe an mindestens zwei Drittel der Termine dieser Lehrveranstaltung teilgenommen bzw. die Inhalte der Lerneinheiten bearbeitet", "y_n"],
 		["Der Stoffumfang, der in der Veranstaltung behandelt wird, ist für mich", "scale_height"],
 		["Der Schwierigkeitsgrad der Veranstaltung ist für mich", "scale_height"],
 		["Der/die Lehrende hat die gesamte Lehrveranstaltung gut strukturiert und nachvollziehbar gegliedert", "scale_applies_alt"],
@@ -112,10 +112,10 @@
 		["Der/die Lehrende fasst regelmäßig die wichtigsten Inhalte der LV zusammen", "scale_applies_alt"],
 		["Der/die Lehrende stellt zu Beginn eines Termins/einer Lerneinheit den Zusammenhang zu dem Termin/der Lerneinheit davor her", "scale_applies_alt"],
 		["Der/die Lehrende stellt immer wieder Bezüge zu bereits behandelten Lerninhalten her", "scale_applies_alt"],
-		["Der/die Lehrende stellt Fragen, die den Studierenden die Gelegenheit geben, zu überprüfen, ob sie den Inhalt verstanden haben", "scale_applies_alt"],
+		["Der/die Lehrende stellt Fragen, die den Studierenden die Gelegenheit geben zu überprüfen, ob sie den Inhalt verstanden haben", "scale_applies_alt"],
 		["Der/die Lehrende vergewissert sich, dass die Studierenden zentrale Aspekte verstanden haben, bevor er/sie im Stoff weitergeht", "scale_applies_alt"],
 		["Der/die Lehrende ist bei Fragen gut erreichbar", "scale_applies_alt"],
-		["Der/die Lehrende gibt mir ein konstruktives Feedback zu meinen Beitragen/Lösungen", "scale_applies_alt"],
+		["Der/die Lehrende gibt mir ein konstruktives Feedback zu meinen Beiträgen/Lösungen", "scale_applies_alt"],
 		["Der/die Lehrende gibt mir zu wenige Rückmeldungen zu meinen Beiträgen/Lösungen", "scale_applies_alt"],
 		["Der/die Lehrende gibt mir konkrete Hinweise zur Verbesserung meiner Leistungen", "scale_applies_alt"],
 		["Der/die Lehrende setzt Modelle, Graphiken oder Schemata so ein, dass sie das Verständnis komplexer Sachverhalte erleichtern", "scale_applies_alt"],
@@ -137,7 +137,7 @@
 		["Der/die Lehrende hat klare Verhaltensregeln für unsere Zusammenarbeit in dieser Lehrveranstaltung kommuniziert", "scale_applies_alt"],
 		["Der/die Lehrende achtet darauf, dass wir die aufgestellten Verhaltensregeln einhalten", "scale_applies_alt"],
 		["Der/die Lehrende bringt wichtige Inhalte gut auf den Punkt", "scale_applies_alt"],
-		["Der/die Lehrende nutzt die Verfügbare Zeit effektiv", "scale_applies_alt"],
+		["Der/die Lehrende nutzt die verfügbare Zeit effektiv", "scale_applies_alt"],
 		["In dieser Lehrveranstaltung habe ich viel dazugelernt", "scale_applies_alt"],
 		["Insgesamt bin ich mit dieser Lehrveranstaltung zufrieden", "scale_applies_alt"]
 	],

src/db/schema.sql

@@ -172,10 +172,10 @@ INSERT INTO "question" ("id", "content", "possibilities", "dimension") VALUES
 ('fb98204a-e7c6-11ee-9404-00620b2c9060', 'Der Umgang mit rein digitalen Kursinhalten im Vergleich zu regelmäßigen wöchentlichen Präsenzveranstaltungen ist', 'scale_difficulty', NULL),
 ('fb984109-e7c6-11ee-9404-00620b2c9060', 'Ich war an mindestens zwei Drittel der Termine dieser Lehrveranstaltung anwesend', 'y_n', NULL),
 ('fb985e1a-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende trägt anregend und engagiert vor', 'scale_applies', 'B2'),
-('fb987b71-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende stellt Fragen, die den Studierenden die Gelegenheit geben, zu überprüfen, ob sie den Inhalt verstanden haben', 'scale_applies', 'A2'),
+('fb987b71-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende stellt Fragen, die den Studierenden die Gelegenheit geben zu überprüfen, ob sie den Inhalt verstanden haben', 'scale_applies', 'A2'),
 ('fb989861-e7c6-11ee-9404-00620b2c9060', 'Ich verfügte bereits vor dem Veranstaltungsbesuch über umfangreiches Wissen zu den in der Lehrveranstaltung behandelten Themengebieten', 'scale_applies', NULL),
 ('fb98b420-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende stellt zu Beginn einer Sitzung den Zusammenhang zur letzen Sitzung her', 'scale_applies', 'A1'),
-('fb98d60e-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende gibt mir ein konstruktives Feedback zu meinen Beitragen/Lösungen', 'scale_applies', 'A2'),
+('fb98d60e-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende gibt mir ein konstruktives Feedback zu meinen Beiträgen/Lösungen', 'scale_applies', 'A2'),
 ('fb98f563-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende stellt immer wieder Bezüge zu bereits behandelten Lerninhalten her', 'scale_applies', 'A1'),
 ('fb9915da-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende fesselt die Studierenden durch eine anregende und engagierte Vortragsweise', 'scale_applies', 'B2'),
 ('fb993295-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende setzt Modelle, Graphiken oder Schemata so ein, dass sie das Verständnis komplexer Sachverhalte erleichtern', 'scale_applies', 'A1'),
@@ -202,8 +202,8 @@ INSERT INTO "question" ("id", "content", "possibilities", "dimension") VALUES
 ('fb9bd3e1-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende ist in der Lage, die Studierenden für die in der Lehrveranstaltung behandelten Inhalte zu interessieren', 'scale_applies', 'B2'),
 ('fb9bf803-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende gibt den Studierenden zu wenige Rückmeldungen zu ihren Beiträgen/Antworten', 'scale_applies', 'A2'),
 ('fb9c1da7-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende ist bei Fragen gut erreichbar', 'scale_applies', 'A2'),
-('fb9c4149-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende nutzt die Verfügbare Zeit effektiv', 'scale_applies', 'C2'),
-('fb9c6523-e7c6-11ee-9404-00620b2c9060', 'Ich habe an mindestens zwei Drittel der Termine dieser Lehrveranstaltung teilgenommen bzw. die Inhalte Lerneinheiten bearbeitet', 'y_n', NULL),
+('fb9c4149-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende nutzt die verfügbare Zeit effektiv', 'scale_applies', 'C2'),
+('fb9c6523-e7c6-11ee-9404-00620b2c9060', 'Ich habe an mindestens zwei Drittel der Termine dieser Lehrveranstaltung teilgenommen bzw. die Inhalte der Lerneinheiten bearbeitet', 'y_n', NULL),
 ('fb9c8f5a-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende verdeutlicht den Anwendungsbezug der Lerninhalte', 'scale_applies', 'B2'),
 ('fb9cb6cb-e7c6-11ee-9404-00620b2c9060', 'Der/die Lehrende regt die Studierenden dazu an, die Richtigkeit ihrer Beitrage/Antworten selbst zu überprüfen', 'scale_applies', 'B2'),
 ('fb9cdee4-e7c6-11ee-9404-00620b2c9060', 'Der Schwierigkeitsgrad der Veranstaltung ist für mich', 'scale_height', 'A1'),

src/lib/contants.py → src/lib/constants.py

@@ -8,4 +8,8 @@ MATHEMATICS_VALUES = os.path.join(RAW_DATA_PATH, "Mathematik")
 CLEAN_DATA_PATH = os.path.normpath(os.path.join(os.getcwd(), "..", "clean_data"))
 OUT_PATH = os.path.normpath(os.path.join(os.getcwd(), '..', "outputs"))
 
-INSTITUTES = ["cs", "math", "bio"]
+INSTITUTE_MAP = {
\ No newline at end of file
+	"Bioinformatik": "bio",
+	"Mathematik": "math",
+	"Informatik": "cs"
+}
\ No newline at end of file

src/lib/scraping.py

 import requests, re
 from bs4 import BeautifulSoup
+from typing import Union
 
 # In the VV the Ids for the Institutes are fixed every semester
 INSTITUTE_IDS = {
@@ -134,6 +135,7 @@ def append_dict(dictionary, key, value) -> dict:
 	res = dictionary
 	res[key] = value
 	return res
+
 def get_catalogue_for(semester: str) -> dict:
 	cs = get_complete_catalogue_for(semester, "cs")
 	cs = {k: append_dict(v, "institute", "cs") for k, v in cs.items()}
@@ -147,3 +149,21 @@ def get_catalogue_for(semester: str) -> dict:
 	merge = math | bio | cs
 
 	return merge
+
+
+def get_course(id: str) -> Union[dict, None]:
+	search_url = "https://www.fu-berlin.de/vv/de/search?utf8=✓&query={0}".format(id)
+	res = requests.get(search_url)
+
+	soup = BeautifulSoup(res.text, 'html.parser')
+
+	main_content = soup.select('#main_content .well')[0]
+	name = main_content.select("h1")[0].text
+	if name == "Erweiterte Suche": return None
+	
+	ret = {
+		"id": id,
+		"name": main_content.select("h1")[0].text,
+		"type": main_content.select(".label_container span")[0].text.strip()
+	}
+	return ret
\ No newline at end of file

src/pipeline.ipynb

@@ -2,14 +2,14 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 155,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [],
    "source": [
     "# Imports and constants\n",
     "import os, pandas\n",
     "from lib.scraping import get_catalogue_for\n",
-    "import lib.contants as CONSTANTS\n",
+    "import lib.constants as CONSTANTS\n",
     "\n",
     "# Add semester here in the form of \"Sommersemester YYYY\" or \"Wintersemester YYYY/YY\"\n",
     "# e.g.: Sommersemester 2023 or Wintersemester 2022/23\n",
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 156,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -63,7 +63,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 157,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -84,7 +84,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 158,
+   "execution_count": 17,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -154,7 +154,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 159,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -174,7 +174,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 160,
+   "execution_count": 19,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -190,7 +190,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 161,
+   "execution_count": 20,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -206,7 +206,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 162,
+   "execution_count": 21,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -222,7 +222,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 163,
+   "execution_count": 22,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -255,7 +255,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 164,
+   "execution_count": 23,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -275,7 +275,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 165,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -307,7 +307,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 166,
+   "execution_count": 25,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -339,37 +339,9 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 167,
+   "execution_count": 27,
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "text/plain": [
-       "71"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": [
-       "77"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "text/plain": [
-       "159"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "map = {\n",
     "\t\"Bioinformatik\": \"bio\",\n",
@@ -396,7 +368,7 @@
     "for k, v in type_map.items():\n",
     "\n",
     "\tregistered = all_registered[all_registered[\"Kategorie\"] == v]\n",
-    "\tdisplay(len(registered))\n",
+    "\n",
     "\tcollegial_num = collegial[collegial[\"Kategorie\"] == v]\n",
     "\tless_five = less_five_registered[less_five_registered[\"Kategorie\"] == v]\n",
     "\t# First key\n",

tests/scraping/get_from_id/test_function_get_from_id.py

+import os, sys
+
+from schema import Schema, Regex
+
+# Makes the lib folder visible
+path = os.path.abspath(__file__)
+steps = 4
+for i in range(steps):
+	path = os.path.dirname(path)
+
+sys.path.append(path)
+import src.lib.scraping as scraping
+
+def test_valid_id():
+	id = "19201401"
+
+	result_schema = Schema({
+		'id': str,
+		'name': str,
+		'type': str
+	})
+
+	res = scraping.get_course(id)
+	assert result_schema.is_valid(res)
+
+def test_invalid_id():
+	res = scraping.get_course("0000000")
+	assert res == None
\ No newline at end of file