Implemented Scraper

Co-authored-by: Dominik Pietrak <dominip89@mi.fu-berlin.de>

admin-frontend/src/App.js

@@ -6,10 +6,13 @@ import dataProviderMapper from './dataProviderMapper';
 import { SportList, SportEdit, SportCreate } from './sportList';
 import { IncompleteList, IncompleteEdit } from './incompleteList';
 import { QuestionList, QuestionEdit, QuestionCreate } from './questionList';
+import { ScraperList } from './scraperList';
 
 import QuestionAnswerIcon from '@material-ui/icons/QuestionAnswer';
 import SportsFootballIcon from '@material-ui/icons/SportsFootball';
-import PlaylistAddCheckIcon from '@material-ui/icons/PlaylistAddCheck';
+import ListAltIcon from '@material-ui/icons/ListAlt';
+import YoutubeSearchedForIcon from '@material-ui/icons/YoutubeSearchedFor';
+
 
 const App = () => (
   <Admin
@@ -28,7 +31,7 @@ const App = () => (
 
     <Resource
       name='sport-incomplete'
-      icon={PlaylistAddCheckIcon}
+      icon={ListAltIcon}
       options={{ label: 'Incomplete Sports' }}
       list={IncompleteList}
       edit={IncompleteEdit}
@@ -42,6 +45,13 @@ const App = () => (
       edit={QuestionEdit}
       create={QuestionCreate}
     />
+
+    <Resource
+      name='sport-scraper'
+      icon={YoutubeSearchedForIcon}
+      options={{ label: 'Sportarten auslesen' }}
+      list={ScraperList}
+    />
   </Admin>
 );
 

admin-frontend/src/dataProviderMapper.js

 import drfProvider from 'ra-data-django-rest-framework';
 import sportIncompleteProvider from './sportIncompleteProvider.js';
+import scraperDataProvider from './scraperDataProvider.js';
 import {
     GET_LIST,
     GET_ONE,
@@ -9,6 +10,7 @@ import {
     DELETE,
     GET_MANY,
     GET_MANY_REFERENCE,
+    DELETE_MANY
 } from 'react-admin';
 
 // Mapping of data providers onto resource names
@@ -22,6 +24,10 @@ const dataProviders = [
     {
         dataProvider: sportIncompleteProvider('http://localhost:8000/api/admin'),
         resources: ['sport-incomplete'],
+    },
+    {
+        dataProvider: scraperDataProvider('http://localhost:8000/api/admin'),
+        resources: ['sport-scraper'],
     }
 ];
 
@@ -32,6 +38,7 @@ export default (type, resource, params) => {
         dp.resources.includes(resource));
 
     // Maps the type of request onto the function of the data provider
+    // "post_scraper" is a custom function to conform with the API endpoint which doesn't conform with normal CRUD operations
     const mappingType = {
         [GET_LIST]: 'getList',
         [GET_ONE]: 'getOne',
@@ -41,10 +48,10 @@ export default (type, resource, params) => {
         [UPDATE]: 'update',
         [UPDATE_MANY]: 'updateMany',
         [DELETE]: 'delete',
+        [DELETE_MANY]: 'deleteMany',
+        ["post_scraper"]: 'post_scraper',
     };
 
-    // Debugging, yaay
-    console.log(type, resource, params)
 
     return dataProviderMapping.dataProvider[mappingType[type]](resource, params);
 };
\ No newline at end of file

admin-frontend/src/defaultDataProvider.js

+import { stringify } from 'query-string';
+import {
+    fetchUtils,
+} from 'ra-core';
+
+/*
+DEFAULT DATA PROVIDER TEMPLATE
+Just for copy and pasting when other data providers are needed.
+*/
+
+
+export {
+    default as tokenAuthProvider,
+    fetchJsonWithAuthToken,
+} from './tokenAuthProvider';
+
+export {
+    default as jwtTokenAuthProvider,
+    fetchJsonWithAuthJWTToken,
+} from './jwtTokenAuthProvider';
+
+const getPaginationQuery = (pagination) => {
+    return {
+        page: pagination.page,
+        page_size: pagination.perPage,
+    };
+};
+
+const getFilterQuery = (filter) => {
+    const { q: search, ...otherSearchParams } = filter;
+    return {
+        ...otherSearchParams,
+        search,
+    };
+};
+
+export const getOrderingQuery = (sort) => {
+    const { field, order } = sort;
+    return {
+        ordering: `${order === 'ASC' ? '' : '-'}${field}`,
+    };
+};
+
+export default (
+    apiUrl,
+    httpClient = fetchUtils.fetchJson
+) => {
+    const getOneJson = (resource, id) =>
+        httpClient(`${apiUrl}/${resource}/${id}/`).then(
+            (response) => response.json
+        );
+
+    return {
+        getList: async (resource, params) => {
+            const query = {
+                ...getFilterQuery(params.filter),
+                ...getPaginationQuery(params.pagination),
+                ...getOrderingQuery(params.sort),
+            };
+            const url = `${apiUrl}/${resource}/?${stringify(query)}`;
+
+            const { json } = await httpClient(url);
+
+            return {
+                data: json.results,
+                total: json.count,
+            };
+        },
+
+        getOne: async (resource, params) => {
+            const data = await getOneJson(resource, params.id);
+            return {
+                data,
+            };
+        },
+
+        getMany: (resource, params) => {
+            return Promise.all(
+                params.ids.map(id => getOneJson(resource, id))
+            ).then(data => ({ data }));
+        },
+
+        getManyReference: async (resource, params) => {
+            const query = {
+                ...getFilterQuery(params.filter),
+                ...getPaginationQuery(params.pagination),
+                ...getOrderingQuery(params.sort),
+                [params.target]: params.id,
+            };
+            const url = `${apiUrl}/${resource}/?${stringify(query)}`;
+
+            const { json } = await httpClient(url);
+            return {
+                data: json.results,
+                total: json.count,
+            };
+        },
+
+        update: async (resource, params) => {
+            const { json } = await httpClient(`${apiUrl}/${resource}/${params.id}/`, {
+                method: 'PATCH',
+                body: JSON.stringify(params.data),
+            });
+            return { data: json };
+        },
+
+        updateMany: (resource, params) =>
+            Promise.all(
+                params.ids.map(id =>
+                    httpClient(`${apiUrl}/${resource}/${id}/`, {
+                        method: 'PATCH',
+                        body: JSON.stringify(params.data),
+                    })
+                )
+            ).then(responses => ({ data: responses.map(({ json }) => json.id) })),
+
+        create: async (resource, params) => {
+            const { json } = await httpClient(`${apiUrl}/${resource}/`, {
+                method: 'POST',
+                body: JSON.stringify(params.data),
+            });
+            return {
+                data: { ...json },
+            };
+        },
+
+        delete: (resource, params) =>
+            httpClient(`${apiUrl}/${resource}/${params.id}/`, {
+                method: 'DELETE',
+            }).then(() => ({ data: params.previousData })),
+
+        deleteMany: (resource, params) =>
+            Promise.all(
+                params.ids.map(id =>
+                    httpClient(`${apiUrl}/${resource}/${id}/`, {
+                        method: 'DELETE',
+                    })
+                )
+            ).then(responses => ({ data: responses.map(({ json }) => json.id) })),
+    };
+};

admin-frontend/src/incompleteList.js

@@ -63,9 +63,9 @@ const IncompleteEditToolbar = props => (
 
 
 export const IncompleteList = props => (
-    <List {...props} aside={<AsideIncompleteList />}>
+    <List {...props} aside={<AsideIncompleteList />} bulkActionButtons={false}>
         {/* rowClick expand means that no edit page is opened, but instead shown below the data row itself without reloading */}
-        <Datagrid rowClick="expand" expand={IncompleteEdit} >
+        <Datagrid rowClick="expand" expand={IncompleteEdit} style={{ tableLayout: "fixed", }} >
 
             {/* Reference Field fetches every Sport object if a reference field to it exists.
             This means that n GETs are made for one load of the incomplete-list.

admin-frontend/src/postScraperButton.js

+import * as React from 'react';
+import { useDataProvider, useNotify, useRedirect, Button } from 'react-admin';
+
+export const PostScraperButton = ({ selectedIds }) => {
+    const notify = useNotify();
+    const redirect = useRedirect();
+    const dataProvider = useDataProvider();
+
+    const postToScraper = () => dataProvider.post_scraper('sport-scraper', selectedIds).then(
+        response => {
+            notify("Ausgewählte Sportarten wurden eingefügt");
+            redirect("sport-incomplete/")
+        }).catch(error => {
+            notify("Etwas ist schief gelaufen!", "warning");
+        });
+
+    return <Button label="Ausgewählte Sportarten hinzufügen" onClick={postToScraper} />
+};
+
+export default PostScraperButton
\ No newline at end of file

admin-frontend/src/questionList.js

@@ -87,7 +87,7 @@ const AsideQuestionCreate = () => (
 
 export const QuestionList = props => (
     <List {...props} aside={<AsideQuestionList />}>
-        <Datagrid rowClick="edit">
+        <Datagrid rowClick="edit" style={{ tableLayout: "fixed", }}>
             <TextField source="text_de" label="Deutscher Fragetext" />
             <TextField source="text_en" label="Englischer Fragetext" />
             <TextField source="criterion" label="Name des Kriteriums" />

admin-frontend/src/scraperDataProvider.js

+import { stringify } from 'query-string';
+import {
+    fetchUtils,
+} from 'ra-core';
+
+
+const getPaginationQuery = (pagination) => {
+    return {
+        page: pagination.page,
+        page_size: pagination.perPage,
+    };
+};
+
+const getFilterQuery = (filter) => {
+    const { q: search, ...otherSearchParams } = filter;
+    return {
+        ...otherSearchParams,
+        search,
+    };
+};
+
+export const getOrderingQuery = (sort) => {
+    const { field, order } = sort;
+    return {
+        ordering: `${order === 'ASC' ? '' : '-'}${field}`,
+    };
+};
+
+
+/*
+WHAT HAS CHANGED IN THIS DATA PROVIDER
+This Data Provider is meant for use of the Sport Scraper.
+The only thing changed here is the "create" function. It has been replaced with a "post_scraper" function, which sends every ticked row to the API.
+The change of the function name has been done because react-admin demands a certain kind of answer with the created object.
+Since this isn't given by the scraper (completely different usecase after all), we had to circumvent this check.
+
+Since the data of the checked rows isn't given, but only the IDs, we need to cache the fetched list of diffs so that it can still be accessed by post_scraper.
+Same trick as in incompleteListProvider, we just save it in a local variable.
+*/
+
+
+
+let cached_scraped_sport_list = []
+
+export default (
+    apiUrl,
+    httpClient = fetchUtils.fetchJson
+) => {
+    const getOneJson = (resource, id) =>
+        httpClient(`${apiUrl}/${resource}/${id}/`).then(
+            (response) => response.json
+        );
+
+    return {
+        getList: async (resource, params) => {
+            const query = {
+                ...getFilterQuery(params.filter),
+                ...getPaginationQuery(params.pagination),
+                ...getOrderingQuery(params.sort),
+            };
+            const url = `${apiUrl}/${resource}/?${stringify(query)}`;
+
+            const { json } = await httpClient(url);
+
+            cached_scraped_sport_list = json.results;
+
+            return {
+                data: json.results,
+                total: json.count,
+            };
+        },
+
+        getOne: async (resource, params) => {
+            const data = await getOneJson(resource, params.id);
+            return {
+                data,
+            };
+        },
+
+        getMany: (resource, params) => {
+            return Promise.all(
+                params.ids.map(id => getOneJson(resource, id))
+            ).then(data => ({ data }));
+        },
+
+        getManyReference: async (resource, params) => {
+            const query = {
+                ...getFilterQuery(params.filter),
+                ...getPaginationQuery(params.pagination),
+                ...getOrderingQuery(params.sort),
+                [params.target]: params.id,
+            };
+            const url = `${apiUrl}/${resource}/?${stringify(query)}`;
+
+            const { json } = await httpClient(url);
+            return {
+                data: json.results,
+                total: json.count,
+            };
+        },
+
+        update: async (resource, params) => {
+            const { json } = await httpClient(`${apiUrl}/${resource}/${params.id}/`, {
+                method: 'PATCH',
+                body: JSON.stringify(params.data),
+            });
+            return { data: json };
+        },
+
+        updateMany: (resource, params) =>
+            Promise.all(
+                params.ids.map(id =>
+                    httpClient(`${apiUrl}/${resource}/${id}/`, {
+                        method: 'PATCH',
+                        body: JSON.stringify(params.data),
+                    })
+                )
+            ).then(responses => ({ data: responses.map(({ json }) => json.id) })),
+
+        post_scraper: async (resource, params) => {
+
+            let sport_list;
+
+            if (cached_scraped_sport_list === []) {
+                sport_list = this.getList(resource, { filter: {}, pagination: { page: 1, perPage: 1000 }, sort: { field: "id", order: "ASC" } });
+            } else {
+                sport_list = cached_scraped_sport_list;
+            }
+
+            let checked_sport_list = [];
+
+            params.forEach(id => {
+                checked_sport_list.push(sport_list[id - 1]);
+            });
+
+            const { json } = await httpClient(`${apiUrl}/sport-scraper/`, {
+                method: 'POST',
+                body: JSON.stringify(checked_sport_list),
+            });
+
+            return {
+                data: { ...json },
+            };
+        },
+
+        delete: (resource, params) =>
+            httpClient(`${apiUrl}/${resource}/${params.id}/`, {
+                method: 'DELETE',
+            }).then(() => ({ data: params.previousData })),
+
+        deleteMany: (resource, params) =>
+            Promise.all(
+                params.ids.map(id =>
+                    httpClient(`${apiUrl}/${resource}/${id}/`, {
+                        method: 'DELETE',
+                    })
+                )
+            ).then(responses => ({ data: responses.map(({ json }) => json.id) })),
+    };
+};

admin-frontend/src/scraperList.js

+import {
+    List,
+    Datagrid,
+    TextField,
+    NumberField,
+    DateField,
+} from 'react-admin';
+
+import Typography from '@material-ui/core/Typography';
+import Card from '@material-ui/core/Card';
+import CardContent from '@material-ui/core/CardContent';
+
+import PostScraperButton from './postScraperButton';
+
+const AsideSportScrape = () => (
+    <div style={{ width: 400, margin: '1em' }}>
+        <Card>
+            <CardContent>
+                <Typography variant="h4" align="center">Tipps&Tricks</Typography>
+                <br />
+                <ul>
+                    <li>
+                        <p>
+                            Hier sind alle Sportarten gelistet, die gerade in <a href="https://www.buchsys.de/fu-berlin/angebote/aktueller_zeitraum/index.html">buchsys.de</a> stehen.
+                        </p>
+                        <p>Diese werden verglichen mit den Sportarten, die momentan in der Sportliste stehen. Hierbei kann es zu vier verschiedenen Fällen kommen:</p>
+                        <ul>
+                            <li><i>Same:</i> Der gefundene Sport steht bereits in der Datenbank und wird bis auf die URL nicht aktualisiert.</li>
+                            <li><i>New:</i> Der gefundene Sport existiert in der Datenbank noch nicht und wird vollkommen neu angelegt. Die Kriterien müssen hier noch ausgefüllt werden.</li>
+                            <li><i>From Archive:</i> Der gefundene Sport existiert zwar, liegt momentan aber in dem Archiv. Der Sport wird wieder aktiviert werden, und die alten Kriterienbewertungen werden von diesem Sport übernommen werden. </li>
+                            <li><i>To be archived:</i> Es wurde kein neuer Sport gefunden, der dem Sport in der Datenbank zu gleichen scheint, also wird der Sport in das Archiv geschoben, um nicht mehr als Sportempfehlung angezeigt zu werden.</li>
+                        </ul>
+                    </li>
+                    <li>
+                        Die Sportarten können mit der Checkbox links ausgewählt werden. Soweit mindestens ein Sport ausgewählt wurde, erscheint der Knopf <b>Ausgewählte Sportarten hinzufügen</b>, um die ausgewählten Änderungen anzuwenden und zu den Incomplete Sports zu springen.
+                    </li>
+                </ul>
+            </CardContent>
+        </Card>
+    </div>
+);
+
+{/* bulkActionButtons should theoretically be able to take a whole Fragment worth of buttons, but React hates us and we return the feeling*/ }
+
+// perPage is set to 1000
+// It is important that the page shows all entries, so that there are no issues while scrolling through the pages and sending the data afterwards
+// So, all Diffs should be displayed on the same page
+export const ScraperList = props => (
+    <List {...props} perPage={1000} bulkActionButtons={<PostScraperButton {...props} />} aside={<AsideSportScrape />}>
+        <Datagrid rowClick="edit">
+            <TextField source="id" />
+            <TextField source="kind_of_diff" label="Art der Differenz" />
+            <NumberField source="old_sport.id" label="Alte ID" />
+            <DateField source="old_sport.last_used" label="Zuletzt aktiviert" />
+            <TextField source="old_sport.name" label="Alter Sport" />
+            <TextField source="new_sport.name" label="Neuer Sport" />
+        </Datagrid>
+    </List>
+);
\ No newline at end of file

admin-frontend/src/sportList.js

@@ -91,7 +91,8 @@ const AsideSportCreate = () => (
 
 export const SportList = props => (
     <List {...props} aside={<AsideSportList />}>
-        <Datagrid rowClick="edit">
+        <Datagrid rowClick="edit" style={{ tableLayout: "fixed", }}>
+
             <TextField source="id" />
             <TextField source="name" />
             <UrlField source="url" />